#!/usr/bin/env python
# encoding: utf-8
"""
logparser.py

This script is used for parsing the log files generated by Logger.py
for the experiment. The information in the log file can be exported into
specific format for plotting. What's more, the summary information for
the experiment will also be generated in human-readable format.

Created by Ryan on 2010-05-28.
Copyright (c) 2010 Liang Wang. All rights reserved.
"""

import sys, os, random
import time, pickle
import math, re

class LogParser(object):
    def __init__(self, log_path):
        try:
            self._peers = list()
            self._log = pickle.Unpickler(open(log_path, 'r')).load()
        except:
            self._log = None
            print("Error: "+log_path+" is corrupted!")
        pass

    def get_peer_info(self):
        return self._log

    pass


class Experiment(object):
    """
    Remark: snapshot should be False by default. Dealing with snapshots log is very time-comsuming
    """
    def __init__(self, experiment_dir, snapshot=True):
        self._dir = experiment_dir
        self._peers = list()
        self._snapshot = snapshot
        self._connr_unchoked = {}
        self._connr_transfer = {}
        self._connr_all = {}
        os.path.walk(self._dir, self.process_one_peer, ())
        
        if snapshot:
            self.snapshot_done()
        pass


    def process_one_peer(self, arg, dirname, names):
        for file in names:
            if file == 'summary.log':
                parser = LogParser(os.path.join(dirname, file))
                peer = parser.get_peer_info()
                if peer:
                    self._peers += [ peer ]

        if self._snapshot and os.path.exists(os.path.join(dirname, 'snapshots.log')) and peer:
            try:
                self.process_snapshot(dirname, os.path.join(dirname, 'snapshots.log'), peer)
            except:
                print "Error: " + os.path.join(dirname, 'snapshots.log') + " is corrupted!"
        pass


    def snapshot_done(self):
        avg_connr = open(os.path.join(self._dir, 'avg_connr.txt'), 'w')
        
        avg_connr.write('Unchoked Connections\n')
        keys = self._connr_unchoked.keys()
        keys.sort()
        for k in keys:
            r = 0.0
            for x in self._connr_unchoked[k]:
                r += x
            r = r/len(self._connr_unchoked[k])
            avg_connr.write(str(k) + '->' + str(r) + '\n')
            
        avg_connr.write('Transfer Connections\n')
        keys = self._connr_transfer.keys()
        keys.sort()
        for k in keys:
            r = 0.0
            for x in self._connr_transfer[k]:
                r += x
            r = r/len(self._connr_transfer[k])
            avg_connr.write(str(k) + '->' + str(r) + '\n')
            
        avg_connr.write('All Connections\n')
        keys = self._connr_all.keys()
        keys.sort()
        for k in keys:
            r = 0.0
            for x in self._connr_all[k]:
                r += x
            r = r/len(self._connr_all[k])
            avg_connr.write(str(k) + '->' + str(r) + '\n')
        pass


    def process_snapshot(self, dirname, file, peer):
        print("Processing " + file)
        peer_id = peer['peer_id']
        my_ip = peer['ip']
        
        snapshots_dir = os.path.join(dirname, '../snapshots')
        if not os.path.exists(snapshots_dir):
            os.mkdir(snapshots_dir, 0700)
        parser = LogParser(file)
        snapshots = parser.get_peer_info()
        
        ul_rate = open(os.path.join(snapshots_dir, peer_id+'_ul_rate.txt'), 'w')
        dl_rate = open(os.path.join(snapshots_dir, peer_id+'_dl_rate.txt'), 'w')
        share_r = open(os.path.join(snapshots_dir, peer_id+'_share_r.txt'), 'w')    # Share ratio
        connc_r = open(os.path.join(snapshots_dir, peer_id+'_connc_r.txt'), 'w')    # The ration this peer connecting to the peers on the same node
        
        unchoked_d = {} # For unchoked conns
        unchoked_t = 0.0
        transfer_d = {} # For conns which are transferring data
        transfer_t = 0.0
        allconns_d = {} # For all conns
        allconns_t = 0.0
        for shot in snapshots:
            try:
                ul_rate.write(str(shot[1]) + '\n')
                dl_rate.write(str(shot[2]) + '\n')
                share_r.write(str(shot[5]) + '\n')
                
                if shot[6]:
                    for x in shot[6]:
                        if (my_ip, x[0]) not in unchoked_d: unchoked_d[(my_ip, x[0])] = 0                      
                        unchoked_d[(my_ip, x[0])] += 1
                        unchoked_t += 1
                connc_r.write('unchoked:' + str(unchoked_d) + ' ~ ' + str(len(shot[6])) + '\t\t')
                    
                if shot[7]:
                    for x in shot[7]:
                        if (my_ip, x[0]) not in transfer_d: transfer_d[(my_ip, x[0])] = 0                      
                        transfer_d[(my_ip, x[0])] += 1
                        transfer_t += 1
                connc_r.write('transfer:' + str(transfer_d) + ' ~ ' + str(len(shot[7])) + '\t\t')

                if shot[8]:
                    for x in shot[8]:
                        if (my_ip, x[0]) not in allconns_d: allconns_d[(my_ip, x[0])] = 0                      
                        allconns_d[(my_ip, x[0])] += 1
                        allconns_t += 1
                connc_r.write('all:' + str(allconns_d) + ' ~ ' + str(len(shot[8])) + '\n')
            except:
                print "Error: snapshot data error!"


        for k in unchoked_d:
            if k not in self._connr_unchoked: self._connr_unchoked[k] = []
            self._connr_unchoked[k] += [unchoked_d[k]/unchoked_t]

        for k in transfer_d:
            if k not in self._connr_transfer: self._connr_transfer[k] = []
            self._connr_transfer[k] += [transfer_d[k]/transfer_t]

        for k in allconns_d:
            if k not in self._connr_all: self._connr_all[k] = []
            self._connr_all[k] += [allconns_d[k]/allconns_t]

        pass


    def export(self):
        avg_ul_rate_for_peer = ''
        avg_dl_rate_for_peer = ''
        avg_both_rate_for_peer = ''
        avg_both_rate_for_sys = ''
        avg_ul_sys_a = 0; avg_ul_sys_b = 0;
        avg_dl_sys_a = 0; avg_dl_sys_b = 0;
        var_ul_sys = 0; var_dl_sys = 0;
        all_info_for_peer = ''
        for peer in self._peers:
            avg_ul_rate_for_peer += str(peer['avg_ul_rate']/1024) + '\n'
            avg_dl_rate_for_peer += str(peer['avg_dl_rate']/1024) + '\n'
            avg_both_rate_for_peer += str(peer['avg_ul_rate']/1024) + '\t' + str(peer['avg_dl_rate']/1024) + '\n'
            all_info_for_peer += time.strftime("%H:%M:%S",time.localtime(peer['start'])) + '\t' + \
                                time.strftime("%H:%M:%S",time.localtime(peer['finish'])) + '\t' + \
                                str(int(peer['avg_ul_rate']/1024)) + '\t' + str(int(peer['avg_dl_rate']/1024)) + '\n'
            
            avg_ul_sys_a += peer['avg_ul_rate']/1024; avg_ul_sys_b += 1;
            var_ul_sys += (peer['avg_ul_rate']/1024)**2
            if int(peer['avg_dl_rate']) > 0:
                avg_dl_sys_a += peer['avg_dl_rate']/1024; avg_dl_sys_b += 1;
                var_dl_sys += (peer['avg_dl_rate']/1024)**2
                
        avg_both_rate_for_sys += str(avg_ul_sys_a/avg_ul_sys_b) + '\t' + str(avg_dl_sys_a/avg_dl_sys_b)
        if avg_dl_sys_b > 1:  # sample standard deviation
            var_dl_sys = math.sqrt((var_dl_sys - avg_dl_sys_b*(avg_dl_sys_a/avg_dl_sys_b)**2)/(avg_dl_sys_b - 1))
        else:
            var_dl_sys = 0
        avg_both_rate_for_sys += '\n[sample std for dl]\t' + str(var_dl_sys)

        open(os.path.join(self._dir, 'avg_ul_rate_for_peer.txt'), 'w').write(avg_ul_rate_for_peer)
        open(os.path.join(self._dir, 'avg_dl_rate_for_peer.txt'), 'w').write(avg_dl_rate_for_peer)
        open(os.path.join(self._dir, 'avg_both_rate_for_peer.txt'), 'w').write(avg_both_rate_for_peer)
        open(os.path.join(self._dir, 'avg_both_rate_for_sys.txt'), 'w').write(avg_both_rate_for_sys)
        open(os.path.join(self._dir, 'all_info_for_peer.txt'), 'w').write(all_info_for_peer)
        pass

    pass



def process_connr(connr_file, folder_path):
    if not os.path.isdir(folder_path):
        os.mkdir(folder_path)

    d_i = ""
    prog=re.compile(r"^\(\'([^\']*)\'[^\']*\'([^\']*)\'[^>]*>(.*)")
    lines = open(connr_file).readlines()
    
    for line in lines:
        if line.startswith("Unchoked Connections"):
            d_i = "unchoked_"
        elif line.startswith("Transfer Connections"):
            d_i = "transfer_"
        elif line.startswith("All Connections"):
            d_i = "allconns_"
        else:
            m = prog.search(line)
            fname = d_i + m.group(1) + "-" + m.group(2) + ".txt"
            open(os.path.join(folder_path, fname), "a").write(m.group(3) + "\n")

    pass
    

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: python logparser.py *experiment folder* [-snapshot|...]")
        sys.exit(1)

    if not os.path.exists(sys.argv[1]):
        print("Experiment folder dosen't exist!")
        sys.exit(1)
    
    snapshot = False
    if "-snapshot" in sys.argv[2:]:
        if int(sys.argv[2:][sys.argv[2:].index("-snapshot") + 1]) > 0:
            snapshot = True

    folders = os.listdir(sys.argv[1])
    folders.sort()

    batch_folder = False
    for folder in folders:
        if folder.startswith("data-"):
            batch_folder = True # It's a batch folder
            break

    if batch_folder:    # For batch exp folder
        summary_dl_sys = ''
        for folder in folders:
            exp_path = os.path.join(sys.argv[1], folder)
            if os.path.isdir(exp_path) and folder.startswith("data-"):
                exp = Experiment(exp_path, snapshot=snapshot)
                exp.export()
                s = open(os.path.join(exp_path, 'avg_both_rate_for_sys.txt'), 'r').readlines()
                summary_dl_sys += s[0].split('\t')[1]
                
                if snapshot:
                    process_connr(os.path.join(exp_path, 'avg_connr.txt'), os.path.join(sys.argv[1], 'summary_connr'))
                
        open(os.path.join(sys.argv[1], 'summary_dl_sys.txt'), 'w').write(summary_dl_sys)
    else:               # For single exp folder
        exp = Experiment(sys.argv[1], snapshot=snapshot)
        exp.export()

    sys.exit(0)