# -*- coding:utf-8 -*-
import subprocess

read_file = 'enwiki_conv_3.txt'
o1 = 't_c_occurrence.txt'
o2 = 't_appear.txt'
o3 = 'c_appear.txt'

def _exec_cmd(cmd):
    proc = subprocess.Popen(cmd,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT
    )
    return proc

def t_c_occurrence_cnt():
    occur_cnt_dict = {}
    with open(read_file, 'r') as fr:
        for line in fr.readlines():
            line = line.split()
            word, context_word = line[0], line[1]
            if word not in occur_cnt_dict:
                occur_cnt_dict[word] = {}
            if context_word not in occur_cnt_dict[word]:
                occur_cnt_dict[word][context_word] = 1
            else:
                occur_cnt_dict[word][context_word] += 1
    for t, cdict in occur_cnt_dict.items():
        fw = open(o1, 'a')
        for c, cnt in cdict.items():
            s = '%s\t%s\t%s\n' % (t, c, cnt)
            fw.write(s)
        fw.close()

def t_appear_cnt():
    cmd = "cut -f1 %s | sort -k 1 | uniq -c | sort -nr" % (o1)
    proc = _exec_cmd(cmd)
    for s in proc.stdout.readlines():
        fw = open(o2, 'a')
        fw.write(s)
        fw.close()

def c_appear_cnt():
    cmd = "cut -f2 %s | sort -k 1 | uniq -c | sort -nr" % (o1)
    proc = _exec_cmd(cmd)
    for s in proc.stdout.readlines():
        fw = open(o3, 'a')
        fw.write(s)
        fw.close()

def get_pair_num():
    cmd = "wc -l %s | cut -d ' ' -f2" % (o1)
    proc = _exec_cmd(cmd)
    stdout, stderr = proc.communicate()
    return stdout

def main():
    t_c_occurrence_cnt()
    t_appear_cnt()
    c_appear_cnt()
    total_num = get_pair_num()
    print total_num

if __name__ == '__main__':
    main()
