# -*- coding: utf-8 -*-
import math
from scipy import io, sparse

t_c_occur = 't_c_occurrence.txt'
t_app = 't_appear.txt'
c_app = 'c_appear.txt'
cul_ppmi = 'ppmi.txt'

threshold = 10
n = 7339305 

def make_appear_dict(input_file):
    appear_dict = {}
    with open(input_file, 'r') as fr:
        for line in fr.readlines():
            line = line.rstrip('\n').split()
            cnt, word = line[0], line[1]
            appear_dict[word] = cnt
    return appear_dict

def culculate_ppmi(t_dict, c_dict):
    input_file = t_c_occur
    output_file = cul_ppmi

    fr = open(input_file, 'r')
    for line in fr.readlines():
        line = line.rstrip('\n').split('\t')
        t_word, c_word, occur_cnt = line[0], line[1], int(line[2])
        if occur_cnt < threshold:
            continue
        ppmi = max(math.log((float(n) * float(occur_cnt)) / (float(t_dict[t_word]) * float(c_dict[c_word]))), 0)
        fw = open(output_file, 'a')
        s = '%s\t%s\t%s\n' % (t_word, c_word, ppmi)
        fw.write(s)
        fw.close()
    fr.close()

def make_word_index(input_file, num):
    word_index = {}
    count = 0
    with open(input_file, 'r') as fr:
        for line in fr.readlines():
            word = line.rstrip('\n').split('\t')[num]
            if word not in word_index:
                word_index[word] = count
                count += 1
    return word_index

def make_matrix(input_file, t_idx, c_idx):
    matrix = sparse.lil_matrix((len(t_idx), len(c_idx)))
    with open(input_file, 'r') as fr:
        for line in fr.readlines():
            line = line.rstrip('\n').split('\t')
            t, c, ppmi = line[0], line[1], float(line[2])
            matrix[t_idx[t],c_idx[c]] = ppmi
    io.savemat('matrix_a', {'matrix_a': matrix})
    print matrix
    
def main():
    #t_dict = make_appear_dict(t_app)
    #c_dict = make_appear_dict(c_app)
    #culculate_ppmi(t_dict, c_dict)
    t_idx = make_word_index(cul_ppmi, 0)
    c_idx = make_word_index(cul_ppmi, 1)
    make_matrix(cul_ppmi, t_idx, c_idx)

if __name__ == '__main__':
    main()
