import sys
import math
from csv import reader
from sklearn.metrics import mutual_info_score
from collections import OrderedDict
from sklearn import cross_validation as cv
import random
import numpy as np
import csv
from sklearn import svm, metrics, preprocessing
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import StratifiedKFold
from sklearn.svm import NuSVC
from itertools import chain, repeat

csv.field_size_limit(sys.maxsize)
with open('CSDMC_API_Train.csv', 'rb') as f:
    reader = csv.reader(f)
    your_list = list(reader)
	
#print your_list[0][0]
print len(your_list)
#print math.log(2)
#print your_list[0][0]
#print your_list[1][0]

list_len = len(your_list)
dic = [dict() for x in range(list_len)]

for i in range(0,list_len):
	text = your_list[i][1]
	for w in text.split():
		if w in dic[i].keys():
                        
			dic[i][w] = dic[i][w]+1
		else:
			dic[i][w] = 1

tf = [dict() for x in range(list_len)]
for i in range (0,list_len):
	for key in dic[i].keys():
		tf[i][key]= float(dic[i][key])/sum(dic[i].values())
		tf[i][key]=math.sqrt(tf[i][key])
		
#print "tf value of HeapAlloc in first document is",tf[0]['HeapAlloc']


		

tokens = dict()
for i in range(0,list_len):
	for k in dic[i].keys():
		if k in tokens.keys():
			continue
		else:
			tokens[k]=1
			
			
d_len = len(dic)
doc_tok = dict()
for d in range(0,d_len):
	for t in tokens.keys():
		if t in dic[d]:
			if t in doc_tok.keys():
				doc_tok[t] = doc_tok[t]+1
			else:
				doc_tok[t] = 1
		else:
			continue

idf = dict()
for k in doc_tok.keys():
	idf[k] = float(len(dic))/doc_tok[k]
	idf[k] = math.log(idf[k])
	
tf_idf = [dict() for x in range(list_len)]
for i in range(0,list_len):
	for k in tf[i].keys():
		tf_idf[i][k] = tf[i][k]*idf[k]
		

csv_dict = [dict() for x in range(list_len)]
for i in range(0,list_len):
	for k in tokens.keys():
		if k in tf_idf[i].keys():
			csv_dict[i][k] = tf_idf[i][k]
		else:
			csv_dict[i][k] = 0
			
for i in range(0,list_len):
	csv_dict[i]['class'] = your_list[i][0]
	




keys = csv_dict[0].keys()
with open('features.csv', 'wb') as output_file:
    dict_writer = csv.DictWriter(output_file, keys)
    dict_writer.writeheader()
    dict_writer.writerows(csv_dict)

ld=dict()
for i in csv_dict[0].keys():
	ld[i] = []
	for j in range(0,list_len):
		ld[i].append(csv_dict[j][i])

#print ld['HeapAlloc']

mi = dict()
for i in csv_dict[0].keys():
	if i == 'class':
		continue
	else:
		mi[i] = mutual_info_score(ld[i], ld['class'])

#print mi
mi1 = []

for key, value in sorted(mi.iteritems(), key=lambda (k,v): (v,k),reverse=True):
    #mi1[key] = value
    #print "%s: %s" % (key, value)
    mi1.append(key)

#mi1 = sorted(mi.iteritems(), key=lambda (k,v): (v,k),reverse=True)
#print mi1


dset = [OrderedDict() for x in range(list_len)]
for i in mi1[:10]:
	for j in range(0,list_len):
		dset[j][i] = csv_dict[j][i]

for i in range(0,list_len):
	dset[i]['class'] = your_list[i][0]

print dset[0].keys()





