# Code used in part 1 of How I used machine learning to classify emails and turn them into insights.
from collections import defaultdict
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.feature_extraction.text import TfidfVectorizer, ENGLISH_STOP_WORDS
from sklearn.pipeline import make_pipeline
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans, MiniBatchKMeans
from sklearn.manifold import TSNE
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import normalize

from helpers import *

emails = pd.read_csv('split_emails.csv')
# Lets create a new frame with the data we need.
email_df = pd.DataFrame(parse_into_emails(emails.message))

# Drop emails with empty body, to or from_ columns.
email_df.drop(email_df.query("body == '' | to == '' | from_ == '' | subject == ''").index, inplace=True)
email_df.to_csv('dataset.csv')
parsed_emails = pd.read_csv('dataset.csv')
subj_dict = defaultdict(list)
for index, row in parsed_emails.iterrows():
    if len(row['subject'].split(":")) == 1:
        if not row['subject'] in subj_dict:
            subj_dict[row['subject']].append(str(row['body']))
            for inner_index, inner_row in parsed_emails.iterrows():
                if len(inner_row['subject'].split(":")) > 1 and inner_row['subject'] == "RE:"+row['subject']:
                    subj_dict[row['subject']].append(str(inner_row['body']))
    #print subj_dict
#print "her", subj_dict

for key in subj_dict:
    if len(subj_dict[key]) > 1:
        print subj_dict[key]

json.dump(subj_dict, open("email_response.json", 'w'))

# print len(email_data.from_.unique()) # 1222 unique email addresses
# print len(email_data.to.unique())  # 1593 unique email addresses
# print email_df.head()

# At this stage we are sure we have sanitized all the data we need.
# print email_data.shape # (9464, 4)

# At this point we are going to tokenize the bodies and convert them
# into a document-term matrix.

# Some note on min_df and max_df
# max_df=0.5 means "ignore all terms that appear in more then 50% of the documents"
# min_df=2 means "ignore all terms that appear in less then 2 documents"
##stopwords = ENGLISH_STOP_WORDS.union(['ect', 'hou', 'com', 'recipient'])
##vect = TfidfVectorizer(analyzer='word', stop_words=stopwords, max_df=0.3, min_df=2)

##X = vect.fit_transform(email_df.body)
##features = vect.get_feature_names()

# Let's print the top 10 terms in document 1
#print top_feats_in_doc(X, features, 1, 10)

# Now we print the top terms across all documents.
#print top_mean_feats(X, features, None, 0.1, 10)

# As clustering algorithm KMeams is a perfect fit.
##n_clusters = 3
##clf = KMeans(n_clusters=n_clusters,
##            max_iter=100,
##            init='k-means++',
##            n_init=1)
##labels = clf.fit_predict(X)

# For larger datasets use mini-batch KMeans, so we dont have to read all data into memory.
# batch_size = 500
# clf = MiniBatchKMeans(n_clusters=n_clusters, init_size=1000, batch_size=batch_size, max_iter=100)
# clf.fit(X)

# Let's plot this with matplotlib to visualize it.
# First we need to make 2D coordinates from the sparse matrix.
##X_dense = X.todense()
##pca = PCA(n_components=2).fit(X_dense)
##coords = pca.transform(X_dense)

# Lets plot it again, but this time we add some color to it.
# This array needs to be at least the length of the n_clusters.
##label_colors = ["#2AB0E9", "#2BAF74", "#D7665E", "#CCCCCC",
##                "#D2CA0D", "#522A64", "#A3DB05", "#FC6514"]
##colors = [label_colors[i] for i in labels]

#plt.scatter(coords[:, 0], coords[:, 1], c=colors)
# Plot the cluster centers
##centroids = clf.cluster_centers_
##centroid_coords = pca.transform(centroids)
# plt.scatter(centroid_coords[:, 0], centroid_coords[:, 1], marker='X', s=200, linewidths=2, c='#444d60')
# plt.show()

#Use this to print the top terms per cluster with matplotlib.
##plot_tfidf_classfeats_h(top_feats_per_cluster(X, labels, features, 0.1, 25))
