#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Start to build final NLP prediction model

@author: joshmagee
Tue Sep 18 15:00:42 2018
"""

import wine
import dill
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.corpus import stopwords

sns.set_palette("dark")

from nltk.tokenize import RegexpTokenizer

#generic wine cleaning
df = wine.read_wine()
df = wine.clean_data(df)
df = wine.select_top10(df)

#let's start by JUST looking at the description and color
df['color'] = df['variety'].apply(wine.coloring)
df['color'] = df['color'].apply(lambda x: 1 if x == 'red' else 0)


df_orig = df.copy()
df = df[['description', 'color']]

def standardize_text(df, text_field):
    df[text_field] = df[text_field].str.replace(r"http\S+", "")
    df[text_field] = df[text_field].str.replace(r"http", "")
    df[text_field] = df[text_field].str.replace(r"@\S+", "")
    df[text_field] = df[text_field].str.replace(r"[^A-Za-z0-9(),!?@\'\`\"\_\n]", " ")
    df[text_field] = df[text_field].str.replace(r",", " ")
    df[text_field] = df[text_field].str.replace(r"  ", " ")
    df[text_field] = df[text_field].str.replace(r"@", "at")
    df[text_field] = df[text_field].str.lower()
    
    stop = stopwords.words('english')
    domain = ['carneros', 'sonoma', 'annapolis', 'paso', 'livermore', 'napa',
              'robles', 'stags', 'soon', 'lot', 'santa', 'central', 'coast',
              'calera', 'ross', 'county', 'rita', 'fort', 'marin', 'got',
              'town', '60', '17', 'wine', 'flavors', 'cabernet', 'pinot',
              'noir', 'zin', 'syrah', 'chardonnay', 'riesling', 'blanc', 
              'sauvignon']
    stop.extend(domain)
    
    df[text_field] = \
        df[text_field].apply(lambda x: \
          ' '.join([item for item in x.split() if item not in stop]))
    return df

df = standardize_text(df, 'description')

#now let's tokenize this
tokenizer = RegexpTokenizer(r'\w+')
df['tokens'] = df['description'].apply(tokenizer.tokenize)

#simple graph of all words/tokens
all_words = [word for tokens in df['tokens'] for word in tokens]
sentence_lengths = [len(tokens) for tokens in df['description']]
vocab = sorted(list(set(all_words)))
print(str(len(all_words)) + ' words total, with a vocabulary size of ' \
      + str(len(vocab)))
print('Max sentence length is ' + str(max(sentence_lengths)))


fig = plt.figure(figsize=(10, 10)) 
plt.xlabel('Sentence length')
plt.ylabel('Number of sentences')
plt.hist(sentence_lengths, bins=50)
plt.grid(True, alpha=0.3)
plt.show()
#plt.savefig('sentence_length_total.png')

#simple bag of words counts
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

def cv(data):
    count_vectorizer = CountVectorizer()
    emb = count_vectorizer.fit_transform(data)
    return emb, count_vectorizer

#question['text'] ---> df['descriptions']
#question['class_label'] ---> df['red/white']
list_corpus = df["description"].tolist()
list_labels = df["color"].tolist()

X_train, X_test, y_train, y_test = \
    train_test_split(list_corpus, list_labels, test_size=0.2, random_state=42)

X_train_counts, count_vectorizer = cv(X_train)
X_test_counts = count_vectorizer.transform(X_test)

from sklearn.decomposition import TruncatedSVD
import matplotlib
import matplotlib.patches as mpatches


def plot_LSA(test_data, test_labels, savepath="PCA_demo.csv", plot=True):
        lsa = TruncatedSVD(n_components=2)
        lsa.fit(test_data)
        lsa_scores = lsa.transform(test_data)
        colors = ['skyblue','crimson']
        if plot:
            plt.scatter(lsa_scores[:,0], lsa_scores[:,1], s=8, alpha=.8, \
                        c=test_labels, cmap=matplotlib.colors.ListedColormap(colors))
            white_patch = mpatches.Patch(color='skyblue', label='White')
            red_patch = mpatches.Patch(color='crimson', label='Red')
            plt.legend(handles=[red_patch, white_patch], prop={'size': 30})



fig = plt.figure(figsize=(10, 10))          
plot_LSA(X_train_counts, y_train)
plt.title('PCA of Wine Descriptions (Bag of Words)')
plt.show()
plt.savefig('nlp_pca.png')


def tfidf(data):
    tfidf_vectorizer = TfidfVectorizer()
    train = tfidf_vectorizer.fit_transform(data)
    return train, tfidf_vectorizer

X_train_tfidf, tfidf_vectorizer = tfidf(X_train)
X_test_tfidf = tfidf_vectorizer.transform(X_test)


fig = plt.figure(figsize=(10, 10))          
plot_LSA(X_train_tfidf, y_train)
plt.title('PCA of Wine Descriptions (TF-IDF)')
plt.show()


def get_most_important_features(vectorizer, model, n=5):
    index_to_word = {v:k for k,v in vectorizer.vocabulary_.items()}
    
    # loop for each class
    classes ={}
    for class_index in range(model.coef_.shape[0]):
        word_importances = [(el, index_to_word[i]) for i,el in enumerate(model.coef_[class_index])]
        sorted_coeff = sorted(word_importances, key = lambda x : x[0], reverse=True)
        tops = sorted(sorted_coeff[:n], key = lambda x : x[0])
        bottom = sorted_coeff[-n:]
        classes[class_index] = {
            'tops':tops,
            'bottom':bottom
        }
    return classes

def plot_important_words(top_scores, top_words, bottom_scores, bottom_words, name):
    y_pos = np.arange(len(top_words))
    top_pairs = [(a,b) for a,b in zip(top_words, top_scores)]
    top_pairs = sorted(top_pairs, key=lambda x: x[1])
    
    bottom_pairs = [(a,b) for a,b in zip(bottom_words, bottom_scores)]
    bottom_pairs = sorted(bottom_pairs, key=lambda x: x[1], reverse=True)
    
    top_words = [a[0] for a in top_pairs]
    top_scores = [a[1] for a in top_pairs]
    
    bottom_words = [a[0] for a in bottom_pairs]
    bottom_scores = [a[1] for a in bottom_pairs]
    
    plt.figure(figsize=(10, 10))  

    plt.subplot(121)
    plt.barh(y_pos,bottom_scores, align='center', alpha=0.5)
    plt.title('White', fontsize=24)
    plt.yticks(y_pos, bottom_words, fontsize=24)
    plt.suptitle('Key words', fontsize=20)
    plt.xlabel('Importance', fontsize=24)
    
    plt.subplot(122)
    plt.barh(y_pos,top_scores, align='center', alpha=0.5)
    plt.title('Red', fontsize=24)
    plt.yticks(y_pos, top_words, fontsize=24)
    plt.suptitle(name, fontsize=20)
    plt.xlabel('Importance', fontsize=24)
    
    plt.subplots_adjust(wspace=0.8)
    plt.tight_layout()
    plt.show()

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, f1_score, precision_score, \
                            recall_score, classification_report

def get_metrics(y_test, y_predicted):  
    # true positives / (true positives+false positives)
    precision = precision_score(y_test, y_predicted, pos_label=None,
                                    average='weighted')             
    # true positives / (true positives + false negatives)
    recall = recall_score(y_test, y_predicted, pos_label=None,
                              average='weighted')
    
    # harmonic mean of precision and recall
    f1 = f1_score(y_test, y_predicted, pos_label=None, average='weighted')
    
    # true positives + true negatives/ total
    accuracy = accuracy_score(y_test, y_predicted)
    return accuracy, precision, recall, f1


clf_tfidf = LogisticRegression(C=30.0, class_weight='balanced', solver='newton-cg', 
                         multi_class='multinomial', n_jobs=-1, random_state=40)
clf_tfidf.fit(X_train_tfidf, y_train)

y_predicted_tfidf = clf_tfidf.predict(X_test_tfidf)

accuracy_tfidf, precision_tfidf, recall_tfidf, f1_tfidf = get_metrics(y_test, y_predicted_tfidf)
print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" % (accuracy_tfidf, precision_tfidf, 
                                                                       recall_tfidf, f1_tfidf))



importance_tfidf = get_most_important_features(tfidf_vectorizer, clf_tfidf, 10)
top_scores = [a[0] for a in importance_tfidf[0]['tops']]
top_words = [a[1] for a in importance_tfidf[0]['tops']]
bottom_scores = [a[0] for a in importance_tfidf[0]['bottom']]
bottom_words = [a[1] for a in importance_tfidf[0]['bottom']]

plot_important_words(top_scores, top_words, bottom_scores, bottom_words,'')#, 
                     #"Most relevant words")
plt.savefig('white_v_red_top10_words.png')

import itertools
from sklearn.metrics import confusion_matrix

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.winter):
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title, fontsize=30)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, fontsize=20)
    plt.yticks(tick_marks, classes, fontsize=20)
    
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.

    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", 
                 color="white" if cm[i, j] < thresh else "black", fontsize=40)
    
    plt.tight_layout()
    plt.ylabel('True label', fontsize=30)
    plt.xlabel('Predicted label', fontsize=30)

    return plt



cm2 = confusion_matrix(y_test, y_predicted_tfidf)
fig = plt.figure(figsize=(10, 10))
plot = plot_confusion_matrix(cm2, classes=['White','Red'], \
                             normalize=False, title='Confusion matrix')
plt.show()
plt.savefig('red_white_confusion_matrix.png')
print("TFIDF confusion matrix")
print(cm2)
print("BoW confusion matrix")






