#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu May 16 13:38:23 2019


@author: kristen.anderson101gmail.com
http://andrewtrick.com/toxic_comments.html
"""

import pandas as pd
import numpy as np
import re
import os
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
#from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import FeatureUnion #unites all arrays into one array
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression 
from sklearn.preprocessing import FunctionTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
# import
df_test = pd.read_csv("test.csv")
df_test_labs = pd.read_csv("test_labels.csv")
df_train = pd.read_csv("train.csv")
df_sub = pd.read_csv("sample_submission.csv")

#init corpus
corpus = []

#loop through df and clean comments
for i in range(0, 50000):
    #reg_exp to replace anything not text to a space and drop to lower case
    comment = re.sub('[^a-zA-Z]', ' ', df_train['comment_text'][i]).lower()
    #split into list for processing
    comment = comment.split()
    #check for stopwords and remove
    comment = [word for word in comment if not word in set(stopwords.words('english'))]
    #stem the word!
    ps = PorterStemmer()
    comment = [str(ps.stem(word)) for word in comment]
    #back to string
    comment = ' '.join(comment)
    corpus.append(comment)
    #track progress
    if i%1000 == 0:
        print((float(i)/len(df_train))*100)
        

#Bag of Words Model - sparse matrix (tokenize)
#cv = CountVectorizer(max_features = 25000)   #max words to store 
cv = TfidfVectorizer(
    stop_words='english',
    sublinear_tf=True,
    strip_accents='unicode',
    analyzer='word',
    token_pattern=r'\w{2,}',  #vectorize 2-character words or more
    ngram_range=(1, 1),
    max_features=30000)
X = cv.fit_transform(corpus).toarray()
y_tox = df_train.iloc[0:50000,2].values
y_sev_tox = df_train.iloc[0:50000,3].values
y_obs = df_train.iloc[0:50000,4].values
y_threat = df_train.iloc[0:50000,5].values
y_insult = df_train.iloc[0:50000,6].values
y_hate = df_train.iloc[0:50000,7].values


#model for each predicted type
tests = {'y_tox' : y_tox, 
         'y_sev_tox' : y_sev_tox, 
         'y_obs' : y_obs, 
         'y_threat' : y_threat, 
         'y_insult' : y_insult, 
         'y_hate' : y_hate}

models = {'y_tox' : GaussianNB(), 
          'y_sev_tox' : GaussianNB(),
          'y_obs' : GaussianNB(),
          'y_threat' : GaussianNB(),
          'y_insult' : GaussianNB(),
          'y_hate' : GaussianNB()}

preds = {}

test_names = ['y_tox', 'y_sev_tox', 'y_obs', 'y_threat', 'y_insult', 'y_hate']

for i in test_names:
    #test_train split (toxic)
    X_train, X_test, y_train, y_test = train_test_split(X, tests[i], test_size = 0.05, random_state = 42)
    
    #Train Model (naive bayes)
    models[i].fit(X_train, y_train)

    #predict
    preds[i] = models[i].predict(X_test)
    
    #review model
    print(i)
    print(confusion_matrix(y_test, preds[i]))
    print(accuracy_score(y_test, preds[i]))
