# -*- coding: utf-8 -*-
"""
Created on Wed Apr 24 04:51:06 2019

@author: youss
"""
#import the packages
import numpy  as np
import matplotlib.pyplot as plt
import pandas as pd
from ast import literal_eval


def cleanEnters(filename):
    f = open(filename,encoding="utf-8")
    oyt =[]
    linet=f.readline()
    oyt.append(linet)
    while True:
         # read line
        linet=f.readline()
        line = f.readline().strip()
        if line.endswith("]"):
            oyt.append(linet)
            
        # in python 2, print line
        # in python 3
        # check if line is not empty
        if not line:
            print()
            break
    f.close()
    f = open(filename,"w",encoding="utf-8")
    f.writelines(oyt)
    f.close()
    print("termino")


#read the data
def read_data(filename):
    data = pd.read_csv(filename, sep='\t')
    #This can be used for safely evaluating strings containing Python expressions 
    #from untrusted sources without the need to parse the values oneself.
    data['tags'] = data['tags'].apply(literal_eval)
    return data
    
cleanEnters("train.tsv")
cleanEnters("validation.tsv")
train = read_data('train.tsv')
validation = read_data('validation.tsv')
test = pd.read_csv('test.tsv', sep='\t')

validation.head()

# initialize X_train, X_val, X_test, y_train, y_val
X_train, y_train = train['title'].values, train['tags'].values
X_val, y_val = validation['title'].values, validation['tags'].values
X_test = test['title'].values

#Cleaning the text
import re 
import nltk 
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer

REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))

def text_prepare(text):
    """
        text: a string
        
        return: modified initial string
    """
    text = text.lower()
    text = re.sub(REPLACE_BY_SPACE_RE," ",text)
    text = re.sub(BAD_SYMBOLS_RE,"",text)
    text = text.split();
    return ' '.join([i for i in text if i not in STOPWORDS])

X_train = [text_prepare(x) for x in X_train]
X_val = [text_prepare(x) for x in X_val]
X_test = [text_prepare(x) for x in X_test]

X_train[:3]


#For each tag and for each word calculate how many times they occur in the train corpus
from collections import defaultdict
# Dictionary of all tags from train corpus with their counts.
tags_counts =  defaultdict(int)
# Dictionary of all words from train corpus with their counts.
words_counts =  defaultdict(int)

for text in X_train:
    for word in text.split():
        words_counts[word] += 1


for tags in y_train:
    for tag in tags:
        tags_counts[tag] += 1

#Described encoding in the function my_bag_of_words with the size of the dictionary equals to 5000. To find the most common words use train data.
most_common_words = sorted(words_counts.items(), key=lambda x: x[1], reverse=True)[:8000]
DICT_SIZE = 6000
WORDS_TO_INDEX = {p[0]:i for i,p in enumerate(most_common_words[:DICT_SIZE])}
INDEX_TO_WORDS = {WORDS_TO_INDEX[k]:k for k in WORDS_TO_INDEX}
ALL_WORDS = WORDS_TO_INDEX.keys()

def my_bag_of_words(text, words_to_index, dict_size):
    """
        text: a string
        dict_size: size of the dictionary
        
        return a vector which is a bag-of-words representation of 'text'
    """
    
    result_vector = np.zeros(dict_size)
    for word in text.split():
        if word in words_to_index:
            result_vector[words_to_index[word]] += 1
    
    return result_vector

#Now appling the implemented function to all samples
from scipy import sparse as sp_sparse

X_train_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_train])
X_val_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_val])
X_test_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_test])
print('X_train shape ', X_train_mybag.shape)
print('X_val shape ', X_val_mybag.shape)
print('X_test shape ', X_test_mybag.shape)

#Now appling the TF-IDF function to all samples
from sklearn.feature_extraction.text import TfidfVectorizer

def tfidf_features(X_train, X_val, X_test):
    """
        X_train, X_val, X_test — samples        
        return TF-IDF vectorized representation of each sample and vocabulary
    """
    tfidf_vectorizer = TfidfVectorizer(min_df=5, max_df=0.9, ngram_range=(1, 2),
                                       token_pattern='(\S+)')   
    X_train=tfidf_vectorizer.fit_transform(X_train)
    X_val=tfidf_vectorizer.transform(X_val)
    X_test=tfidf_vectorizer.transform(X_test)
                                       
    return X_train, X_val, X_test, tfidf_vectorizer.vocabulary_


X_train_tfidf, X_val_tfidf, X_test_tfidf, tfidf_vocab = tfidf_features(X_train, X_val, X_test)
tfidf_reversed_vocab = {i:word for word,i in tfidf_vocab.items()}



#model 
from sklearn.preprocessing import MultiLabelBinarizer

mlb = MultiLabelBinarizer(classes=sorted(tags_counts.keys()))
y_train = mlb.fit_transform(y_train)
y_val = mlb.fit_transform(y_val)

from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.decomposition.nmf import NMF
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import AdaBoostClassifier
import pickle

def train_classifier(X_train, y_train, number, C=1.0, penalty='l2'):
    """
      X_train, y_train — training data
      
      return: trained classifier
    """
    print("empezo el train_classifier")
    lr = LogisticRegression(solver='newton-cg',C=C, penalty=penalty,n_jobs=-1)
    # lr.fit(X_train, y_train)
    ovr = OneVsRestClassifier(lr)
    ovr.fit(X_train, y_train)
    print("SOY OVR!!!!!!!!!")
    print(ovr)
    if number == 1:
        pickle_out = open("ovrMyBag","wb")
        pickle.dump(ovr,pickle_out)
        pickle_out.close()
    else:
        pickle_out = open("ovrTdif","wb")
        pickle.dump(ovr,pickle_out)
        pickle_out.close()
    return ovr
import os;

if(os.path.exists("ovrMyBag") and  os.path.exists("ovrTdif")):
    print("entre aquiii")
    pickle_inMyBack = open("ovrMyBag","rb")
    pickle_inTdif = open("ovrTdif","rb")
    classifier_mybag = pickle.load(pickle_inMyBack)
    classifier_tfidf = pickle.load(pickle_inTdif)
else:
    classifier_mybag = train_classifier(X_train_mybag, y_train,1)
    classifier_tfidf = train_classifier(X_train_tfidf, y_train,2)


y_val_predicted_labels_mybag = classifier_mybag.predict(X_val_mybag)
y_val_predicted_scores_mybag = classifier_mybag.decision_function(X_val_mybag)

y_val_predicted_labels_tfidf = classifier_tfidf.predict(X_val_tfidf)
y_val_predicted_scores_tfidf = classifier_tfidf.decision_function(X_val_tfidf)

y_val_pred_inversed = mlb.inverse_transform(y_val_predicted_labels_tfidf)
y_val_inversed = mlb.inverse_transform(y_val)


for i in range(len(X_val)):
    print('Title:\t{}\nTrue labels:\t{}\nPredicted labels:\t{}\n\n'.format(
        X_val[i],
        ','.join(y_val_inversed[i]),
        ','.join(y_val_pred_inversed[i])
    ))
    
#Evaluation 
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score 
from sklearn.metrics import average_precision_score
from sklearn.metrics import recall_score

def print_evaluation_scores(y_val, predicted):
    
    print('Accuracy Score',accuracy_score(y_val, predicted))
    print('F1 Score',f1_score(y_val, predicted, average='weighted'))
    print('Precision Score',average_precision_score(y_val, predicted))
    

print('Bag-of-words')
print_evaluation_scores(y_val, y_val_predicted_labels_mybag)
print('Tfidf')
print_evaluation_scores(y_val, y_val_predicted_labels_tfidf)
