# -*- coding=utf-8 -*-
import os
import pandas as pd
import numpy as np
from datetime import datetime
import math
import nltk
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag
from nltk.stem.porter import PorterStemmer
import re
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from gensim import corpora, models, similarities
import gensim
import pyttsx3

pd.set_option('display.max_columns', None)
path = "D:/Pycodes/cogx/data"
engine = pyttsx3.init()

def any_in_str(str, critical_words):
    cont = False
    for each in critical_words:
        if each.lower() in str.lower():
            cont = True
            break
    return cont


def read_questions(input_question, csv_name, path=path, index=-1):
    df = pd.read_csv(os.path.join(path, csv_name), encoding="ISO-8859-1", header=None)
    text = np.array(df.iloc[:, 1]).tolist()
    text.append(input_question)
    # print(text)

    for i in range(len(text)):
        text_ = clean_text(text[i])
        if clean_text(text[i]) != "":
            text[i] = text_
        else:
            text[i] = 'NaN'
    if index != -1:
        df.iloc[index, 1] = ' '.join(list(set(df.iloc[index, 1].split() + input_question.split())))
        df.to_csv(os.path.join(path, csv_name), header=0, index=0)
    return text


def get_wordnet_pos(treebank_tag):
    '''
    :param treebank_tag:
    :return:
    '''
    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        return None


def clean_text(text):
    text = text.replace('\n', " ")  # new line
    text = re.sub(r"-", " ", text)
    # text = re.sub(r"\d+/\d+/\d+", "", text) # date
    # text = re.sub(r"[0-2]?[0-9]:[0-6][0-9]", "", text)  # time
    text = re.sub(r"[\w]+@[\.\w]+", "", text)  # email-address
    text = re.sub(r"[a-zA-z]+://[^\s]*", "", text)  # url
    # text = re.sub(r"((http|www)[a-zA-Z]*(com|html)*)|((http|www)*[a-zA-Z]*(com|html))", "", text)
    text = re.sub('(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)', ' ', text)
    wnl = WordNetLemmatizer()
    pure_text = ''
    for letter in text:
        if letter.isalpha() or letter == ' ':
            pure_text += letter
    text = ' '.join(wnl.lemmatize(word_tag[0], get_wordnet_pos(word_tag[1]) or wordnet.NOUN).lower() for word_tag in
                    pos_tag(pure_text.split()) if len(word_tag[0]) > 1 and len(word_tag[0]) < 30)
    # print(text)
    return text


def word2vec(corpus_list):
    '''
    convert the text into vectors after deleting the stop-words
    '''
    stopwords = [
                    "a",
                    "able",
                    "about",
                    "above",
                    "according",
                    "accordingly",
                    "across",
                    "actually",
                    "after",
                    "afterwards",
                    "again",
                    "against",
                    "ain't",
                    "aint",
                    "ain´t",
                    "ain’t",
                    "all",
                    "allow",
                    "allows",
                    "almost",
                    "alone",
                    "along",
                    "already",
                    "also",
                    "although",
                    "always",
                    "am",
                    "among",
                    "amongst",
                    "an",
                    "and",
                    "another",
                    "any",
                    "anybody",
                    "anyhow",
                    "anyone",
                    "anything",
                    "anyway",
                    "anyways",
                    "anywhere",
                    "apart",
                    "appear",
                    "appreciate",
                    "appropriate",
                    "are",
                    "aren't",
                    "arent",
                    "aren´t",
                    "aren’t",
                    "around",
                    "as",
                    "aside",
                    "ask",
                    "asking",
                    "associated",
                    "at",
                    "available",
                    "away",
                    "awfully",
                    "b",
                    "be",
                    "became",
                    "because",
                    "become",
                    "becomes",
                    "becoming",
                    "been",
                    "before",
                    "beforehand",
                    "behind",
                    "being",
                    "believe",
                    "below",
                    "beside",
                    "besides",
                    "best",
                    "better",
                    "between",
                    "beyond",
                    "both",
                    "brief",
                    "but",
                    "by",
                    "c",
                    "came",
                    "can",
                    "can't",
                    "cannot",
                    "cant",
                    "can´t",
                    "can’t",
                    "cause",
                    "causes",
                    "certain",
                    "certainly",
                    "clearly",
                    "co",
                    "com",
                    "come",
                    "comes",
                    "concerning",
                    "consequently",
                    "consider",
                    "considering",
                    "containing",
                    "contains",
                    "corresponding",
                    "could",
                    "couldn't",
                    "couldnt",
                    "couldn´t",
                    "couldn’t",
                    "course",
                    "currently",
                    "d",
                    "definitely",
                    "despite",
                    "did",
                    "didn't",
                    "didnt",
                    "didn´t",
                    "didn’t",
                    "do",
                    "does",
                    "doesn't ",
                    "doesnt ",
                    "doesn´t ",
                    "doesn’t ",
                    "doing",
                    "don't",
                    "done",
                    "dont",
                    "don´t",
                    "don’t",
                    "down",
                    "downwards",
                    "during",
                    "e",
                    "each",
                    "edu",
                    "eg",
                    "eight",
                    "either",
                    "else",
                    "elsewhere",
                    "enough",
                    "entirely",
                    "especially",
                    "et",
                    "etc",
                    "even",
                    "ever",
                    "every",
                    "everybody",
                    "everyone",
                    "everything",
                    "everywhere",
                    "ex",
                    "exactly",
                    "example",
                    "except",
                    "f",
                    "far",
                    "few",
                    "fifth",
                    "first",
                    "five",
                    "followed",
                    "following",
                    "follows",
                    "for",
                    "former",
                    "formerly",
                    "forth",
                    "four",
                    "from",
                    "further",
                    "furthermore",
                    "g",
                    "get",
                    "gets",
                    "getting",
                    "given",
                    "gives",
                    "go",
                    "goes",
                    "going",
                    "gone",
                    "got",
                    "gotten",
                    "greetings",
                    "h",
                    "had",
                    "hadn't",
                    "hadnt",
                    "hadn´t",
                    "hadn’t",
                    "happens",
                    "hardly",
                    "has",
                    "hasn't",
                    "hasnt",
                    "hasn´t",
                    "hasn’t",
                    "have",
                    "haven't",
                    "havent",
                    "haven´t",
                    "haven’t",
                    "having",
                    "he",
                    "he'd",
                    "he'll",
                    "he's",
                    "hed",
                    "hell",
                    "hence",
                    "her",
                    "here",
                    "here's",
                    "hereafter",
                    "hereby",
                    "herein",
                    "heres",
                    "hereupon",
                    "here´s",
                    "here’s",
                    "hers",
                    "herself",
                    "hes",
                    "he´d",
                    "he´ll",
                    "he´s",
                    "he’d",
                    "he’ll",
                    "he’s",
                    "hi",
                    "him",
                    "himself",
                    "his",
                    "hither",
                    "hopefully",
                    "how's",
                    "howbeit",
                    "however",
                    "hows",
                    "how´s",
                    "how’s",
                    "i",
                    "i'd",
                    "i'll",
                    "i'm",
                    "i've",
                    "id",
                    "ie",
                    "if",
                    "ignored",
                    "ill",
                    "im",
                    "immediate",
                    "in",
                    "inasmuch",
                    "inc",
                    "indeed",
                    "indicate",
                    "indicated",
                    "indicates",
                    "inner",
                    "insofar",
                    "instead",
                    "into",
                    "inward",
                    "is",
                    "isn't",
                    "isnt",
                    "isn´t",
                    "isn’t",
                    "it",
                    "it's",
                    "its",
                    "itself",
                    "it´s",
                    "it’s",
                    "ive",
                    "i´d",
                    "i´ll",
                    "i´m",
                    "i´ve",
                    "i’d",
                    "i’ll",
                    "i’m",
                    "i’ve",
                    "j",
                    "just",
                    "k",
                    "keep",
                    "keeps",
                    "kept",
                    "known",
                    "knows",
                    "l",
                    "last",
                    "lately",
                    "later",
                    "latter",
                    "latterly",
                    "least",
                    "less",
                    "lest",
                    "let",
                    "let's",
                    "lets",
                    "let´s",
                    "let’s",
                    "like",
                    "liked",
                    "likely",
                    "little",
                    "look",
                    "looking",
                    "looks",
                    "ltd",
                    "m",
                    "mainly",
                    "many",
                    "may",
                    "maybe",
                    "me",
                    "mean",
                    "meanwhile",
                    "merely",
                    "might",
                    "more",
                    "moreover",
                    "most",
                    "mostly",
                    "much",
                    "must",
                    "mustn't",
                    "mustnt",
                    "mustn´t",
                    "mustn’t",
                    "my",
                    "myself",
                    "n",
                    "namely",
                    "nd",
                    "near",
                    "nearly",
                    "necessary",
                    "need",
                    "needs",
                    "neither",
                    "never",
                    "nevertheless",
                    "nine",
                    "no",
                    "nobody",
                    "non",
                    "none",
                    "noone",
                    "nor",
                    "normally",
                    "not",
                    "nothing",
                    "novel",
                    "now",
                    "nowhere",
                    "o",
                    "obviously",
                    "of",
                    "off",
                    "often",
                    "oh",
                    "on",
                    "once",
                    "one",
                    "ones",
                    "only",
                    "onto",
                    "or",
                    "other",
                    "others",
                    "otherwise",
                    "ought",
                    "our",
                    "ours",
                    "ourselves",
                    "out",
                    "outside",
                    "over",
                    "overall",
                    "own",
                    "p",
                    "particular",
                    "particularly",
                    "per",
                    "perhaps",
                    "placed",
                    "please",
                    "plus",
                    "possible",
                    "presumably",
                    "probably",
                    "provides",
                    "q",
                    "que",
                    "quite",
                    "qv",
                    "r",
                    "rather",
                    "rd",
                    "re",
                    "really",
                    "reasonably",
                    "regarding",
                    "regardless",
                    "regards",
                    "relatively",
                    "respectively",
                    "right",
                    "s",
                    "said",
                    "same",
                    "saw",
                    "say",
                    "saying",
                    "says",
                    "second",
                    "secondly",
                    "see",
                    "seeing",
                    "seem",
                    "seemed",
                    "seeming",
                    "seems",
                    "seen",
                    "self",
                    "selves",
                    "sensible",
                    "sent",
                    "serious",
                    "seriously",
                    "seven",
                    "several",
                    "shall",
                    "shan't",
                    "shant",
                    "shan´t",
                    "shan’t",
                    "she",
                    "she'd",
                    "she'll",
                    "she's",
                    "shed",
                    "shell",
                    "shes",
                    "she´d",
                    "she´ll",
                    "she´s",
                    "she’d",
                    "she’ll",
                    "she’s",
                    "should",
                    "shouldn't",
                    "shouldnt",
                    "shouldn´t",
                    "shouldn’t",
                    "since",
                    "six",
                    "so",
                    "some",
                    "somebody",
                    "somehow",
                    "someone",
                    "something",
                    "sometime",
                    "sometimes",
                    "somewhat",
                    "somewhere",
                    "soon",
                    "specified",
                    "specify",
                    "specifying",
                    "still",
                    "sub",
                    "such",
                    "sup",
                    "sure",
                    "t",
                    "take",
                    "taken",
                    "tell",
                    "tends",
                    "th",
                    "than",
                    "that",
                    "that's",
                    "thats",
                    "that´s",
                    "that’s",
                    "the",
                    "their",
                    "theirs",
                    "them",
                    "themselves",
                    "then",
                    "thence",
                    "there",
                    "there's",
                    "thereafter",
                    "thereby",
                    "therefore",
                    "therein",
                    "theres",
                    "thereupon",
                    "there´s",
                    "there’s",
                    "these",
                    "they",
                    "they'd",
                    "they'll",
                    "they're",
                    "they've",
                    "theyd",
                    "theyll",
                    "theyre",
                    "theyve",
                    "they´d",
                    "they´ll",
                    "they´re",
                    "they´ve",
                    "they’d",
                    "they’ll",
                    "they’re",
                    "they’ve",
                    "think",
                    "third",
                    "this",
                    "thorough",
                    "thoroughly",
                    "those",
                    "though",
                    "three",
                    "through",
                    "throughout",
                    "thru",
                    "thus",
                    "to",
                    "together",
                    "too",
                    "took",
                    "toward",
                    "towards",
                    "tried",
                    "tries",
                    "truly",
                    "try",
                    "trying",
                    "twice",
                    "two",
                    "u",
                    "un",
                    "under",
                    "unfortunately",
                    "unless",
                    "unlikely",
                    "until",
                    "unto",
                    "up",
                    "upon",
                    "us",
                    "use",
                    "used",
                    "useful",
                    "uses",
                    "using",
                    "usually",
                    "uucp",
                    "v",
                    "various",
                    "very",
                    "via",
                    "viz",
                    "vs",
                    "w",
                    "want",
                    "wants",
                    "was",
                    "wasn't",
                    "wasnt",
                    "wasn´t",
                    "wasn’t",
                    "way",
                    "we",
                    "we'd",
                    "we'll",
                    "we're",
                    "we've",
                    "wed",
                    "welcome",
                    "well",
                    "went",
                    "were",
                    "weren't",
                    "werent",
                    "weren´t",
                    "weren’t",
                    "weve",
                    "we´d",
                    "we´ll",
                    "we´re",
                    "we´ve",
                    "we’d",
                    "we’ll",
                    "we’re",
                    "we’ve",
                    "whatever",
                    "whereafter",
                    "whereas",
                    "whereby",
                    "wherein",
                    "whereupon",
                    "wherever",
                    "whither",
                    "whoever",
                    "whole",
                    "will",
                    "willing",
                    "wish",
                    "with",
                    "within",
                    "without",
                    "won't",
                    "wonder",
                    "wont",
                    "won´t",
                    "won’t",
                    "would",
                    "wouldn't",
                    "wouldnt",
                    "wouldn´t",
                    "wouldn’t",
                    "x",
                    "y",
                    "yet",
                    "you",
                    "you'd",
                    "you'll",
                    "you're",
                    "you've",
                    "youd",
                    "youll",
                    "your",
                    "youre",
                    "yours",
                    "yourself",
                    "yourselves",
                    "youve",
                    "you´d",
                    "you´ll",
                    "you´re",
                    "you´ve",
                    "you’d",
                    "you’ll",
                    "you’re",
                    "you’ve",
                    "z",
                    "zero"
                ] + ['ain', 'aren', 'couldn', 'didn', 'doesn', 'doesnt', 'don', 'hadn', 'hasn', 'haven', 'isn', 'll',
                     'mustn',
                     'shan', 'shouldn', 've', 'wasn', 'weren', 'won', 'wouldn']
    # stopwords = ENGLISH_STOP_WORDS.union(stopwords)
    stopwords = stopwords
    # texts is the list of all the questions, and each of the question is a str of selected words
    # texts = [' '.join([word for word in doc.lower().split() if word not in stopwords]) for doc in corpus_list]

    # tf-idf
    vect = CountVectorizer(stop_words=stopwords).fit(corpus_list)
    feature_names = vect.get_feature_names()  # words
    # print(feature_names)
    trans_vect = vect.transform(corpus_list)
    tfidf = TfidfTransformer().fit_transform(trans_vect)
    max_value = tfidf.max(axis=0).toarray().ravel()
    sorted_by_tfidf = max_value.argsort()
    # print([feature_names[i] for i in sorted_by_tfidf[-40:]])
    # print(tfidf.toarray())
    return tfidf

def cosine_similarity(x, y, norm=False):
    assert len(x) == len(y), "len(x) != len(y)"
    zero_list = [0] * len(x)
    if x == zero_list or y == zero_list:
        return float(1) if x == y else float(0)
    res = np.array([[x[i] * y[i], x[i] * x[i], y[i] * y[i]] for i in range(len(x))])
    cos = sum(res[:, 0]) / (np.sqrt(sum(res[:, 1])) * np.sqrt(sum(res[:, 2])))
    return 0.5 * cos + 0.5 if norm else cos

def find_the_closest(tfidf):
    '''
    :param tfidf: the list of tf-idf
    :return: the index of the best Q
    '''
    q = tfidf.pop()
    temp = {}
    for each in tfidf:
        index = tfidf.index(each)
        cos = cosine_similarity(q, each)
        temp[str(index)] = cos
    index = int(max(temp, key=temp.get))
    if temp[str(index)] < similarity_threshold:
        index = -1
    return index

similarity_threshold = 0.5
answer_df = pd.read_csv(os.path.join(path, 'answer.csv'), header=None, encoding='gbk')
answer_list = np.array(answer_df.iloc[:, 1]).tolist()

def robot(question):
    corpus_list = read_questions(question, "question.csv")
    tfidf = word2vec(corpus_list).toarray().tolist()
    answer_index = find_the_closest(tfidf)
    answer = answer_list[answer_index]
    if answer_index != -1:
        read_questions(question, "question.csv", index=answer_index)  # update question database
    return answer

while True:
    q = input('Please input your question:')
    a = robot(str(q))
    print(a)
    engine.say(a)
    engine.runAndWait()