import pandas as pd
import numpy as np
import gensim
from fuzzywuzzy import fuzz
from nltk.corpus import stopwords
from tqdm import tqdm
from pyemd import emd
from scipy.stats import skew, kurtosis
from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis
from nltk import word_tokenize

def wmd(s1, s2):
    s1 = str(s1).lower().split()
    s2 = str(s2).lower().split()
    stop_words = stopwords.words('english')
    s1 = [w for w in s1 if w not in stop_words]
    s2 = [w for w in s2 if w not in stop_words]
    return model.wmdistance(s1, s2)


def norm_wmd(s1, s2):
    s1 = str(s1).lower().split()
    s2 = str(s2).lower().split()
    stop_words = stopwords.words('english')
    s1 = [w for w in s1 if w not in stop_words]
    s2 = [w for w in s2 if w not in stop_words]
    return norm_model.wmdistance(s1, s2)


def sent2vec(s):
    words = str(s).lower()
    words = word_tokenize(words)
    words = [w for w in words if not w in stop_words]
    words = [w for w in words if w.isalpha()]
    M = []
    for w in words:
        try:
            M.append(model[w])
        except:
            continue
    M = np.array(M)
    v = M.sum(axis=0)
    return v / np.sqrt((v ** 2).sum())


data = pd.read_csv('data/train.csv')
data = data.fillna(' ')
data = data.drop(['id', 'qid1', 'qid2', 'is_duplicate'], axis=1)

dataTest = pd.read_csv('data/test.csv')
dataTest = dataTest.fillna(' ')
dataTest = dataTest.drop(['test_id'], axis=1)

data['len_q1'] = data.question1.apply(lambda x: len(str(x)))
data['len_q2'] = data.question2.apply(lambda x: len(str(x)))
data['diff_len'] = data.len_q1 - data.len_q2
data['len_char_q1'] = data.question1.apply(lambda x: len(''.join(set(str(x).replace(' ', '')))))
data['len_char_q2'] = data.question2.apply(lambda x: len(''.join(set(str(x).replace(' ', '')))))
data['len_word_q1'] = data.question1.apply(lambda x: len(str(x).split()))
data['len_word_q2'] = data.question2.apply(lambda x: len(str(x).split()))
data['common_words'] = data.apply(lambda x: len(set(str(x['question1']).lower().split()).intersection(set(str(x['question2']).lower().split()))), axis=1)
data['fuzz_qratio'] = data.apply(lambda x: fuzz.QRatio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_WRatio'] = data.apply(lambda x: fuzz.WRatio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_partial_ratio'] = data.apply(lambda x: fuzz.partial_ratio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_partial_token_set_ratio'] = data.apply(lambda x: fuzz.partial_token_set_ratio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_partial_token_sort_ratio'] = data.apply(lambda x: fuzz.partial_token_sort_ratio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_token_set_ratio'] = data.apply(lambda x: fuzz.token_set_ratio(str(x['question1']), str(x['question2'])), axis=1)
data['fuzz_token_sort_ratio'] = data.apply(lambda x: fuzz.token_sort_ratio(str(x['question1']), str(x['question2'])), axis=1)

dataTest['len_q1'] = dataTest.question1.apply(lambda x: len(str(x)))
dataTest['len_q2'] = dataTest.question2.apply(lambda x: len(str(x)))
dataTest['diff_len'] = dataTest.len_q1 - dataTest.len_q2
dataTest['len_char_q1'] = dataTest.question1.apply(lambda x: len(''.join(set(str(x).replace(' ', '')))))
dataTest['len_char_q2'] = dataTest.question2.apply(lambda x: len(''.join(set(str(x).replace(' ', '')))))
dataTest['len_word_q1'] = dataTest.question1.apply(lambda x: len(str(x).split()))
dataTest['len_word_q2'] = dataTest.question2.apply(lambda x: len(str(x).split()))
dataTest['common_words'] = dataTest.apply(lambda x: len(set(str(x['question1']).lower().split()).intersection(set(str(x['question2']).lower().split()))), axis=1)
dataTest['fuzz_qratio'] = dataTest.apply(lambda x: fuzz.QRatio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_WRatio'] = dataTest.apply(lambda x: fuzz.WRatio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_partial_ratio'] = dataTest.apply(lambda x: fuzz.partial_ratio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_partial_token_set_ratio'] = dataTest.apply(lambda x: fuzz.partial_token_set_ratio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_partial_token_sort_ratio'] = dataTest.apply(lambda x: fuzz.partial_token_sort_ratio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_token_set_ratio'] = dataTest.apply(lambda x: fuzz.token_set_ratio(str(x['question1']), str(x['question2'])), axis=1)
dataTest['fuzz_token_sort_ratio'] = dataTest.apply(lambda x: fuzz.token_sort_ratio(str(x['question1']), str(x['question2'])), axis=1)

stop_words = stopwords.words('english')
model = gensim.models.KeyedVectors.load_word2vec_format('data/GoogleNews-vectors-negative300.bin', binary=True)
data['wmd'] = data.apply(lambda x: wmd(x['question1'], x['question2']), axis=1)
dataTest['wmd'] = dataTest.apply(lambda x: wmd(x['question1'], x['question2']), axis=1)


norm_model = gensim.models.KeyedVectors.load_word2vec_format('data/GoogleNews-vectors-negative300.bin', binary=True)
norm_model.init_sims(replace=True)
data['norm_wmd'] = data.apply(lambda x: norm_wmd(x['question1'], x['question2']), axis=1)
dataTest['norm_wmd'] = dataTest.apply(lambda x: norm_wmd(x['question1'], x['question2']), axis=1)

question1_vectors = np.zeros((data.shape[0], 300))
question1_test_vectors = np.zeros((dataTest.shape[0], 300))
error_count = 0
error_test_count = 0

for i, q in tqdm(enumerate(data.question1.values)):
    question1_vectors[i, :] = sent2vec(q)

for i, q in tqdm(enumerate(dataTest.question1.values)):
    question1_test_vectors[i, :] = sent2vec(q)

question2_vectors  = np.zeros((data.shape[0], 300))
question2_test_vectors  = np.zeros((dataTest.shape[0], 300))
for i, q in tqdm(enumerate(data.question2.values)):
    question2_vectors[i, :] = sent2vec(q)

for i, q in tqdm(enumerate(dataTest.question2.values)):
    question2_test_vectors[i, :] = sent2vec(q)

data['cosine_distance'] = [cosine(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['cityblock_distance'] = [cityblock(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['jaccard_distance'] = [jaccard(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['canberra_distance'] = [canberra(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['euclidean_distance'] = [euclidean(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['minkowski_distance'] = [minkowski(x, y, 3) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['braycurtis_distance'] = [braycurtis(x, y) for (x, y) in zip(np.nan_to_num(question1_vectors),
                                                          np.nan_to_num(question2_vectors))]

data['skew_q1vec'] = [skew(x) for x in np.nan_to_num(question1_vectors)]
data['skew_q2vec'] = [skew(x) for x in np.nan_to_num(question2_vectors)]
data['kur_q1vec'] = [kurtosis(x) for x in np.nan_to_num(question1_vectors)]
data['kur_q2vec'] = [kurtosis(x) for x in np.nan_to_num(question2_vectors)]

# test features
dataTest['cosine_distance'] = [cosine(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['cityblock_distance'] = [cityblock(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['jaccard_distance'] = [jaccard(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['canberra_distance'] = [canberra(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['euclidean_distance'] = [euclidean(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['minkowski_distance'] = [minkowski(x, y, 3) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['braycurtis_distance'] = [braycurtis(x, y) for (x, y) in zip(np.nan_to_num(question1_test_vectors),
                                                          np.nan_to_num(question2_test_vectors))]

dataTest['skew_q1vec'] = [skew(x) for x in np.nan_to_num(question1_test_vectors)]
dataTest['skew_q2vec'] = [skew(x) for x in np.nan_to_num(question2_test_vectors)]
dataTest['kur_q1vec'] = [kurtosis(x) for x in np.nan_to_num(question1_test_vectors)]
dataTest['kur_q2vec'] = [kurtosis(x) for x in np.nan_to_num(question2_test_vectors)]

data.to_csv('data/train_features.csv', index=False)
dataTest.to_csv('data/test_features.csv', index=False)