#!/usr/bin/env python3
# Author: Carlijn Assen


from nltk.tokenize import TweetTokenizer, RegexpTokenizer
from itertools import permutations
from time import sleep
import numpy as np
import re
import math
import string
import pickle
import operator


def del_symbols(text):
    result = re.sub(r"[()+-.../#]", r"", text)
    return result


def del_emojis(text):
    return text.encode('ascii', 'ignore').decode('ascii')


def del_tw(text):
    result = re.sub(r"https\S+", "", text)
    return result


def del_rt(text):
    result = re.sub(r"RT", "", text)
    return result


def del_punct(s):
    result = s.translate(str.maketrans("", "", string.punctuation))
    return result


def highest_score():

    docs = pickle.load(open('docs.pickle', 'rb'))
    words_index = pickle.load(open('words_index.pickle', 'rb'))
    tw_tok = TweetTokenizer(preserve_case = False, strip_handles = True, reduce_len = True)
    score = 0.0
    score_list = {}

    for item in docs:


        words = docs[item][2]
        words = del_emojis(words)
        words = del_tw(words)
        words = del_symbols(words)
        words = del_punct(words)
        words = del_rt(words)
        words = tw_tok.tokenize(words)

    perms = (permutations(words, 2))

    for i, j in perms:
        w1 = i
        w2 = j

    if w1 in words_index or w2 in words_index:
        w1_count = 0
        w2_count = 0
        idds = []

    if w1 not in words_index and w2 in words_index:
        idds += list(words_index[w2].keys())
    elif w2 not in words_index and w1 in words_index:
        idds += list(words_index[w1].keys())
    elif w1 in words_index and w2 in words_index:
        idds += list(words_index[w1].keys())
        idds += list(words_index[w2].keys())

    for item in idds:
        words = docs[item][2].split()
        w1_count = w1_count + words.count(w1)
        w2_count = w2_count + words.count(w2)
        words_found = w1_count + w2_count
        if words_found == 1:
            swf = 1
        else:
            swf = 2
            tf = swf / words_found
            df = swf / len(words)
            idf = np.log(len(docs.keys()) / (df + 1))
            tf_idf = tf * idf
    

        if math.isnan(tf_idf):
            tf_idf = 0
        else:
            tf_idf = round(tf_idf, 3)

        if tf_idf > score:
            score = tf_idf
    score_list[score] = (w1, w2)
    for key in sorted(score_list.keys()):
        print('Highest tf-idf score = {0}'.format(key))
        print('Query: {0}'.format(score_list[key]))




highest_score()

