#!/usr/bin/env python3
# Author: Carlijn Assen


from nltk.tokenize import TweetTokenizer, RegexpTokenizer
from itertools import permutations
from time import sleep
import numpy as np
import re
import math
import string
import pickle
import operator


def del_symbols(text):
    result = re.sub(r"[()+-.../#]", r"", text)
    return result


def del_emojis(text):
    return text.encode('ascii', 'ignore').decode('ascii')


def del_tw(text):
    result = re.sub(r"https\S+", "", text)
    return result


def del_rt(text):
    result = re.sub(r"RT", "", text)
    return result


def del_punct(s):
    result = s.translate(str.maketrans("", "", string.punctuation))
    return result


def highest_score():

    docs = pickle.load(open('docs.pickle', 'rb'))
    tw_tok = TweetTokenizer(preserve_case = False, strip_handles = True, reduce_len = True)
    perm_list = []
    all_tweets = 0
    term_in_document = 0
    t_list = []
    values = []
    binary_list = []
    
    for item in docs:

        words = docs[item][2]
        words = del_emojis(words)
        words = del_tw(words)
        words = del_symbols(words)
        words = del_punct(words)
        words = del_rt(words)
        words = tw_tok.tokenize(words)

        perms = (permutations(words, 2))
    
        for i, j in perms:
            w1 = i
            w2 = j
        
        all_tweets += 1
        words = docs[item][2].split()
        tweet_len = len(words)
        if w1 in words or w2 in words:
            term_in_document += 1
            w_occ = words.count(w1) + words.count(w2)

            tf = w_occ/tweet_len
            idf = np.log(all_tweets/term_in_document)
            tf_idf = tf * idf
            tf_idf = round(tf_idf, 6)
            t_list.append((tf_idf, w1, w2))
    first_value = t_list[0][0]
    total = first_value
    maximum = first_value
    for i in range(0, len(t_list)):
        next_value = float(t_list[i][0])
        total += next_value
        if maximum <= next_value:
            binary_list.append(str(1))
            maximum = next_value
        else:
            binary_list.append(str(0))
            
    last_occ = dict(map(reversed, enumerate(binary_list)))['1']
    #last_occ=len(binary_list)-binary_list[::-1].index('1')
    
    print("The highest TF-IDF score = {0}".format(maximum))
    print("Which can be obtained with the following query: {0} {1}".format(t_list[last_occ][1], t_list[last_occ][2]))
        
             
highest_score()
