import math
import operator
import re
import string

import nltk
from rank_bm25 import BM25Okapi
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import pandas as pd
import scipy.stats as ss
import numpy as np
from nltk.corpus import stopwords
from rake_nltk import Rake

from xmlreader import XMLReader
from evaluation import evaluate_dataset
from jsonreadersolution import JSONReaderSolution
from pagerank import PageRank



class Unsupervised_Approach:
    def __init__(self, dswa):
        self.unprepared_dswa = dswa;
        self.dswa = self.convert_dataset(dswa)
        self.num_phrases = 10
        self.rank_method = 'ordinal'

    def returnSolution(self):
        # calculate keyphrases using different scoring
        tfidf = self.calculate_tfidf()
        # self.write_list_to_file(tfidf, '../data/rankings/tfidf.txt')

        tf = self.calculate_tf()
        # self.write_list_to_file(tf, '../data/rankings/tf.txt')

        bm25, idf = self.calculate_bm25()

        pagerank = self.calculate_pagerank()
        # self.write_list_to_file(bm25, '../data/rankings/bm.txt')

        rake = self.calculate_rake()

        rankings = [tf, idf, tfidf, bm25, pagerank, rake]

        solution = []
        print('compute ranking...')
        for doc in range(self.dataset_length):
            rrf_score = {}
            terms = []

            # get all keyhrases generated by rankings
            for ranking in rankings:
                terms += list(ranking[doc].keys())
            unique_terms = list(set(terms))

            for term in unique_terms:
                ranks = []
                for ranking in rankings:
                    if ranking[doc].get(term) is not None:
                        ranks.append(ranking[doc].get(term))
                rrf_score[term] = sum([1 / (50 + r) for r in ranks])

            # select best keyphrases according reciprocal rank fusion for document
            sorted_rrf_scores = sorted(rrf_score.items(), key=operator.itemgetter(1), reverse=True)
            solution.append(sorted_rrf_scores[:10])
        return solution

    def convert_dataset(self, dataset):
        """
        Conversion of nested list representation into list of strings if needed.
        """
        if isinstance(dataset[0], str):
            new_dataset = dataset
        else:
            new_dataset = []
            for doc in dataset:
                new_doc = ' '.join(w[0] for s in doc for w in s)
                new_dataset.append(new_doc)
        self.dataset_length = len(new_dataset)
        return new_dataset

    def calculate_tf(self):
        print('compute TF...')
        # tokenize and generate ngrams
        tf = CountVectorizer(stop_words="english", analyzer='word',
                             ngram_range=(1, 3), token_pattern=r'(?u)\b[A-Za-z]+\b')
        tf_vectorizer_vectors = tf.fit_transform(self.dswa)

        tf_solution = []

        for i, v in enumerate(tf_vectorizer_vectors):
            # get score for each phrase in document
            tf_df = pd.DataFrame(v.T.todense(), index=tf.get_feature_names(), columns=['score'])
            filter_short = [np.sign(len(name) - 1) for name in tf_df.index]
            length_scaler = [len(name) for name in tf_df.index]
            values = tf_df['score'] * filter_short * length_scaler
            tf_df['scaled_score'] = values

            # select and save best phrases with ranks for document
            tf_df = tf_df.sort_values(by='scaled_score', ascending=False)
            best_scores = (tf_df.head(self.num_phrases))
            tf_solution.append(
                dict(zip(best_scores.index, ss.rankdata([-1 * i for i in best_scores['scaled_score']], method=self.rank_method))))

        return tf_solution

    def calculate_tfidf(self):
        print('compute TFIDF...')
        # tokenize and generate ngrams
        tfidf = TfidfVectorizer(use_idf=True, stop_words='english', ngram_range=(1, 3),
                                token_pattern=r'(?u)\b[A-Za-z]+\b')
        tfidf_vectorizer_vectors = tfidf.fit_transform(self.dswa)

        tfidf_solution = []

        for i, v in enumerate(tfidf_vectorizer_vectors):
            # get score for each phrase in document
            tfidf_df = pd.DataFrame(v.T.todense(), index=tfidf.get_feature_names(), columns=['score'])
            lengths = [1 / (1 + math.exp(-len(name))) for name in tfidf_df.index]
            values = tfidf_df['score'] * lengths
            tfidf_df['scaled_score'] = values

            # select and save best phrases with ranks for document
            tfidf_df = tfidf_df.sort_values(by='scaled_score', ascending=False)
            best_scores = (tfidf_df.head(self.num_phrases))
            tfidf_solution.append(
                dict(zip(best_scores.index, ss.rankdata([-1 * i for i in best_scores['scaled_score']], method=self.rank_method))))

        return tfidf_solution

    def calculate_bm25(self):
        print('compute BM25...')
        # tokenize and generate ngrams
        terms_per_doc = [self.ngram_list(3, doc) for doc in self.dswa]

        bm25 = BM25Okapi(terms_per_doc)
        bm25_solution = []
        idf_solution = []

        for doc_ind in range(self.dataset_length):
            score_per_document = {}

            # get score for each phrase in document
            for term in terms_per_doc[doc_ind]:
                score_per_document[term] = bm25.get_scores(term)[doc_ind]

            # select and save best phrases with ranks for document
            sorted_scores = sorted(score_per_document.items(), key=operator.itemgetter(1), reverse=True)
            best_scores = sorted_scores[:self.num_phrases]
            bm25_solution.append(dict(zip([b[0] for b in best_scores],
                                          ss.rankdata([-1 * i for i in [b[1] for b in best_scores]], method=self.rank_method))))

        for doc_ind in range(self.dataset_length):
            score_per_document = {}
            # get score for each phrase in document
            for term in terms_per_doc[doc_ind]:
                score_per_document[term] = bm25.idf[term]
                # select and save best phrases with ranks for document
                sorted_scores = sorted(score_per_document.items(), key=operator.itemgetter(1), reverse=True)
                best_scores = sorted_scores[:self.num_phrases]
                idf_solution.append(dict(zip([b[0] for b in best_scores],
                                             ss.rankdata([-1 * i for i in [b[1] for b in best_scores]],
                                                         method=self.rank_method))))
        return bm25_solution, idf_solution

    def calculate_rake(self):
        r = Rake(min_length=1, max_length=3, stopwords=stopwords.words('english'),  punctuations=".,?;:'!%$&*1234567890")
        solution = []
        for doc in self.dswa:
            doc = re.sub('\W+', ' ', doc)
            r.extract_keywords_from_text(doc)
            solution.append(dict(zip(r.get_ranked_phrases()[:self.num_phrases], range(1, (self.num_phrases + 1)))))
        return solution

    def calculate_pagerank(self):
        pagerank = PageRank(self.unprepared_dswa);
        solution_scores = pagerank.returnSolution()
        solution = []
        for i, s in enumerate(solution_scores):
            phrases = [w[0] for w in s]
            solution.append(dict(zip(phrases, range(1, (self.num_phrases + 1) ))))
        return solution

    @staticmethod
    def ngram_list(max_size, text, stop_word_list=stopwords.words("english")):
        stop_word_set = set(stop_word_list) if stop_word_list else []
        text = text.lower()
        word_list = nltk.word_tokenize(text, language='english')
        word_list = list(filter(lambda x: len(x) > 2, word_list))
        word_list = list(filter(lambda token: any(t not in string.punctuation for t in token), word_list))
        all_ngrams = []
        for n_words in range(2, max_size):
            all_ngrams += list(nltk.ngrams(word_list, n_words))
        ngram_list = []
        for ngram in all_ngrams:
            lowered_ngram_tokens = map(lambda token: token.lower(), ngram)
            if not any(token in stop_word_set for token in lowered_ngram_tokens):
                ngram_list.append(' '.join(ngram))
        ngram_list += word_list
        return ngram_list

    def write_list_to_file(self, data, filename):
        """Write the list to csv file."""

        with open(filename, "w") as outfile:
            for entries in data:
                text = ""
                for key in entries.keys():
                    text += str(key) + ',' + str(entries.get(key)) + ','
                outfile.write(text)
                outfile.write("\n")


def test_unsupervised():
    print('Start TEST')
    xmlReader = XMLReader('../data/SemEval-2010', True)
    dswa = xmlReader.readXML()

    un_xml = Unsupervised_Approach(dswa)
    solution_xml = un_xml.returnSolution()

    jsonReader = JSONReaderSolution('../data/SemEval-2010', True)
    gt_solution = jsonReader.readJSON()
    stats = evaluate_dataset(gt_solution, solution_xml, True)

    # Print result
    for element in solution_xml:
        print(element)


def main():
    test_unsupervised()


if __name__ == '__main__':
    main()
