#---------------------------------------------------------------------------
#Author: Kahini Wadhawan
#---------------------------------------------------------------------------
#---------------------------------------------------------------------------
#This file creates features from bow corpus. It applies TF-IDF, LSI and LDA
# to find topics.
#---------------------------------------------------------------------------

import gensim
import logging
import os
from gensim import corpora, models
from operator import itemgetter
import json

# logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
#                     level=logging.INFO)


#----------------------------------------------------------------------------------
# Model Parameters
#----------------------------------------------------------------------------------
topics_num = 5   #revisit - make it 50
topwords_num = 20  #revisit - make it 20
top_topics_num = 5 # revisit - make it 10

#----------------------------------------------------------------------------------
# Loading dictionary and corpus generated by gensim bow model
# by vectorize_docs file
#----------------------------------------------------------------------------------
gensim_models_DIR = "data/gensim_models"
dictionary = gensim.corpora.Dictionary.load(os.path.join(gensim_models_DIR, "twitterSearch.dict"))
corpus = gensim.corpora.MmCorpus(os.path.join(gensim_models_DIR, "twitterSearch.mm"))

#------------------------------------------------------------------------------------
# Loading documents
#------------------------------------------------------------------------------------
twitter_texts_DIR = "data/twitter_texts"
json_DIR = "viz/static/json/"


def transform(model_val):

    #--------------------------------------------------------------------
    # Applying transformation on vectorized corpus
    # Projection to reduced dimension - can try LDA, PCA and other algorithms
    #--------------------------------------------------------------------
    #topics_num = 5
    tfidf = models.TfidfModel(corpus, normalize=True)
    if (model_val == 'lsi'):
        model = models.LsiModel(tfidf[corpus], id2word=dictionary, num_topics=topics_num
                                ,onepass=True, power_iters=2)
    elif(model_val== 'lda'):
        clipped_corpus = gensim.utils.ClippedCorpus(corpus, corpus.num_docs)
        model = models.LdaModel(clipped_corpus, id2word=dictionary, num_topics=topics_num,
                                passes=10)


    #-----------------------------------------------------------------
    # Printing top words for each LDA topic
    #-----------------------------------------------------------------
    #_ = model.print_topics(-1)
    print(model.show_topics(num_topics=-1, num_words=topwords_num,
                            log=False, formatted=True))


    #-------------------------------------------------------------------
    # Writing coordinates to a csv file
    #-------------------------------------------------------------------
    csv_file_path = os.path.join(gensim_models_DIR, "transform_coords.csv")
    if os.path.isfile(csv_file_path):
        os.remove(csv_file_path)
    fcoords = open(csv_file_path, 'wb')
    for vector in model[corpus]:
        if len(vector) != topics_num:
            continue
        #vector[0][1]....vector[topics_num-1][1] - docs - topic dist.
        fcoords.write("%6.4f\t%6.4f\n" % (vector[0][1], vector[1][1]))
    fcoords.close()

    #returning trained model, can be used later
    return model


def apply_model(model):
    #text = "A blood cell, also called a hematocyte, is a cell produced by hematopoiesis and normally found in blood."

    #----------------------------------------------------------------------------
    # Loading tweets_single text file into text
    #----------------------------------------------------------------------------
    text_file_path = os.path.join(twitter_texts_DIR,'tweets_single.txt')
    fp = open(text_file_path,'r')
    text = fp.read()
    #print('text is :: ', text)

    #----------------------------------------------------------------------------
    # transform text into the bag-of-words space
    # Apply trained model to unseen document
    #----------------------------------------------------------------------------
    bow_vector = dictionary.doc2bow(tokenize(text))
    #print([(dictionary[id], count) for id, count in bow_vector])
    # transform into LDA space
    lda_vector = model[bow_vector]

    #-----------------------------------------------------------------------
    # Update the trained model with online example that just came
    #-----------------------------------------------------------------------
    # model.update(other_corpus)

    #--------------------------------------------------------------------------
    # Getting statistics
    #--------------------------------------------------------------------------
    doc_topics_dist = lda_vector
    print('lda_vector ::', lda_vector)
    # print the document's single most prominent LDA topic
    print('most prominent topic ::', model.print_topic(max(lda_vector,
                                                    key=lambda item: item[1])[0]))
    # print each topic and its top n words in the document
    topics_words_dist = model.show_topics(num_topics=-1, num_words=topwords_num,
                                               log=False, formatted=True)
    print('show topics :: ', topics_words_dist)

    gen_topics_dict(doc_topics_dist, topics_words_dist)


def gen_topics_dict(doc_topics_dist, topics_words_dist):
    doc_topics_dist.sort(key=itemgetter(1), reverse=True)
    #take top_topics_num topics
    doc_topics_dist_top = doc_topics_dist[:top_topics_num]

    topics_words_dist_dict = {}
    for tup in topics_words_dist:
        words_prob = tup[1].split('+')
        #print('words_prob :: ',words_prob)
        topics_words_dist_dict[tup[0]] = words_prob
    #print('topics_words_dist_dict :: ',topics_words_dist_dict)

    topics_dist_top = {}
    json_dict = {'name':"",'children':[]}   #this children - circle for all topics
    for topic_tup in doc_topics_dist_top:
        topic_dict = {}
        print('topic id :: ',topic_tup[0])
        #mapping topic and words
        topic_words = topics_words_dist_dict[topic_tup[0]]
        topic_name = topic_words[0].split('*')[1]
        topic_dict['name'] = topic_name
        topic_dict['children'] = []
        #topics_dist_top[topic_tup[0]] = topics_words_dist_dict[topic_tup[0]]
        for word_prob in topic_words:
            word_dict = {}
            word_size,word_name = word_prob.split('*')
            word_dict['name'] = word_name
            word_dict['size'] = float(word_size)
            #append word_dict to topic_dict children list
            topic_dict['children'].append(word_dict)

        #append topic_dict to main json_dict children list
        json_dict['children'].append(topic_dict)

    #print('json dict :: ', json_dict)
    #print('topics_dist_top :: ', topics_dist_top)
    gen_json(json_dict)


def tokenize(text):
    tokens = gensim.utils.tokenize(text, lowercase=True, deacc=True,
                                        errors="ignore")
    #print('tokens :: ', tokens)
    return tokens


def gen_json(json_dict):
    # get json string from that dictionary
    topics_json=json.dumps(json_dict)
    print topics_json
    #write json to file
    file_path = os.path.join(json_DIR,'topics.json')  #revisit - to put politician name
    write_tofile(file_path,topics_json)


def write_tofile(file_path,text):
    fp = open(file_path,'wb')
    fp.write(text)
    fp.close()


#testing
model = transform('lda')
apply_model(model)
