__author__ = 'elisabethpaulson'

from TwitterSBA import *
from SBA_text_analysis_gensim import *

####### RELATE TWITTER DATA TO SBA WEB CONTENT #################

# IMPORT CORPUS FROM FEMA WEBSITE
corpus=corpora.MmCorpus("SBA.mm")

# TF-IDF
corpus_tfidf=tfidf2[corpus]

#Run LSI, HDP, or LDA model on corpus from FEMA website. Uses topics generated by tweets and scores FEMA content according to these topics
### FIND WHICH DOCUMENTS FROM FEMA SITE ARE SIMILAR TO TOPICS FROM TWEETS
SBAcorpus_lda=lda2[corpus_tfidf]

#dictionary=corpora.Dictionary.load("SBA.dict")

m=-1
for doc in corpus_lda: #scans through content on SBA's site
    m=m+1
    doc=sorted(doc,key=lambda  item: -item[1])
    if doc!=[]:
        if doc[0][1]>.5: #specify a similarity score threshold
            print standard_links[m],' relates to topic ', doc[0][0]

### FIND WHICH TWEETS BEST MATCH EACH TOPIC
for i in range(5):
    string=''
    for j in range(6):
        string+=' '+lda2.show_topic(i,6)[j][1]
    print'TOPIC ', i, string
    # FIND WHAT TOPIC BEST MATCHES TERM
    doc=string #Make up query term
    vec_bow=dictionary2.doc2bow(doc.lower().split())
    #print doc.upper().split()
    #print vec_bow
    vec_lda=lda2[vec_bow]
    #print(vec_lda) # print vector that scores each document according to how well it matches query

    # WHICH WHICH DOCUMENT BEST MATCHES TERM
    index1=similarities.MatrixSimilarity(corpus_lda)
    index2=similarities.MatrixSimilarity(SBAcorpus_lda)
    #print index
    #index.save('FEMA.index')
    #index=similarities.MatrixSimilarity.load('FEMA.index')

    # PRINT TOP 5 FEMA DOCUMENTS THAT BEST MATCH QUERY TERM
    sims1=index1[vec_lda]
    sims2=index2[vec_lda]
    sims1=sorted(enumerate(sims1),key=lambda  item: -item[1])
    sims2=sorted(enumerate(sims2),key=lambda  item: -item[1])
    #pprint(sims[:3])
    for i in sims1[:3]:
        print(full_tweets[i[0]])
    for i in sims2[:3]:
        print(links[i[0]])
