#!/usr/bin/python
import pandas as pd
#Beautiful Soup to remove html tags, markups
from bs4 import BeautifulSoup
from gensim import *
from string import *
import re
import nltk
from nltk.corpus import stopwords
import sys
import os

def clean(raw_review):
	print len(raw_review)
	clean_reviews = []
	for r in raw_review:
		bs = BeautifulSoup(r).get_text()
		letters = re.sub("[^a-zA-Z]", " ", bs)
		words = letters.lower().split()
		stops = set(stopwords.words("english"))                  
		meaningful_words = [w for w in words if not w in stops]   
		clean_reviews.append(" ".join(meaningful_words))
	return clean_reviews

def main():
	print "Generating corpus"
	corpus = gen_corpus()														#calling gen_corpus() and retrieving the corpus generated by it
	print "Corpus generated"
	tfidf = models.TfidfModel(corpus)
	corpus_tfidf = tfidf[corpus]
	
	print "Reading test file"
	test_data = pd.read_csv("labeledTestData.tsv", header=0, delimiter="\t", quoting=3)
	print "Read test file"
	#print test_data["review"][0]
	#for doc in corpus_tfidf:
	#		print doc

	dictionary = corpora.Dictionary.load('lsa.dict')			#loading the dictionary saved earlier
	#print dictionary
	lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=30)
	'''
	num_topics determines the number of dimensions the vector is mapped to- 
	too high a value will lead to a sparse grid; too low a value leads to 
	inaccurate clustering of words and hence wrong answers.
	'''
	#print lsi
	review = clean(test_data["review"])										#removing punctuation at the end of words
	#print review[0]
	#review = test_data["review"]										#removing punctuation at the end of words
	done = 0
	correct = 0
	incorrect = 0
	index = similarities.MatrixSimilarity(lsi[corpus])						#checking similarity of articles
	f = open("results_lsa_stem_temp.csv", "w")	
	print "Testing.."
	f.write("\"id\",\"sentiment\"")
	for r in review:	
		vec_bow = dictionary.doc2bow(review[done].split())
		vec_lsi = lsi[vec_bow]													#implementing the LSI model in built in the gensim library
		#print vec_lsi

		sims = index[vec_lsi]
		'''
		Checking for k nearest neighbours of the article and hence 
		classifying it as a crime or a non crime article
		'''
		#print sims
		k = len(sims)/8															#Setting the value of k
		#print k
		test_review_cosine = sims[-1]
		num_selected=0
		reviews_selected=[]
		for i in xrange(len(sims)-1):
			if num_selected < k:
				reviews_selected.append((i,sims[i]))
				num_selected += 1
			else:
				min_tuple = min(x[1] for x in reviews_selected)
				if sims[i] > min_tuple:
					for j in xrange(len(reviews_selected)):
						if reviews_selected[j][1] == min_tuple:
							reviews_selected.pop(j)
							break
					reviews_selected.append((i,sims[i]))
		#print reviews_selected

		num_pos=0																#initializing the sum of cosines variables to 0
		sum_pos=0																#initializing the sum of cosines variables to 0
		num_neg=0
		sum_neg=0
		threshold = (len(sims)-1)/2												#number of crime and non-crime articles are equal
		for i in xrange(len(reviews_selected)):
			if reviews_selected[i][0] >= threshold:
				num_neg += 1
				sum_neg += reviews_selected[i][1]
			else:
				num_pos += 1
				sum_neg += reviews_selected[i][1]
		'''
		If the number of crime and non-crime articles come out to be the same in k nearest neighbours,
		then we decide based on the sum of the cosine values of the crime and non-crime articles selected
		'''
		sentiment = 1
		if num_neg > num_pos or (num_neg == num_pos and sum_neg > sum_pos):
			sentiment = 0
		elif num_neg < num_pos or (num_neg == num_pos and sum_neg < sum_pos):
			sentiment = 1
		if(sentiment==test_data["sentiment"][done]):
			correct += 1
		else:
			incorrect += 1
		done += 1
		f.write(str(test_data["id"][done]) + ","+str(sentiment))
		print str(test_data["id"][done]) + ","+str(sentiment) + "\tDone-" + str(done) + ",Correct - " + str(correct) + ",Incorrect - " + str(incorrect)



def get_reviews(reviews, sentiments, sentiment):

	sentiment_reviews = []
	i = 0
	for r in reviews:
		if(sentiments[i]==sentiment):
			sentiment_reviews.append(reviews[i])
		i += 1
	return sentiment_reviews

def get_text(reviews1, reviews2):

	text = []
	for r in reviews1:
		text.append(r)
	for r in reviews2:
		text.append(r)
	return text


def gen_corpus():

	print "Reading data.."
	training_data = pd.read_csv("labeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
	
	positive_reviews = get_reviews(training_data["review"],training_data["sentiment"],1)
	negative_reviews = get_reviews(training_data["review"],training_data["sentiment"],0)
	positive_reviews = clean(positive_reviews)
	print len(positive_reviews)
	negative_reviews = clean(negative_reviews)
	print len(negative_reviews)

	if len(positive_reviews) > len(negative_reviews):						#Keeping the no. of crime and non crime articles same for training
		positive_reviews = positive_reviews[:len(negative_reviews)]
	elif len(positive_reviews) < len(negative_reviews):
		negative_reviews = negative_reviews[:len(positive_reviews)]

	reviews = get_text(positive_reviews, negative_reviews)
	texts = [[word for word in doc.split()] for doc in reviews]
	dictionary = corpora.Dictionary(texts)									
	dictionary.save('lsa.dict')									#saving the dictionary to be accessed at some later point of time
	print dictionary
	corpus = [dictionary.doc2bow(text) for text in texts]

	corpora.MmCorpus.serialize('lsa.mm', corpus)
	#print dictionary.token2id												#uncomment the print statement to see the id assigned to each word
	return corpus															#returning the corpus to the main function
		

if __name__=='__main__':
	main()
