#!/usr/bin/python

from gensim import *
from string import *
import sys
import os

documents = []

def main():

	corpus=gen_corpus()														#calling gen_corpus() and retrieving the corpus generated by it

	tfidf = models.TfidfModel(corpus)
	corpus_tfidf = tfidf[corpus]

	#for doc in corpus_tfidf:
	#		print doc

	test_article=open(sys.argv[1])											#open the article to be tested for crime
	doc = test_article.read()
	dictionary = corpora.Dictionary.load('/tmp/deerwester.dict')			#loading the dictionary saved earlier
	
	lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=30)
	'''
	num_topics determines the number of dimensions the vector is mapped to- 
	too high a value will lead to a sparse grid; too low a value leads to 
	inaccurate clustering of words and hence wrong answers.
	'''
	#print lsi
	doc = doc.lower().translate(None,punctuation)							#removing punctuation at the end of words
	vec_bow = dictionary.doc2bow(doc.split())
	vec_lsi = lsi[vec_bow]													#implementing the LSI model in built in the gensim library
	#print vec_lsi

	index = similarities.MatrixSimilarity(lsi[corpus])						#checking similarity of articles
	sims = index[vec_lsi]
	'''
	Checking for k newarest neighbours of the article and hence 
	classifying it as a crime or a non crime article
	'''
	k=len(sims)/8															#Setting the value of k
	test_article_cosine = sims[-1]
	num_selected=0
	articles_selected=[]
	for i in xrange(len(sims)-1):
		if num_selected < k:
			articles_selected.append((i,sims[i]))
			num_selected += 1
		else:
			min_tuple = min(x[1] for x in articles_selected)
			if sims[i] > min_tuple:
				for j in xrange(len(articles_selected)):
					if articles_selected[j][1] == min_tuple:
						articles_selected.pop(j)
						break
				articles_selected.append((i,sims[i]))
	#print articles_selected

	num_crime=0																#initializing the sum of cosines variables to 0
	sum_crime=0																#initializing the sum of cosines variables to 0
	num_non_crime=0
	sum_non_crime=0
	threshold = (len(sims)-1)/2												#number of crime and non-crime articles are equal
	for i in xrange(len(articles_selected)):
		if articles_selected[i][0] >= threshold:
			num_non_crime += 1
			sum_non_crime += articles_selected[i][1]
		else:
			num_crime += 1
			sum_crime += articles_selected[i][1]
	'''
	If the number of crime and non-crime articles come out to be the same in k nearest neighbours,
	then we decide based on the sum of the cosine values of the crime and non-crime articles selected
	'''
	if num_non_crime > num_crime or (num_non_crime == num_crime and sum_non_crime > sum_crime):
		print 'Not a crime article'
	elif num_non_crime < num_crime or (num_non_crime == num_crime and sum_non_crime < sum_crime):
		print 'Crime Article'

#---------------------------------------What we were doing earlier (simply taking sums of cosines of all and deciding)-------------------------
	'''
	Simple classification of an article as a crime or non-crime
	based on cosine values
	
	for i in sims[6:12]:
		sum_non_crime+=i
	for i in sims[:6]:
		sum_crime+=i
	if sum_crime > sum_non_crime:
		print 'crime'
	else:
		print 'non-crime'
	#print list(enumerate(sims))											#uncomment to see the cosine values of each article
	'''
#----------------------------------------------------------------------------------------------------------------------------------------------

def gen_corpus():
	crime_articles = os.listdir('News_articles_lsa/crime')						#Retrieving the crime and non crime articles
	non_crime_articles = os.listdir('News_articles_lsa/non_crime')
	if len(crime_articles) > len(non_crime_articles):						#Keeping the no. of crime and non crime articles same for training
		crime_articles = crime_articles[:len(non_crime_articles)]
	elif len(crime_articles) < len(non_crime_articles):
		non_crime_articles = non_crime_articles[:len(crime_articles)]

	for i in xrange(len(crime_articles)):									#Opening the crime articles
		doc=open('News_articles_lsa/crime/'+crime_articles[i])
		documents.append(doc.read().lower().translate(None,punctuation))	#removing punctuation at the end of words
	for i in xrange(len(crime_articles)):
		doc=open('News_articles_lsa/non_crime/'+non_crime_articles[i])			#Opening the non crime articles
		documents.append(doc.read().lower().translate(None,punctuation))	#removing punctuation at the end of words


	stoplist = set('too a the that have they also were was there is with it on by at its an which for of and to in \n \r'.split())																						   #getting rid of common words, prepositions, etc
	texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]

	dictionary = corpora.Dictionary(texts)									
	dictionary.save('/tmp/deerwester.dict')									#saving the dictionary to be accessed at some later point of time
	corpus = [dictionary.doc2bow(text) for text in texts]

	corpora.MmCorpus.serialize('/tmp/deerwester.mm', corpus)
	#print dictionary.token2id												#uncomment the print statement to see the id assigned to each word
	return corpus															#returning the corpus to the main function
		

if __name__=='__main__':
	main()
