import nltk

from nltk.tokenize import word_tokenize  #breakes sentenses in to word array
from nltk.stem import WordNetLemmatizer # removes all ing ,ed and all the other tens and the 

import numpy as np 
import random 
import pickle
from collections import Counter

lemmatizer = WordNetLemmatizer()

hm_lines = 10000000

#bag of words
def create_lexicon(pos,neg):
	lexicon = []
	with open(pos,'r') as f:
		contents = f.readlines()
		for l in contents[:hm_lines]:
			all_words = word_tokenize(l)
			lexicon += list(all_words)

	with open(neg,'r') as f:
		contents = f.readlines()
		for l in contents[:hm_lines]:
			all_words = word_tokenize(l)
			lexicon += list(all_words)

	lexicon = [lemmatizer.lemmatize(i) for i in lexicon]
	w_countes = Counter(lexicon)
	l2 = []

	for w in w_countes:
		# we have to remove the most obvious words and less most common word 
		if 1000 > w_countes[w] > 50:
			l2.append(w)
	 # Now we can take this lexicon, and use it as our bag of words that we will look for in a string. 
	# print(l2)
	return l2

def sample_handeling(sample,lexicon,classification):
	featureSet = []

	with open(sample,'r') as f:
		contents = f.readlines()
		for l in contents[:hm_lines]:
			current_words = word_tokenize(l.lower())
			current_words = [lemmatizer.lemmatize(i) for i in current_words]
			features = np.zeros(len(lexicon))
			for word in current_words:
				if word.lower() in lexicon:
					index_value = lexicon.index(word.lower())
					features[index_value] += 1
			features = list(features)
			featureSet.append([features,classification])
	return featureSet

def create_features_Sets_and_labels(pos,neg,test_size = 0.1):
	lexicon = create_lexicon(pos,neg)
	features = []
	features += sample_handeling('pos.txt',lexicon,[1,0])
	features += sample_handeling('pos.txt',lexicon,[0,1])
	random.shuffle(features)
	
	features = np.array(features)

	testing_size = int(test_size * len(features))

	train_x = list(features[:,0][:-testing_size])
	train_y = list(features[:,1][:-testing_size])

	test_x = list(features[:,0][-testing_size:])
	test_y = list(features[:,1][-testing_size:])

	return train_x ,train_y , test_x,test_y


if __name__ == '__main__':
	train_x ,train_y , test_x,test_y = create_features_Sets_and_labels('pos.txt','neg.txt')
	# with open('sentiment_set.pickle','wb') as f:
	# 	pickle.dump([train_x,train_y,test_x,test_y],f)