old_contents: import gzip
  import json
  import nltk
  import numpy as np 
  
  from collections import defaultdict
  from nltk.stem import SnowballStemmer	
  from utils import check_sent, normalize
  from nltk.tokenize import RegexpTokenizer
  
  tokenizer 	= RegexpTokenizer(r'\w+')
  stemmer 	= SnowballStemmer("english")
  
  class QAdoc:
  
  	def __init__(self, itemId, questionType, answerType, question, answer, V, WordIDMap):
  
  		self.itemId 		= itemId
  		self.questionType 	= questionType
  		self.answerType 	= answerType
  		self.question 		= question
  		self.answer 		= answer
  		self.Question 		= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(question) if stemmer.stem(word) in WordIDMap]
  		self.Answer 		= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(answer) if stemmer.stem(word) in WordIDMap]
  		self.qFeature 		= {}
  		self.aFeature 		= {}
  		self.create_QAFeature()
  
  
  	def create_QAFeature(self):
  
  		for wordId in self.Question:
  			if wordId in self.qFeature:
  				self.qFeature[wordId] += 1
  			
  			else:
  				self.qFeature[wordId] = np.float64(1)
  
  
  		for wordId in self.Answer:
  			if wordId in self.aFeature:
  				self.aFeature[wordId] += 1
  			
  			else:
  				self.aFeature[wordId] = np.float64(1)
  		
  		normalize(self.qFeature)
  		normalize(self.aFeature)
  
  		
  
  class ReviewDoc:
  
  	def __init__(self, itemId, reviewText, sentences, SPerItem, V, WordIDMap):
  
  		self.itemId 	= itemId
  		self.reviewText = reviewText
  		self.Review 	= []
  		self.create_ReviewFromSentence(sentences, SPerItem, V, WordIDMap)
  
  	def create_ReviewFromSentence(self, sentences, SPerItem, V, WordIDMap):
  
  		review_sentence = self.reviewText.split('.')
  		
  		for sent in review_sentence:
  			if check_sent(sent, WordIDMap):
  				obj = Sentence(self.itemId, sent, V, WordIDMap, self)
  				sentences.append(obj)
  				self.Review.append(len(sentences)-1)
  				SPerItem[self.itemId].append(len(sentences)-1)
  				obj.create_SFeature()
  			
  
  class Sentence: 
  
  	def __init__(self, itemId, Review, V, WordIDMap, ReviewObj):
  
  		self.itemId 	= itemId
  		self.sent 		= Review
  		self.rObj 		= ReviewObj
  		self.Sent 		= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(Review) if stemmer.stem(word) in WordIDMap]
  		self.sFeature 	= {}
  				
  	def create_SFeature(self):
  
  		for wordId in self.Sent:
  			if wordId in self.sFeature:
  				self.sFeature[wordId] += 1
  			
  			else:
  				self.sFeature[wordId] = np.float64(1)
  		
  		normalize(self.sFeature)
  
  		
new_contents: import gzip
  import json
  import nltk
  import numpy as np 
  
  from collections import defaultdict
  from nltk.stem import SnowballStemmer	
  from utils import check_sent, normalize
  from nltk.tokenize import RegexpTokenizer
  
  tokenizer 	= RegexpTokenizer(r'\w+')
  stemmer 	= SnowballStemmer("english")
  
  class QAdoc:
  
  	def __init__(self, itemId, questionType, answerType, question, answer, V, WordIDMap):
  
  		self.itemId 		= itemId
  		self.questionType 	= questionType
  		self.answerType 	= answerType
  		self.question 		= question
  		self.answer 		= answer
  		self.Question 		= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(question) if stemmer.stem(word) in WordIDMap]
  		self.Answer 		= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(answer) if stemmer.stem(word) in WordIDMap]
  		self.qFeature 		= {}
  		self.aFeature 		= {}
  		self.create_QAFeature()
  
  
  	def create_QAFeature(self):
  
  		for wordId in self.Question:
  			if wordId in self.qFeature:
  				self.qFeature[wordId] += 1
  			
  			else:
  				self.qFeature[wordId] = np.float64(1)
  
  
  		for wordId in self.Answer:
  			if wordId in self.aFeature:
  				self.aFeature[wordId] += 1
  			
  			else:
  				self.aFeature[wordId] = np.float64(1)
  		
  		normalize(self.qFeature)
  		normalize(self.aFeature)
  
  		
  
  class ReviewDoc:
  
  	def __init__(self, itemId, reviewText, sentences, SPerItem, V, WordIDMap):
  
  		self.itemId 	= itemId
  		self.reviewText = reviewText
  		self.Review 	= []
  		self.create_ReviewFromSentence(sentences, SPerItem, V, WordIDMap)
  
  	def create_ReviewFromSentence(self, sentences, SPerItem, V, WordIDMap):
  
  		review_sentence = self.reviewText.split('.')
  		
  		for sent in review_sentence:
  			if check_sent(sent, WordIDMap):
  				obj = Sentence(self.itemId, sent, V, WordIDMap, self)
  				sentences.append(obj)
  				self.Review.append(len(sentences)-1)
  				SPerItem[self.itemId].append(len(sentences)-1)
  				obj.create_SFeature()
  			
  
  class Sentence: 
  
  	def __init__(self, itemId, Review, V, WordIDMap, ReviewObj):
  
  		self.itemId 	= itemId
  		self.sent 	= Review
  		self.rObj 	= ReviewObj
  		self.Sent 	= [WordIDMap[stemmer.stem(word)] for word in tokenizer.tokenize(Review) if stemmer.stem(word) in WordIDMap]
  		self.sFeature 	= {}
  				
  	def create_SFeature(self):
  
  		for wordId in self.Sent:
  			if wordId in self.sFeature:
  				self.sFeature[wordId] += 1
  			
  			else:
  				self.sFeature[wordId] = np.float64(1)
  		
  		normalize(self.sFeature)
  
  		
  
subject: Use Snowball Stemmer