old_contents: #!/usr/bin/python
  
  import os
  from sklearn.feature_extraction.text import CountVectorizer
  from sklearn.ensemble import RandomForestClassifier
  from KaggleWord2VecUtility import KaggleWord2VecUtility
  import pandas as pd
  import numpy as np
  from memory_profiler import profile
  
  @profile
  def bag_of_words_model():
      labeledtrain_data = pd.read_csv('labeledTrainData.tsv', header=0, delimiter='\t', quoting=3)
      #unlabeledtrain_data = pd.read_csv('unlabeledTrainData.tsv', header=0, delimiter='\t', quoting=3)
      
      clean_labeledtrain_reviews = map( lambda r: ' '.join(KaggleWord2VecUtility.review_to_wordlist(r, remove_stopwords=False)) , labeledtrain_data['review'] )
  
      vectorizer = CountVectorizer(analyzer = 'word', tokenizer = None,  preprocessor = None, stop_words = 'english', max_features = 1000)
  
      train_review_subset_x = clean_labeledtrain_reviews[::2]
      train_review_subset_y = labeledtrain_data['sentiment'][::2]
      test_review_subset_x = clean_labeledtrain_reviews[1::2]
      test_review_subset_y = labeledtrain_data['sentiment'][1::2]
      
      train_data_features = vectorizer.fit_transform(train_review_subset_x).toarray()
      
      forest = RandomForestClassifier(n_estimators = 100)
      forest = forest.fit( train_data_features, train_review_subset_y )
      
      test_data_features = vectorizer.transform(test_review_subset_x).toarray()
      
      print forest.score( test_data_features , test_review_subset_y )
      
      #test_data = pd.read_csv('testData.tsv', header=0, delimiter='\t', quoting=3)
      #clean_labeledtrain_reviews = map( lambda r: KaggleWord2VecUtility.review_to_wordlist(r, remove_stopwords=False) , test_data['review'] )
  
  if __name__ == '__main__':
      bag_of_words_model()
  
new_contents: #!/usr/bin/python
  
  import os
  from sklearn.feature_extraction.text import CountVectorizer
  from sklearn.ensemble import RandomForestClassifier
  from KaggleWord2VecUtility import KaggleWord2VecUtility
  import pandas as pd
  import numpy as np
  
  def clean_review_function(review):
      list_of_words = KaggleWord2VecUtility.review_to_wordlist(review, remove_stopwords=False)
      return ' '.join(list_of_words)
  
  from memory_profiler import profile
  
  @profile
  def bag_of_words_model():
      labeledtrain_data = pd.read_csv('labeledTrainData.tsv', header=0, delimiter='\t', quoting=3)
      #unlabeledtrain_data = pd.read_csv('unlabeledTrainData.tsv', header=0, delimiter='\t', quoting=3)
  
      clean_labeledtrain_reviews = map( clean_review_function, labeledtrain_data['review'] )
  
  
      vectorizer = CountVectorizer(analyzer = 'word', tokenizer = None,  preprocessor = None, stop_words = None, max_features = 1000)
  
  
      train_review_subset_x = clean_labeledtrain_reviews[::2]
      train_review_subset_y = labeledtrain_data['sentiment'][::2]
      test_review_subset_x = clean_labeledtrain_reviews[1::2]
      test_review_subset_y = labeledtrain_data['sentiment'][1::2]
      
      train_data_features = vectorizer.fit_transform(train_review_subset_x).toarray()
  
      
      forest = RandomForestClassifier(n_estimators = 100)
      forest = forest.fit( train_data_features, train_review_subset_y )
      
      test_data_features = vectorizer.transform(test_review_subset_x).toarray()
  
      
      print forest.score( test_data_features , test_review_subset_y )
      
      del train_review_subset_x, train_review_subset_y, test_review_subset_x, test_review_subset_y, test_data_features, train_data_features
  
      
      train_data_features = vectorizer.fit_transform(clean_labeledtrain_reviews).toarray()
  
      
      forest = forest.fit( train_data_features, labeledtrain_data['sentiment'] )
  
      
      test_data = pd.read_csv('testData.tsv', header=0, delimiter='\t', quoting=3)
  
      
      clean_test_reviews = map( clean_review_function , test_data['review'] )
  
  
      test_data_features = vectorizer.transform(clean_test_reviews).toarray()
  
  
      result = forest.predict(test_data_features)
      
      output = pd.DataFrame(data={'id': test_data['id'], 'sentiment': result})
      output.to_csv('bag_of_words_model.csv', index=False, quoting=3)
  
  if __name__ == '__main__':
      bag_of_words_model()
  
subject: split of cleaner function, add submission