old_contents: # coding=utf-8
  import sys
  reload(sys)
  sys.setdefaultencoding('utf-8')
  
  # sc is an existing SparkContext.
  from pyspark.sql import SQLContext
  sqlContext = SQLContext(sc)
  
  directory = "/Volumes/JS'S FIT/gnip-json"
  datasets = sqlContext.read.json(directory)
  
  file_count = datasets.where(datasets['verb'].isNull()).count()
  assert file_count == 21888
  # expecting 21888
  
  info_dataset = datasets.select('info')
  info_dataset.registerTempTable('info')
  all_tweets_count = info_dataset.select('info.activity_count').groupBy().sum('activity_count').collect()
  # expecting 2682988
  
  all_posts = datasets.where(datasets['verb'] == 'post')
  all_posts_count = all_posts.count()
  assert all_posts_count == 1570398
  print '{} posts'.format(all_posts_count)
  # expecting 1570398
  
  all_shares = datasets.where(datasets['verb'] == 'share')
  all_shares_count = all_shares.count()
  assert all_shares_count == 1112590
  print '{} shares'.format(all_shares_count)
  # expecting 1112590
  
  assert all_tweets_count[0][0] == all_posts_count + all_shares_count
  
  sample_seed = 2016
  number_of_instructional_samples = 30
  sample_posts = all_posts.rdd.takeSample(False, number_of_instructional_samples, sample_seed)
  sample_posts_count = len(sample_posts)
  print '{} sample posts'.format(sample_posts_count)
  
  sample_posts_file = "./output/sample_posts.json"
  sample_posts_jsons = sample_posts.toJSON().collect()
  with open(sample_posts_file, 'a') as f:
  	for post in sample_posts_jsons:
  		f.write(post)
  		f.write('\n')
  
new_contents: # coding=utf-8
  import sys
  reload(sys)
  sys.setdefaultencoding('utf-8')
  
  # sc is an existing SparkContext.
  from pyspark.sql import SQLContext
  sqlContext = SQLContext(sc)
  
  directory = "/Volumes/JS'S FIT/gnip-json"
  datasets = sqlContext.read.json(directory)
  
  file_count = datasets.where(datasets['verb'].isNull()).count()
  # expecting 21888
  assert file_count == 21888
  
  info_dataset = datasets.select('info')
  info_dataset.registerTempTable('info')
  all_tweets_count = info_dataset.select('info.activity_count').groupBy().sum('activity_count').collect()
  # expecting 2682988
  
  all_posts = datasets.where(datasets['verb'] == 'post')
  all_posts_count = all_posts.count()
  # expecting 1570398
  assert all_posts_count == 1570398
  print '{} posts'.format(all_posts_count)
  
  all_shares = datasets.where(datasets['verb'] == 'share')
  all_shares_count = all_shares.count()
  # expecting 1112590
  assert all_shares_count == 1112590
  print '{} shares'.format(all_shares_count)
  
  assert all_tweets_count[0][0] == all_posts_count + all_shares_count
  
  sample_seed = 2016
  number_of_instructional_samples = 30
  sample_posts = all_posts.rdd.takeSample(False, number_of_instructional_samples, sample_seed)
  sample_posts_count = len(sample_posts)
  print '{} sample posts'.format(sample_posts_count)
  
  sample_posts_file = "./output/sample_posts.json"
  sample_posts_jsons = sample_posts.toJSON().collect()
  with open(sample_posts_file, 'a') as f:
  	for post in sample_posts_jsons:
  		f.write(post)
  		f.write('\n')
  
subject: Rearrange comments