import pandas as pd
import json
import os
import nltk
from pandas.io.json import json_normalize

#Replace this with the location of twitter data
tweets_path = '/Users/jd186046/Documents/Trainings/Think Big Bootcamp/Project/Data/tweet/'

tweets = []

list_files = os.listdir(tweets_path)
for tw_file in list_files:
    with open(tweets_path + tw_file) as f:
        for line in f:
            tweets.append(json.loads(line))


column_list = [    
#     'contributors',
#     'coordinates',
#     'created_at',
#     'display_text_range',
#     'entities.hashtags',
#     'entities.media',
#     'entities.symbols',
#     'entities.urls',
#     'entities.user_mentions',
#     'extended_entities.media',
#     'extended_tweet.display_text_range',
#     'extended_tweet.entities.hashtags',
#     'extended_tweet.entities.media',
#     'extended_tweet.entities.symbols',
#     'extended_tweet.entities.urls',
#     'extended_tweet.entities.user_mentions',
#     'extended_tweet.extended_entities.media',
#     'extended_tweet.full_text',
     'favorite_count',
#     'favorited',
#     'filter_level',
#     'geo',
     'id',
#     'id_str',
#     'in_reply_to_screen_name',
#     'in_reply_to_status_id',
#     'in_reply_to_status_id_str',
#     'in_reply_to_user_id',
#     'in_reply_to_user_id_str',
#     'is_quote_status',
#     'lang',
#     'place',
#     'possibly_sensitive',
     'quote_count',
#     'quoted_status.contributors',
#     'quoted_status.coordinates',
#     'quoted_status.created_at',
#     'quoted_status.display_text_range',
#     'quoted_status.entities.hashtags',
#     'quoted_status.entities.media',
#     'quoted_status.entities.symbols',
#     'quoted_status.entities.urls',
#     'quoted_status.entities.user_mentions',
#     'quoted_status.extended_entities.media',
#     'quoted_status.extended_tweet.display_text_range',
#     'quoted_status.extended_tweet.entities.hashtags',
#     'quoted_status.extended_tweet.entities.media',
#     'quoted_status.extended_tweet.entities.symbols',
#     'quoted_status.extended_tweet.entities.urls',
#     'quoted_status.extended_tweet.entities.user_mentions',
#     'quoted_status.extended_tweet.extended_entities.media',
#     'quoted_status.extended_tweet.full_text',
#     'quoted_status.favorite_count',
#     'quoted_status.favorited',
#     'quoted_status.filter_level',
#     'quoted_status.geo',
#     'quoted_status.id',
#     'quoted_status.id_str',
#     'quoted_status.in_reply_to_screen_name',
#     'quoted_status.in_reply_to_status_id',
#     'quoted_status.in_reply_to_status_id_str',
#     'quoted_status.in_reply_to_user_id',
#     'quoted_status.in_reply_to_user_id_str',
#     'quoted_status.is_quote_status',
#     'quoted_status.lang',
#     'quoted_status.place',
#     'quoted_status.place.bounding_box.coordinates',
#     'quoted_status.place.bounding_box.type',
#     'quoted_status.place.country',
#     'quoted_status.place.country_code',
#     'quoted_status.place.full_name',
#     'quoted_status.place.id',
#     'quoted_status.place.name',
#     'quoted_status.place.place_type',
#     'quoted_status.place.url',
#     'quoted_status.possibly_sensitive',
#     'quoted_status.quote_count',
#     'quoted_status.quoted_status_id',
#     'quoted_status.quoted_status_id_str',
#     'quoted_status.reply_count',
#     'quoted_status.retweet_count',
#     'quoted_status.retweeted',
#     'quoted_status.source',
#     'quoted_status.text',
#     'quoted_status.truncated',
#     'quoted_status.user.contributors_enabled',
#     'quoted_status.user.created_at',
#     'quoted_status.user.default_profile',
#     'quoted_status.user.default_profile_image',
#     'quoted_status.user.description',
#     'quoted_status.user.favourites_count',
#     'quoted_status.user.follow_request_sent',
#     'quoted_status.user.followers_count',
#     'quoted_status.user.following',
#     'quoted_status.user.friends_count',
#     'quoted_status.user.geo_enabled',
#     'quoted_status.user.id',
#     'quoted_status.user.id_str',
#     'quoted_status.user.is_translator',
#     'quoted_status.user.lang',
#     'quoted_status.user.listed_count',
#     'quoted_status.user.location',
#     'quoted_status.user.name',
#     'quoted_status.user.notifications',
#     'quoted_status.user.profile_background_color',
#     'quoted_status.user.profile_background_image_url',
#     'quoted_status.user.profile_background_image_url_https',
#     'quoted_status.user.profile_background_tile',
#     'quoted_status.user.profile_banner_url',
#     'quoted_status.user.profile_image_url',
#     'quoted_status.user.profile_image_url_https',
#     'quoted_status.user.profile_link_color',
#     'quoted_status.user.profile_sidebar_border_color',
#     'quoted_status.user.profile_sidebar_fill_color',
#     'quoted_status.user.profile_text_color',
#     'quoted_status.user.profile_use_background_image',
#     'quoted_status.user.protected',
#     'quoted_status.user.screen_name',
#     'quoted_status.user.statuses_count',
#     'quoted_status.user.time_zone',
#     'quoted_status.user.translator_type',
#     'quoted_status.user.url',
#     'quoted_status.user.utc_offset',
#     'quoted_status.user.verified',
#     'quoted_status_id',
#     'quoted_status_id_str',
     'reply_count',
     'retweet_count',
     'retweeted',
#     'retweeted_status.contributors',
#     'retweeted_status.coordinates',
#     'retweeted_status.created_at',
#     'retweeted_status.display_text_range',
#     'retweeted_status.entities.hashtags',
#     'retweeted_status.entities.media',
#     'retweeted_status.entities.symbols',
#     'retweeted_status.entities.urls',
#     'retweeted_status.entities.user_mentions',
#     'retweeted_status.extended_entities.media',
#     'retweeted_status.extended_tweet.display_text_range',
#     'retweeted_status.extended_tweet.entities.hashtags',
#     'retweeted_status.extended_tweet.entities.media',
#     'retweeted_status.extended_tweet.entities.symbols',
#     'retweeted_status.extended_tweet.entities.urls',
#     'retweeted_status.extended_tweet.entities.user_mentions',
#     'retweeted_status.extended_tweet.extended_entities.media',
#     'retweeted_status.extended_tweet.full_text',
#     'retweeted_status.favorite_count',
#     'retweeted_status.favorited',
#     'retweeted_status.filter_level',
#     'retweeted_status.geo',
#     'retweeted_status.id',
#     'retweeted_status.id_str',
#     'retweeted_status.in_reply_to_screen_name',
#     'retweeted_status.in_reply_to_status_id',
#     'retweeted_status.in_reply_to_status_id_str',
#     'retweeted_status.in_reply_to_user_id',
#     'retweeted_status.in_reply_to_user_id_str',
#     'retweeted_status.is_quote_status',
#     'retweeted_status.lang',
#     'retweeted_status.place',
#     'retweeted_status.place.bounding_box.coordinates',
#     'retweeted_status.place.bounding_box.type',
#     'retweeted_status.place.country',
#     'retweeted_status.place.country_code',
#     'retweeted_status.place.full_name',
#     'retweeted_status.place.id',
#     'retweeted_status.place.name',
#     'retweeted_status.place.place_type',
#     'retweeted_status.place.url',
#     'retweeted_status.possibly_sensitive',
#     'retweeted_status.quote_count',
#     'retweeted_status.quoted_status.contributors',
#     'retweeted_status.quoted_status.coordinates',
#     'retweeted_status.quoted_status.created_at',
#     'retweeted_status.quoted_status.display_text_range',
#     'retweeted_status.quoted_status.entities.hashtags',
#     'retweeted_status.quoted_status.entities.media',
#     'retweeted_status.quoted_status.entities.symbols',
#     'retweeted_status.quoted_status.entities.urls',
#     'retweeted_status.quoted_status.entities.user_mentions',
#     'retweeted_status.quoted_status.extended_entities.media',
#     'retweeted_status.quoted_status.extended_tweet.display_text_range',
#     'retweeted_status.quoted_status.extended_tweet.entities.hashtags',
#     'retweeted_status.quoted_status.extended_tweet.entities.media',
#     'retweeted_status.quoted_status.extended_tweet.entities.symbols',
#     'retweeted_status.quoted_status.extended_tweet.entities.urls',
#     'retweeted_status.quoted_status.extended_tweet.entities.user_mentions',
#     'retweeted_status.quoted_status.extended_tweet.extended_entities.media',
#     'retweeted_status.quoted_status.extended_tweet.full_text',
#     'retweeted_status.quoted_status.favorite_count',
#     'retweeted_status.quoted_status.favorited',
#     'retweeted_status.quoted_status.filter_level',
#     'retweeted_status.quoted_status.geo',
#     'retweeted_status.quoted_status.id',
#     'retweeted_status.quoted_status.id_str',
#     'retweeted_status.quoted_status.in_reply_to_screen_name',
#     'retweeted_status.quoted_status.in_reply_to_status_id',
#     'retweeted_status.quoted_status.in_reply_to_status_id_str',
#     'retweeted_status.quoted_status.in_reply_to_user_id',
#     'retweeted_status.quoted_status.in_reply_to_user_id_str',
#     'retweeted_status.quoted_status.is_quote_status',
#     'retweeted_status.quoted_status.lang',
#     'retweeted_status.quoted_status.place',
#     'retweeted_status.quoted_status.possibly_sensitive',
#     'retweeted_status.quoted_status.quote_count',
#     'retweeted_status.quoted_status.quoted_status_id',
#     'retweeted_status.quoted_status.quoted_status_id_str',
#     'retweeted_status.quoted_status.reply_count',
#     'retweeted_status.quoted_status.retweet_count',
#     'retweeted_status.quoted_status.retweeted',
#     'retweeted_status.quoted_status.source',
#     'retweeted_status.quoted_status.text',
#     'retweeted_status.quoted_status.truncated',
#     'retweeted_status.quoted_status.user.contributors_enabled',
#     'retweeted_status.quoted_status.user.created_at',
#     'retweeted_status.quoted_status.user.default_profile',
#     'retweeted_status.quoted_status.user.default_profile_image',
#     'retweeted_status.quoted_status.user.description',
#     'retweeted_status.quoted_status.user.favourites_count',
#     'retweeted_status.quoted_status.user.follow_request_sent',
#     'retweeted_status.quoted_status.user.followers_count',
#     'retweeted_status.quoted_status.user.following',
#     'retweeted_status.quoted_status.user.friends_count',
#     'retweeted_status.quoted_status.user.geo_enabled',
#     'retweeted_status.quoted_status.user.id',
#     'retweeted_status.quoted_status.user.id_str',
#     'retweeted_status.quoted_status.user.is_translator',
#     'retweeted_status.quoted_status.user.lang',
#     'retweeted_status.quoted_status.user.listed_count',
#     'retweeted_status.quoted_status.user.location',
#     'retweeted_status.quoted_status.user.name',
#     'retweeted_status.quoted_status.user.notifications',
#     'retweeted_status.quoted_status.user.profile_background_color',
#     'retweeted_status.quoted_status.user.profile_background_image_url',
#     'retweeted_status.quoted_status.user.profile_background_image_url_https',
#     'retweeted_status.quoted_status.user.profile_background_tile',
#     'retweeted_status.quoted_status.user.profile_banner_url',
#     'retweeted_status.quoted_status.user.profile_image_url',
#     'retweeted_status.quoted_status.user.profile_image_url_https',
#     'retweeted_status.quoted_status.user.profile_link_color',
#     'retweeted_status.quoted_status.user.profile_sidebar_border_color',
#     'retweeted_status.quoted_status.user.profile_sidebar_fill_color',
#     'retweeted_status.quoted_status.user.profile_text_color',
#     'retweeted_status.quoted_status.user.profile_use_background_image',
#     'retweeted_status.quoted_status.user.protected',
#     'retweeted_status.quoted_status.user.screen_name',
#     'retweeted_status.quoted_status.user.statuses_count',
#     'retweeted_status.quoted_status.user.time_zone',
#     'retweeted_status.quoted_status.user.translator_type',
#     'retweeted_status.quoted_status.user.url',
#     'retweeted_status.quoted_status.user.utc_offset',
#     'retweeted_status.quoted_status.user.verified',
#     'retweeted_status.quoted_status_id',
#     'retweeted_status.quoted_status_id_str',
#     'retweeted_status.reply_count',
#     'retweeted_status.retweet_count',
#     'retweeted_status.retweeted',
#     'retweeted_status.source',
#     'retweeted_status.text',
#    'retweeted_status.truncated',
#     'retweeted_status.user.contributors_enabled',
#     'retweeted_status.user.created_at',
#     'retweeted_status.user.default_profile',
#     'retweeted_status.user.default_profile_image',
#     'retweeted_status.user.description',
#     'retweeted_status.user.favourites_count',
#     'retweeted_status.user.follow_request_sent',
     'retweeted_status.user.followers_count',
#     'retweeted_status.user.following',
     'retweeted_status.user.friends_count',
#     'retweeted_status.user.geo_enabled',
     'retweeted_status.user.id',
#     'retweeted_status.user.id_str',
#     'retweeted_status.user.is_translator',
#     'retweeted_status.user.lang',
#     'retweeted_status.user.listed_count',
#     'retweeted_status.user.location',
     'retweeted_status.user.name',
#     'retweeted_status.user.notifications',
#     'retweeted_status.user.profile_background_color',
#     'retweeted_status.user.profile_background_image_url',
#     'retweeted_status.user.profile_background_image_url_https',
#     'retweeted_status.user.profile_background_tile',
#     'retweeted_status.user.profile_banner_url',
#     'retweeted_status.user.profile_image_url',
#     'retweeted_status.user.profile_image_url_https',
#     'retweeted_status.user.profile_link_color',
#     'retweeted_status.user.profile_sidebar_border_color',
#     'retweeted_status.user.profile_sidebar_fill_color',
#     'retweeted_status.user.profile_text_color',
#     'retweeted_status.user.profile_use_background_image',
#     'retweeted_status.user.protected',
     'retweeted_status.user.screen_name',
#     'retweeted_status.user.statuses_count',
#     'retweeted_status.user.time_zone',
#     'retweeted_status.user.translator_type',
#     'retweeted_status.user.url',
#     'retweeted_status.user.utc_offset',
#     'retweeted_status.user.verified',
#     'retweeted_status.withheld_in_countries',
#     'source',
     'text',
#     'timestamp_ms',
#     'truncated',
#     'user.contributors_enabled',
#     'user.created_at',
#     'user.default_profile',
#     'user.default_profile_image',
#     'user.description',
#     'user.favourites_count',
#     'user.follow_request_sent',
     'user.followers_count',
#     'user.following',
     'user.friends_count',
#     'user.geo_enabled',
     'user.id',
#     'user.id_str',
#     'user.is_translator',
#     'user.lang',
#     'user.listed_count',
#     'user.location',
     'user.name',
#     'user.notifications',
#     'user.profile_background_color',
#     'user.profile_background_image_url',
#     'user.profile_background_image_url_https',
#     'user.profile_background_tile',
#     'user.profile_banner_url',
#     'user.profile_image_url',
#     'user.profile_image_url_https',
#     'user.profile_link_color',
#     'user.profile_sidebar_border_color',
#     'user.profile_sidebar_fill_color',
#     'user.profile_text_color',
#     'user.profile_use_background_image',
#     'user.protected',
     'user.screen_name',
#     'user.statuses_count',
#     'user.time_zone',
#     'user.translator_type',
#     'user.url',
#     'user.utc_offset',
#     'user.verified',
#     'withheld_in_countries'
]

tweets_df = json_normalize(tweets)
tweets_min_df = tweets_df[column_list]

print("Parsed {0} tweets in total".format(len(tweets_min_df)))

tweets_min_df.to_csv("tweets.csv", header=True, encoding="utf-8")            

