#Imports
import json
import re
import pandas as pd
import matplotlib.pyplot as plt

#Read data into an array
tweets_data_path = 'twitter_data.txt'

tweets_data = []
tweets_file = open(tweets_data_path, "r")
for line in tweets_file.read():
    try:
        tweet = json.loads(line)
        tweets_data.append(tweet)
    except:
        continue

print(len(tweets_data))

tweets = pd.DataFrame()

tweets['text'] = map(lambda tweet: tweet['text'], tweets_data)
tweets['lang'] = map(lambda tweet: tweet['lang'], tweets_data)
tweets['country'] = map(lambda tweet: tweet['place']['country'] if tweet['place'] != None else None, tweets_data)
tweets['python'] = tweets['text'].apply(lambda tweet: word_in_text('python', tweet))
tweets['javascript'] = tweets['text'].apply(lambda tweet: word_in_text('javascript', tweet))
tweets['ruby'] = tweets['text'].apply(lambda tweet: word_in_text('ruby', tweet))
tweets['programming'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet))
tweets['tutorial'] = tweets['text'].apply(lambda tweet: word_in_text('tutorial', tweet))
tweets['relevant'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet) or word_in_text('tutorial', tweet))
tweets['link'] = tweets['text'].apply(lambda tweet: extract_link(tweet))

tweets_relevant = tweets[tweets['relevant'] == True]
tweets_relevant_with_link = tweets_relevant[tweets_relevant['link'] != '']

#Print number of tweets for each programming language
print (tweets['python'].values_counts()[True])
print (tweets['javascript'].values_counts()[True])
print (tweets['ruby'].values_counts()[True])

#Print relevent tweet count
print (tweets['programming'].value_counts()[True])
print (tweets['tutorial'].value_counts()[True])
print (tweets['relevant'].value_counts()[True])
print (tweets[tweets['relevant'] == True]['python'].value_counts()[True])
print (tweets[tweets['relevant'] == True]['javascript'].value_counts()[True])
print (tweets[tweets['relevant'] == True]['ruby'].value_counts()[True])

#Print links found
print (tweets_relevant_with_link[tweets_relevant_with_link['python'] == True]['link'])
print (tweets_relevant_with_link[tweets_relevant_with_link['javascript'] == True]['link'])
print (tweets_relevant_with_link[tweets_relevant_with_link['ruby'] == True]['link'])

#Graphics
#By Language
tweets_by_lang = tweets['lang'].value_counts()
fig, ax = plt.subplots()
ax.tick_params(axis = 'x', labelsize=15)
ax.tick_params(axis = 'y', labelsize=10)
ax.set_xlabel('Languages', fontsize=15)
ax.set_ylabel('Number of Tweets', fontsize=15)
ax.set_title('Top 5 Languages', fontsize=15, fontweight='bold')
tweets_by_lang[:5].plot(ax = ax, kind = 'bar', color = 'red')

#By Country
tweets_by_country = tweets['country'].value_counts()
fig, ax = plt.subplots()
ax.tick_params(axis = 'x', labelsize=15)
ax.tick_params(axis = 'y', labelsize=10)
ax.set_xlabel('Countries', fontsize=15)
ax.set_ylabel('Number of Tweets', fontsize=15)
ax.set_title('Top 5 Countries', fontsize=15, fontweight='bold')
tweets_by_lang[:5].plot(ax = ax, kind = 'bar', color = 'blue')

#Language Comparison
prg_langs = ['python', 'javascript', 'ruby']
tweets_by_prg_lang = [tweets['python'].value_counts()[True], tweets['javascript'].value_counts()[True], tweets['ruby'].value_counts()[True]]
x_pos = list(range(len(prg_langs)))
width = 0.8
fig, ax = plt.subplots()
plt.bar(x_pos, tweets_by_prg_lang, width, alpha = 1, color = 'g')
ax.set_ylabel('Number of Tweets', fontsize = 15)
ax.set_title('Ranking: Python vs Javascript vs Ruby (Raw Data)', fontsize = 10, fontweight = 'bold')
ax.set_xticks([p + 0.4 * width for p in x_pos])
ax.set_xticklabels(prg_langs)
plt.grid()

tweets_by_prg_lang = [tweets[tweets['relevant'] == True]['python'].value_counts()[True],
                        tweets[tweets['relevant'] == True]['javascript'].value_counts()[True],
                        tweets[tweets['relevant'] == True]['ruby'].value_counts()[True]]
x_pos = list(range(len(prg_langs)))
width = 0.8
fig, ax = plt.subplots()
plt.bar(x_pos, tweets_by_prg_lang, width,alpha=1,color='g')
ax.set_ylabel('Number of tweets', fontsize=15)
ax.set_title('Ranking: python vs. javascript vs. ruby (Relevant data)', fontsize=10, fontweight='bold')
ax.set_xticks([p + 0.4 * width for p in x_pos])
ax.set_xticklabels(prg_langs)
plt.grid()


#Functions
def word_in_text (word, text):
    word = word.lower()
    text = text.lower()
    match = re.search(word, text)
    if match:
        return True
    return False

def extract_link(text):
    regex = r'https?://[^\s<>"]+|www\.[^\s<>"]+'
    match = re.search(regex, text)
    if match:
        return match.group()
    return ''
