#!/usr/bin/env python
# coding: utf-8

# In[2]:

#making some changes here
import pandas as pd
#natural language toolkit
import nltk
#import the stopword list
from nltk.corpus import stopwords
#import tokenizer
from nltk.tokenize import RegexpTokenizer
#import Lematizer
from nltk.stem import WordNetLemmatizer
#Import stemmer.
from nltk.stem.porter import PorterStemmer
from textblob import TextBlob
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from textblob import Word


# PART1 : CLEANING THE DATA FRAME
# 
# input:
#         - elon tweets
#         
#         - tesla stock prices from 5 April 2017 to 5 March 2016
#         
# process: 
#         - clean data frame
#         - clean tweets 
#         - Create 3 dictionaries:
#                                     1. Days : prices
#                                     2. Days : tweets
#                                     3. Days : EMPTY
#                                     
#         [Using days as a key, we will match the prices and  tweets with all days in a year. There will be days with noprice and days with no tweets]
#          
#         
# output:
#         - final data frame called: taula_final
#         
#                 Days | Prices | Tweets per day | Tweets
#         

# In[3]:


elon = pd.read_csv('tweets.csv' ,sep=';;',engine='python')#elon tweets
tesla = pd.read_csv('2017TSLA.csv',  sep = ",")#tesla stock price


# In[4]:


elon.columns


# In[5]:


dataframe = elon['id,created_at,text'].str.split(',', n=2, expand=True)#convert elon data into a dataset
dataframe.head()


# In[6]:


#clean dataframe:

elon['Id'] = dataframe[0]#rename columns
elon['Date'] = dataframe[1]
elon['Text'] = dataframe[2]
elon = elon.drop(columns = ['id,created_at,text','Unnamed: 1'])#delate useless columns
x = [x[0:10] for x in elon['Date']]#list with the dates(no hour)
elon['Date'] = x#change the values of the column 'Date' with the list created before
elon.head()


# In[7]:


#pip install -U textblob 


# In[8]:


#CLEAN TWEETS

stop = stopwords.words('english')
#Making all tweets lowercase
elon['Text'] = elon['Text'].apply(lambda x: " ".join(x.lower() for x in str(x).split()))
#Removing Punctuation
elon['Text'] = elon['Text'].str.replace('[^\w\s]','')

freq = pd.Series(' '.join(elon['Text']).split()).value_counts()[-100:]
#Removing Rare Words
freq = list(freq.index)
elon['Text'] = elon['Text'].apply(lambda x: " ".join(x for x in x.split() if x not in freq))


elon['Text'] = elon['Text'].apply(lambda x: " ".join(x for x in x.split() if x not in stop))

#Common word
common = pd.Series(' '.join(elon['Text']).split()).value_counts()[:10]
common

#Rare words
rare = pd.Series(' '.join(elon['Text']).split()).value_counts()[-100:]
rare

#Delete rare words
rare_del = list(rare.index)
elon['Text'] = elon['Text'].apply(lambda x: " ".join(x for x in x.split() if x not in rare_del))
elon['Text'].head()


st = PorterStemmer()
elon['Text'][:5].apply(lambda x: " ".join([st.stem(word) for word in x.split()]))

elon['Text'] = elon['Text'].apply(lambda x: " ".join([Word(word).lemmatize() for word in x.split()]))


elon['Text'][:5].apply(lambda x: str(TextBlob(x).correct()))



elon['Text'].head()


# In[9]:


d = elon['Text']#delate first b
for i in range(len(d)):
    if d[i][0]=='b':
        d[i] = d[i][1:]


elon['Text'] = d


# In[10]:


elon.head()


# In[12]:


#number of tweets during the same day

def tweets_per_day(date):#COUNT NUMBER OF TWEETS IN A DAY 
    date = list(date)
    fe = {}
    for i in range(len(date)):
        if date[i] in fe:#executes if its true
            fe[date[i]] += 1#if its true, sum one to the value
        else: 

            fe[date[i]] = 1
    t_per_day = fe.items()
    d = [[x[0][:],x[1]] for x in t_per_day]
    return d[1:] 

elon_date = tweets_per_day(elon['Date'])
elon_date[:5]


# In[13]:


frequency = [x[1] for x in elon_date]
text = list(elon['Text'][1:])


# In[14]:


#using the number of tweets per day, create a dictionary where the key is the day and the value the tweets 
def create_vector_of_tweets(frequency=frequency, text=text): 
    t=[]
    k = 0 #counter
    for i in frequency:#iterate over the list of frequencies
        u = []
        for j in range(1,i+1): # frequency number == number of tweets inside the sme
            k += 1 
            
            if k < len(text):
                u.append(text[k])#appload all the tweets for the given frequency

        t.append(u)# after iterating the frequency, appload the tweets tp the list
    dies = [x[0] for x in elon_date]
    dies_tweet =[x for x in zip(dies,t)]#zip days and the list of list of tweets
    return dict(dies_tweet)


    


# In[15]:


create_vector_of_tweets()


# In[16]:


#prices of Tesla stock's

date_tesla =list(tesla['Date'])[::-1]#Tesla 
Open = list(tesla['Open'])
Close = list(tesla['Close'])
high = list(tesla['High'])


# In[17]:


dif_per = [(x[1]-x[0])/x[0] for x in zip(Open, Close)]#list of prices
dif_open_close = [x for x in zip(tesla['Date'], dif_per)][::-1]#days and prices
dif_open_close[:5]


# In[18]:



def generate_dates():# create a list of all the days during the given dates
    years = [2016,2017]
    months =  ['0'+str(i) for i in range(1,10)]+[i for i in range(10,13)]
    short_months = ['04','06','09',11]
    days = ['0'+str(i) for i in range(1,10)]+[i for i in range(10,32)]
    item=[]
    li=[29,30,31]
    for i in years:
        for j in months: 
            for k in days:
                item.append(str(i)+'-'+str(j)+'-'+str(k))
                for y in li:
                    if j=='02' and k==y:       
                        item.remove(str(i)+'-'+str(j)+'-'+str(y))
        for u in short_months:
             if k==31:
                item.remove(str(i)+'-'+str(u)+'-'+str(k))
        
                
    
   
    return item[::-1][271:]
    
#def compare_tweets_changes(freq, changes):
dates = generate_dates()
    
dates[0:6]    
    


# In[19]:


from collections import Counter 
  
def join_dates_X(dates, t_p=dif_open_close):
    dates = generate_dates()
    #dictionary with days
    t_prices = dict(zip(dates,[1000 for x in range(len(dates))]))#dictionary with days
    tesla_prices = dict(t_p)#dictionary with prices
  
    # initialising dictionaries 
    d1 = Counter(t_prices) 
    d2 = Counter(tesla_prices) 

    # combining dictionaries 
    # using Counter 
    day_price = d1 + d2
    f = {i:day_price[i]-1000 for i in day_price}
    return f


# In[23]:


join_dates_X(dates, t_p=dif_open_close)#prices of all days


# In[24]:


t_p=create_vector_of_tweets()
dates = generate_dates()
t_prices = dict(zip(dates,[ None for x in range(len(dates))]))
#tesla_prices = t_p
#d1 = Counter(t_prices) 
#d2 = Counter(tesla_prices) 

for character in t_prices:
    for key, value in t_p.items():
        if key == character:
            t_prices[character] = value
            


# In[25]:


t_prices


# In[26]:


dates_t_prices = join_dates_X(dates, dif_open_close)#prices of all days
dates_elon_tweets = join_dates_X(dates, elon_date)#tweets for all days

enumerar = dict(zip(dates,[ x for x in range(len(dates))]))

#tweets for all days)#tweets for all days


# In[1]:



def concadenate_lists(list1,list2,list3,list4):#same len()
    t = { k: [list1[k],list2[k],list3[k],list4[k]] for k in list1 }
    return t
    

date_price_tweet = concadenate_lists(dates_t_prices,dates_elon_tweets,t_prices,enumerar)
date_price_tweet


# In[28]:


dat = [[x,date_price_tweet[x][0],date_price_tweet[x][1],date_price_tweet[x][2] ] for x in date_price_tweet]
taula_final = pd.DataFrame(dat, columns = ['Date','Price','Number tweets','Tweets'])[:370]
llargaria = [(len(str(x))-4) for x in taula_final['Tweets']]
taula_final.insert( loc=4, column='Llargaria', value=llargaria)
enumerar = [x for x in range(len(llargaria))]
taula_final.insert( loc=1, column='Enumerar', value=enumerar)


# In[29]:


taula_final.head()


# PART 2: Common words and price plots
# 
# PROCESS: - Look for biggest changes in price
# 

# In[32]:


date_price_tweet


# In[37]:


date_price_tweet
def get_m_values(dicci= date_price_tweet, n=20, dataset = taula_final):
    # match values of the prices with it's dateand tweets

    list1 = dataset['Price'] 
    x = sorted(list(list1))
    maxim = x[-n:]
    minim = x[:n]# biggest changes in price #n: threshold
    
    # match values of the prices with it's dateand tweets
    ma =[[x,date_price_tweet[x][0],date_price_tweet[x][2],date_price_tweet[x][3] ] for x in date_price_tweet if date_price_tweet[x][0] in maxim]
    mi =[[x,date_price_tweet[x][0],date_price_tweet[x][2],date_price_tweet[x][3] ] for x in date_price_tweet if date_price_tweet[x][0] in minim]
  

    t = [ma[::-1],mi]
    return t


# In[38]:


get_m_values()


# In[39]:


maxi = get_m_values()[0]
mini = get_m_values()[1]
points_maxims=[[x[0],x[1],x[3]] for x in maxi]
points_minims=[[x[0],x[1],x[3]] for x in mini]


# In[40]:



def get_m_words(list2,m,threshold=3):#m=[maxi,mini]# from the dates, get the most frequent words in different tweets

    maxim_i_minims =list2  
    tweets_m = [x[2] for x in m]
    t = [str(i).split() for i in tweets_m]
    frequencies = {}
    words = t
    for i in words:
        for j in range(len(i)):
            if i[j] in frequencies:#executes if its true
                frequencies[i[j]] += 1#if its true, sum one to the value

            else: 

                frequencies[i[j]] = 1
            #if its false (is not in dict) 
    resultat = [[x,frequencies[x]] for x in frequencies if int(frequencies[x]) >threshold ]
    return resultat


# In[41]:


maxim_words =  get_m_words(get_m_values(),maxi) 

mini_words = get_m_words(get_m_values(),mini) 


# In[42]:


print('Words more used during stock gains : '+str(maxim_words))


# In[43]:


print('Words more used during stock drops : '+str(mini_words))


# In[44]:


maxim_words


# In[45]:


mini_words 


# In[46]:


get_ipython().run_line_magic('matplotlib', 'inline')
from matplotlib import pyplot as plt


# In[47]:


def standarize(x):
    return (x -min(x))/(max(x)-min(x))


# In[54]:


import numpy as np
#not_weekend =[x for x in taula_final['Price'] if x!=0]
standa_volume = standarize(tesla['Volume'])/10
not_weekend =[x for x in taula_final['Price'] ]
average_volum =  np.mean(standa_volume)
average_prices = np.mean(not_weekend)

plt.plot(not_weekend,'blue')
plt.plot([np.mean(average_prices) for x in range(len(not_weekend))], 'green')
plt.scatter([x[2] for x in points_maxims],[x[1] for x in points_maxims], color='orange')
plt.scatter([x[2] for x in points_minims],[x[1] for x in points_minims],color='brown')
plt.legend(['Stock Prices changes','Average price changes','Max prices change','Min prices change'],loc ='lower left',fontsize = 7)


# In[55]:


import numpy as np
not_weekend =[x for x in taula_final['Price'] if x!=0]

average_volum =  np.mean(standa_volume)
average_prices = np.mean(not_weekend)
plt.plot(not_weekend,'blue')
plt.plot(standa_volume, 'violet')
plt.plot([np.mean(average_volum) for x in range(len(standa_volume))], 'red')
plt.plot([np.mean(average_prices) for x in range(len(standa_volume))], 'green')
plt.legend(['Stock Prices changes','Volume','Average volume','Average price changes'],loc ='lower left',fontsize = 7)







