

import math, re, string, requests, json
from itertools import product
from inspect import getsourcefile
from os.path import abspath, join, dirname
import tweepy
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import csv
import cgi
##Constants##

# (empirically derived mean sentiment intensity rating increase for booster words)
B_INCR = 0.293
B_DECR = -0.293

# (empirically derived mean sentiment intensity rating increase for using
# ALLCAPs to emphasize a word)
C_INCR = 0.733

N_SCALAR = -0.74

# for removing punctuation
REGEX_REMOVE_PUNCTUATION = re.compile('[%s]' % re.escape(string.punctuation))

PUNC_LIST = [".", "!", "?", ",", ";", ":", "-", "'", "\"",
             "!!", "!!!", "??", "???", "?!?", "!?!", "?!?!", "!?!?"]
NEGATE = \
["aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
 "ain't", "aren't", "can't", "couldn't", "daren't", "didn't", "doesn't",
 "dont", "hadnt", "hasnt", "havent", "isnt", "mightnt", "mustnt", "neither",
 "don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't",
 "neednt", "needn't", "never", "none", "nope", "nor", "not", "nothing", "nowhere",
 "oughtnt", "shant", "shouldnt", "uhuh", "wasnt", "werent",
 "oughtn't", "shan't", "shouldn't", "uh-uh", "wasn't", "weren't",
 "without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"]

# booster/dampener 'intensifiers' or 'degree adverbs'
# http://en.wiktionary.org/wiki/Category:English_degree_adverbs

BOOSTER_DICT = \
{"absolutely": B_INCR, "amazingly": B_INCR, "awfully": B_INCR, "completely": B_INCR, "considerably": B_INCR,
 "decidedly": B_INCR, "deeply": B_INCR, "effing": B_INCR, "enormously": B_INCR,
 "entirely": B_INCR, "especially": B_INCR, "exceptionally": B_INCR, "extremely": B_INCR,
 "fabulously": B_INCR, "flipping": B_INCR, "flippin": B_INCR,
 "fricking": B_INCR, "frickin": B_INCR, "frigging": B_INCR, "friggin": B_INCR, "fully": B_INCR, "fucking": B_INCR,
 "greatly": B_INCR, "hella": B_INCR, "highly": B_INCR, "hugely": B_INCR, "incredibly": B_INCR,
 "intensely": B_INCR, "majorly": B_INCR, "more": B_INCR, "most": B_INCR, "particularly": B_INCR,
 "purely": B_INCR, "quite": B_INCR, "really": B_INCR, "remarkably": B_INCR,
 "so": B_INCR, "substantially": B_INCR,
 "thoroughly": B_INCR, "totally": B_INCR, "tremendously": B_INCR,
 "uber": B_INCR, "unbelievably": B_INCR, "unusually": B_INCR, "utterly": B_INCR,
 "very": B_INCR,
 "almost": B_DECR, "barely": B_DECR, "hardly": B_DECR, "just enough": B_DECR,
 "kind of": B_DECR, "kinda": B_DECR, "kindof": B_DECR, "kind-of": B_DECR,
 "less": B_DECR, "little": B_DECR, "marginally": B_DECR, "occasionally": B_DECR, "partly": B_DECR,
 "scarcely": B_DECR, "slightly": B_DECR, "somewhat": B_DECR,
 "sort of": B_DECR, "sorta": B_DECR, "sortof": B_DECR, "sort-of": B_DECR}

# check for special case idioms using a sentiment-laden keyword known to VADER
SPECIAL_CASE_IDIOMS = {"the shit": 3, "the bomb": 3, "bad ass": 1.5, "yeah right": -2,
                       "cut the mustard": 2, "kiss of death": -1.5, "hand to mouth": -2}




consumer_key = 'XXXXXXXXXXXXXXXXXXXXXXXXX'
consumer_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
access_token = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
access_token_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
 



def FetchTweets(screen_name):

    auth = OAuthHandler(consumer_key, consumer_secret)
            # set access token and secret
    auth.set_access_token(access_token, access_token_secret)
            # create tweepy API object to fetch tweets
    api = tweepy.API(auth)
    
    #initialize a list to hold all the tweepy Tweets
    alltweets = []  
    
    #make initial request for most recent tweets (200 is the maximum allowed count)
    new_tweets = api.user_timeline(screen_name = screen_name,count=200)
    
    #save most recent tweets
    
    ###alltweets.extend(new_tweets)
    
    #save the id of the oldest tweet less one
    ###oldest = alltweets[-1].id - 1
    
    #keep grabbing tweets until there are no tweets left to grab
    
    #transform the tweepy tweets into a 2D array that will populate the csv 
    outtweets = [[tweet.id_str, tweet.created_at, tweet.text.encode("utf-8")] for tweet in new_tweets]
    outtweets2 = [[tweet.text.encode("utf-8")] for tweet in new_tweets]
    #write the csv  
    with open('%s_tweets.csv' % screen_name, 'wb') as f:
        writer = csv.writer(f)
        writer.writerow(["id","created_at","text"])
        writer.writerows(outtweets)
    
    pass

    with open('%s_tweets2.csv' % screen_name, 'wb') as f:
        writer = csv.writer(f)
        writer.writerow(["text"])
        writer.writerows(outtweets2)
    
    pass

##Static methods##

def negated(input_words, include_nt=True):
    """
    Determine if input contains negation words
    """
    neg_words = []
    neg_words.extend(NEGATE)
    for word in neg_words:
        if word in input_words:
            return True
    if include_nt:
        for word in input_words:
            if "n't" in word:
                return True
    if "least" in input_words:
        i = input_words.index("least")
        if i > 0 and input_words[i-1] != "at":
            return True
    return False


def normalize(score, alpha=15):
    """
    Normalize the score to be between -1 and 1 using an alpha that
    approximates the max expected value
    """
    norm_score = score/math.sqrt((score*score) + alpha)
    if norm_score < -1.0: 
        return -1.0
    elif norm_score > 1.0:
        return 1.0
    else:
        return norm_score


def DifferentiateAllEmaphasized(words):
    """
    Check whether just some words in the input are ALL CAPS
    :param list words: The words to inspect
    :returns: `True` if some but not all items in `words` are ALL CAPS
    """
    is_different = False
    AllEmaphasizedWords = 0
    for word in words:
        if word.isupper():
            AllEmaphasizedWords += 1
    EmaphasizedDiff = len(words) - AllEmaphasizedWords
    if EmaphasizedDiff > 0 and EmaphasizedDiff < len(words):
        is_different = True
    return is_different


def PrecedingWordEffect(word, ScoreAddn, is_cap_diff):
    """
    Check if the preceding words increase, decrease, or negate/nullify the
    ScoreAddn
    """
    scalar = 0.0
    WordLower = word.lower()
    if WordLower in BOOSTER_DICT:
        scalar = BOOSTER_DICT[WordLower]
        if ScoreAddn < 0:
            scalar *= -1
        #check if booster/dampener word is in ALLCAPS (while others aren't)
        if word.isupper() and is_cap_diff:
            if ScoreAddn > 0:
                scalar += C_INCR
            else: scalar -= C_INCR
    return scalar

class SentiText(object):
    """
    Identify sentiment-relevant string-level properties of input text.
    """
    def __init__(self, text):
        if not isinstance(text, str):
            text = str(text.encode('utf-8'))
        self.text = text
        self.WordsAndEmoticons = self._WordsAndEmoticons()
        # doesn't separate words from\
        # adjacent punctuation (keeps emoticons & contractions)
        self.is_cap_diff = DifferentiateAllEmaphasized(self.WordsAndEmoticons)

    def PunctuationRem(self):
        """
        Returns mapping of form:
        {
            'cat,': 'cat',
            ',cat': 'cat',
        }
        """
        no_punc_text = REGEX_REMOVE_PUNCTUATION.sub('', self.text)
        # removes punctuation (but loses emoticons & contractions)
        words_only = no_punc_text.split()
        # remove singletons
        words_only = set( w for w in words_only if len(w) > 1 )
        # the product gives ('cat', ',') and (',', 'cat')
        punc_before = {''.join(p): p[1] for p in product(PUNC_LIST, words_only)}
        punc_after = {''.join(p): p[0] for p in product(words_only, PUNC_LIST)}
        words_punc_dict = punc_before
        words_punc_dict.update(punc_after)
        return words_punc_dict

    def _WordsAndEmoticons(self):
        """
        Removes leading and trailing puncutation
        Leaves contractions and most emoticons
            Does not preserve punc-plus-letter emoticons (e.g. :D)
        """
        wes = self.text.split()
        words_punc_dict = self.PunctuationRem()
        wes = [we for we in wes if len(we) > 1]
        for i, we in enumerate(wes):
            if we in words_punc_dict:
                wes[i] = words_punc_dict[we]
        return wes

class SentimentIntensityAnalyzer(object):
    """
    Give a sentiment intensity score to sentences.
    """
    def __init__(self, lexicon_file="lexicons.txt"):
        _this_module_file_path_ = abspath(getsourcefile(lambda:0))
        lexicon_full_filepath = join(dirname(_this_module_file_path_), lexicon_file)
        with open(lexicon_full_filepath) as f:
            self.lexicon_full_filepath = f.read()
        self.lexicon = self.MakeLexiconDict()

    def MakeLexiconDict(self):
        """
        Convert lexicon file to a dictionary
        """
        lex_dict = {}
        for line in self.lexicon_full_filepath.split('\n'):
            (word, measure) = line.strip().split('\t')[0:2]
            lex_dict[word] = float(measure)
        return lex_dict

    def SentencePolarityScores(self, text):
        """
        Return a float for sentiment strength based on the input text.
        Positive values are positive ScoreAddn, negative value are negative
        ScoreAddn.
        """
        sentitext = SentiText(text)
        #text, WordsAndEmoticons, is_cap_diff = self.preprocess(text)

        sentiments = []
        WordsAndEmoticons = sentitext.WordsAndEmoticons
        for item in WordsAndEmoticons:
            ScoreAddn = 0
            i = WordsAndEmoticons.index(item)
            if (i < len(WordsAndEmoticons) - 1 and item.lower() == "kind" and \
                WordsAndEmoticons[i+1].lower() == "of") or \
                item.lower() in BOOSTER_DICT:
                sentiments.append(ScoreAddn)
                continue

            sentiments = self.SentimentScoreAddn(ScoreAddn, sentitext, item, i, sentiments)

        sentiments = self.CheckEffectOfBut(WordsAndEmoticons, sentiments)
        
        ScoreAddn_dict = self.FinalScoreScoreAddn(sentiments, text)

        return ScoreAddn_dict

    def SentimentScoreAddn(self, ScoreAddn, sentitext, item, i, sentiments):
        is_cap_diff = sentitext.is_cap_diff
        WordsAndEmoticons = sentitext.WordsAndEmoticons
        item_lowercase = item.lower()
        if item_lowercase in self.lexicon:
            #get the sentiment ScoreAddn
            ScoreAddn = self.lexicon[item_lowercase]

            #check if sentiment laden word is in ALL CAPS (while others aren't)
            if item.isupper() and is_cap_diff:
                if ScoreAddn > 0:
                    ScoreAddn += C_INCR
                else:
                    ScoreAddn -= C_INCR

            for start_i in range(0,3):
                if i > start_i and WordsAndEmoticons[i-(start_i+1)].lower() not in self.lexicon:
                    # dampen the scalar modifier of preceding words and emoticons
                    # (excluding the ones that immediately preceed the item) based
                    # on their distance from the current item.
                    s = PrecedingWordEffect(WordsAndEmoticons[i-(start_i+1)], ScoreAddn, is_cap_diff)
                    if start_i == 1 and s != 0:
                        s = s*0.95
                    if start_i == 2 and s != 0:
                        s = s*0.9
                    ScoreAddn = ScoreAddn+s
                    ScoreAddn = self.CheckEffectOfNever(ScoreAddn, WordsAndEmoticons, start_i, i)
                    if start_i == 2:
                        ScoreAddn = self.CheckEffectOfIdioms(ScoreAddn, WordsAndEmoticons, i)

                        # future work: consider other sentiment-laden idioms
                        # other_idioms =
                        # {"back handed": -2, "blow smoke": -2, "blowing smoke": -2,
                        #  "upper hand": 1, "break a leg": 2,
                        #  "cooking with gas": 2, "in the black": 2, "in the red": -2,
                        #  "on the ball": 2,"under the weather": -2}

            ScoreAddn = self.CheckEffectOfLeast(ScoreAddn, WordsAndEmoticons, i)

        sentiments.append(ScoreAddn)
        return sentiments

    def CheckEffectOfLeast(self, ScoreAddn, WordsAndEmoticons, i):
        # check for negation case using "least"
        if i > 1 and WordsAndEmoticons[i-1].lower() not in self.lexicon \
           and WordsAndEmoticons[i-1].lower() == "least":
            if WordsAndEmoticons[i-2].lower() != "at" and WordsAndEmoticons[i-2].lower() != "very":
                ScoreAddn = ScoreAddn*N_SCALAR
        elif i > 0 and WordsAndEmoticons[i-1].lower() not in self.lexicon \
             and WordsAndEmoticons[i-1].lower() == "least":
            ScoreAddn = ScoreAddn*N_SCALAR
        return ScoreAddn

    def CheckEffectOfBut(self, WordsAndEmoticons, sentiments):
        # check for modification in sentiment due to contrastive conjunction 'but'
        if 'but' in WordsAndEmoticons or 'BUT' in WordsAndEmoticons:
            try:
                bi = WordsAndEmoticons.index('but')
            except ValueError:
                bi = WordsAndEmoticons.index('BUT')
            for sentiment in sentiments:
                si = sentiments.index(sentiment)
                if si < bi:
                    sentiments.pop(si)
                    sentiments.insert(si, sentiment*0.5)
                elif si > bi:
                    sentiments.pop(si)
                    sentiments.insert(si, sentiment*1.5)
        return sentiments

    def CheckEffectOfIdioms(self, ScoreAddn, WordsAndEmoticons, i):
        onezero = "{0} {1}".format(WordsAndEmoticons[i-1], WordsAndEmoticons[i])

        twoonezero = "{0} {1} {2}".format(WordsAndEmoticons[i-2],
                                       WordsAndEmoticons[i-1], WordsAndEmoticons[i])

        twoone = "{0} {1}".format(WordsAndEmoticons[i-2], WordsAndEmoticons[i-1])

        threetwoone = "{0} {1} {2}".format(WordsAndEmoticons[i-3],
                                        WordsAndEmoticons[i-2], WordsAndEmoticons[i-1])

        threetwo = "{0} {1}".format(WordsAndEmoticons[i-3], WordsAndEmoticons[i-2])

        sequences = [onezero, twoonezero, twoone, threetwoone, threetwo]

        for seq in sequences:
            if seq in SPECIAL_CASE_IDIOMS:
                ScoreAddn = SPECIAL_CASE_IDIOMS[seq]
                break

        if len(WordsAndEmoticons)-1 > i:
            zeroone = "{0} {1}".format(WordsAndEmoticons[i], WordsAndEmoticons[i+1])
            if zeroone in SPECIAL_CASE_IDIOMS:
                ScoreAddn = SPECIAL_CASE_IDIOMS[zeroone]
        if len(WordsAndEmoticons)-1 > i+1:
            zeroonetwo = "{0} {1} {2}".format(WordsAndEmoticons[i], WordsAndEmoticons[i+1], WordsAndEmoticons[i+2])
            if zeroonetwo in SPECIAL_CASE_IDIOMS:
                ScoreAddn = SPECIAL_CASE_IDIOMS[zeroonetwo]

        # check for booster/dampener bi-grams such as 'sort of' or 'kind of'
        if threetwo in BOOSTER_DICT or twoone in BOOSTER_DICT:
            ScoreAddn = ScoreAddn+B_DECR
        return ScoreAddn

    def CheckEffectOfNever(self, ScoreAddn, WordsAndEmoticons, start_i, i):
        if start_i == 0:
            if negated([WordsAndEmoticons[i-1]]):
                    ScoreAddn = ScoreAddn*N_SCALAR
        if start_i == 1:
            if WordsAndEmoticons[i-2] == "never" and\
               (WordsAndEmoticons[i-1] == "so" or
                WordsAndEmoticons[i-1] == "this"):
                ScoreAddn = ScoreAddn*1.5
            elif negated([WordsAndEmoticons[i-(start_i+1)]]):
                ScoreAddn = ScoreAddn*N_SCALAR
        if start_i == 2:
            if WordsAndEmoticons[i-3] == "never" and \
               (WordsAndEmoticons[i-2] == "so" or WordsAndEmoticons[i-2] == "this") or \
               (WordsAndEmoticons[i-1] == "so" or WordsAndEmoticons[i-1] == "this"):
                ScoreAddn = ScoreAddn*1.25
            elif negated([WordsAndEmoticons[i-(start_i+1)]]):
                ScoreAddn = ScoreAddn*N_SCALAR
        return ScoreAddn

    def PunctuationEmphasis(self, sum_s, text):
        # add emphasis from exclamation points and question marks
        ep_amplifier = self.AmplifyEmphasisExclam(text)
        qm_amplifier = self.AmplifyEmphasisQM(text)
        punct_emph_amplifier = ep_amplifier+qm_amplifier
        return punct_emph_amplifier

    def AmplifyEmphasisExclam(self, text):
        # check for added emphasis resulting from exclamation points (up to 4 of them)
        ep_count = text.count("!")
        if ep_count > 4:
            ep_count = 4
        # (empirically derived mean sentiment intensity rating increase for
        # exclamation points)
        ep_amplifier = ep_count*0.292
        return ep_amplifier

    def AmplifyEmphasisQM(self, text):
        # check for added emphasis resulting from question marks (2 or 3+)
        qm_count = text.count("?")
        qm_amplifier = 0
        if qm_count > 1:
            if qm_count <= 3:
                # (empirically derived mean sentiment intensity rating increase for
                # question marks)
                qm_amplifier = qm_count*0.18
            else:
                qm_amplifier = 0.96
        return qm_amplifier

    def WeighSepPandNSenti(self, sentiments):
        # want separate positive versus negative sentiment scores
        pos_sum = 0.0
        neg_sum = 0.0
        neu_count = 0
        for sentiment_score in sentiments:
            if sentiment_score > 0:
                pos_sum += (float(sentiment_score) +1) # compensates for neutral words that are counted as 1
            if sentiment_score < 0:
                neg_sum += (float(sentiment_score) -1) # when used with math.fabs(), compensates for neutrals
            if sentiment_score == 0:
                neu_count += 1
        return pos_sum, neg_sum, neu_count

    def FinalScoreScoreAddn(self, sentiments, text):
        if sentiments:
            sum_s = float(sum(sentiments))
            # compute and add emphasis from punctuation in text
            punct_emph_amplifier = self.PunctuationEmphasis(sum_s, text)
            if sum_s > 0:
                sum_s += punct_emph_amplifier
            elif  sum_s < 0:
                sum_s -= punct_emph_amplifier

            compound = normalize(sum_s)
            # discriminate between positive, negative and neutral sentiment scores
            pos_sum, neg_sum, neu_count = self.WeighSepPandNSenti(sentiments)

            if pos_sum > math.fabs(neg_sum):
                pos_sum += (punct_emph_amplifier)
            elif pos_sum < math.fabs(neg_sum):
                neg_sum -= (punct_emph_amplifier)

            total = pos_sum + math.fabs(neg_sum) + neu_count
            pos = math.fabs(pos_sum / total)
            neg = math.fabs(neg_sum / total)
            neu = math.fabs(neu_count / total)

        else:
            compound = 0.0
            pos = 0.0
            neg = 0.0
            neu = 0.0

        sentiment_dict = \
            {"neg" : round(neg, 3),
             "neu" : round(neu, 3),
             "pos" : round(pos, 3),
             "compound" : round(compound, 4)}

        return sentiment_dict







if __name__ == '__main__':
    # --- examples -------
    #FetchTweets("realDonaldTrump")
    print("\n\n\n\n\n\nOutput from here:\n\n\n\n\n")
    fwri=open('tweets.txt','w')

    analyzer = SentimentIntensityAnalyzer()
   
    def csv_dict_reader(file_obj):
        """
        Read a CSV file using csv.DictReader
        """
        
        reader = csv.DictReader(file_obj, delimiter=',')
        Co=0.0
        Cp=0.0
        Cq=0.0
        CT=0.0
        for line in reader:
            CT+=1
            vs = analyzer.SentencePolarityScores(line['text'])
            #print("{:-<65} {}".format(sentence, str(vs)))
            
            for k in vs:
                    if k=='compound':
                        if vs[k]>0:
                            print "Positive sentiment in the Tweet: ",line['text'],"\n\n"
                            print vs[k]
                            Co+=1
                            print Co
                        elif vs[k]==0:
                            print "Neutral sentiment in the Tweet: ",line['text'],"\n\n"
                            print vs[k]
                            Cp+=1
                            print Cp
                        else:
                            print "Negative sentiment in the Tweet: ",line['text'],"\n\n"
                            print vs[k]
                            Cq+=1
                            print Cq



        print "Total: ",float(Co/CT)*100,"% Positive Tweets."
        print "Total: ",float(Cp/CT)*100,"% Neutral Tweets."
        print "Total: ",float(Cq/CT)*100,"% Negative Tweets."
        print "Total: ",CT," Tweets."
        


    with open("realDonaldTrump_tweets.csv") as f_obj:
        csv_dict_reader(f_obj)



    
    



    