import re,math,sys
import newspaper
import pickle
import random
import pyrebase
from goose3 import Goose
from collections import Counter
import os, datetime
import pandas as pd
import hashlib
from . import indian_scraper_plug
from langdetect import detect

INDIAN_LANGUAGES = ['hi','guj','ma']

max_article_addition = 15
ideal = 20.0
n_bullets = 4
stopwords = set()

max_sentences = 3
max_articles = 5
max_local_summaries = 10

SUMMARIES = dict()

email=" "
password=" "

config={
        "apiKey": "",
        "authDomain": "briefly-c7ef1.firebaseapp.com",
        "databaseURL": "https://briefly-c7ef1.firebaseio.com",
        "storageBucket": "briefly-c7ef1.appspot.com"
}

firebase = pyrebase.initialize_app(config)
auth=firebase.auth()
user=auth.sign_in_with_email_and_password(email,password)

def refresh(user):
    user=auth.refresh(user['refreshToken'])

db=firebase.database()

def summary(url):
    article=newspaper.Article(url)
    article.download()
    article.parse()
    title = article.title
    date = ""
    try:
	    image=article.top_image
    except:
        image = "http://www.sahalnews.com/wp-content/uploads/2014/12/news-update-.jpg"
    article.nlp()
    # detect here
    iso_lang = detect(title)
    if iso_lang in INDIAN_LANGUAGES:
	    summary = indian_scraper_plug.summary(article.text,article.title,iso_lang)
    else:
        summary = article.summary
    return (title,"",image,summary)

'''
def load_stopwords(language):
    """
    Loads language-specific stopwords for keyword selection
    """
    global stopwords

    # stopwords for nlp in English are not the regular stopwords
    # to pass the tests
    # can be changed with the tests
    if language == 'en':
        stopwordsFile ="./stopwords-nlp-en.txt"
    else:
        stopwordsFile = path.join(settings.STOPWORDS_DIR,\
                                  'stopwords-{}.txt'.format(language))
    with open(stopwordsFile, 'r', encoding='utf-8') as f:
        stopwords.update(set([w.strip() for w in f.readlines()]))


def summarize(url='', title='', text='', max_sents=5):
    if not text or not title or max_sents <= 0:
        return []

    summaries = []
    sentences = split_sentences(text)
    keys = keywords(text)
    titleWords = split_words(title)

    # Score sentences, and use the top 5 or max_sents sentences
    ranks = score(sentences, titleWords, keys).most_common(max_sents)
    for rank in ranks:
        summaries.append(rank[0])
    summaries.sort(key=lambda summary: summary[0])
    return [summary[1] for summary in summaries]


def score(sentences, titleWords, keywords):
    """Score sentences based on different features
    """
    senSize = len(sentences)
    ranks = Counter()
    for i, s in enumerate(sentences):
        sentence = split_words(s)
        titleFeature = title_score(titleWords, sentence)
        sentenceLength = length_score(len(sentence))
        sentencePosition = sentence_position(i + 1, senSize)
        sbsFeature = sbs(sentence, keywords)
        dbsFeature = dbs(sentence, keywords)
        frequency = (sbsFeature + dbsFeature) / 2.0 * 10.0
        # Weighted average of scores from four categories
        totalScore = (titleFeature*1.5 + frequency*2.0 +
                      sentenceLength*1.0 + sentencePosition*1.0)/4.0
        ranks[(i, s)] = totalScore
    return ranks

def sbs(words, keywords):
    score = 0.0
    if (len(words) == 0):
        return 0
    for word in words:
        if word in keywords:
            score += keywords[word]
    return (1.0 / math.fabs(len(words)) * score) / 10.0

def dbs(words, keywords):
    if (len(words) == 0):
        return 0
    summ = 0
    first = []
    second = []

    for i, word in enumerate(words):
        if word in keywords:
            score = keywords[word]
            if first == []:
                first = [i, score]
            else:
                second = first
                first = [i, score]
                dif = first[0] - second[0]
                summ += (first[1] * second[1]) / (dif ** 2)
    # Number of intersections
    k = len(set(keywords.keys()).intersection(set(words))) + 1
    return (1 / (k * (k + 1.0)) * summ)

def split_words(text):
    """Split a string into array of words
    """
    try:
        text = re.sub(r'[^\w ]', '', text)  # strip special chars
        return [x.strip('.').lower() for x in text.split()]
    except TypeError:
        return None

def keywords(text):
    """Get the top 10 keywords and their frequency scores ignores blacklisted
    words in stopwords, counts the number of occurrences of each word, and
    sorts them in reverse natural order (so descending) by number of
    occurrences.
    """
    NUM_KEYWORDS = 10
    text = split_words(text)
    # of words before removing blacklist words
    if text:
        num_words = len(text)
        text = [x for x in text if x not in stopwords]
        freq = {}
        for word in text:
            if word in freq:
                freq[word] += 1
            else:
                freq[word] = 1

        min_size = min(NUM_KEYWORDS, len(freq))
        keywords = sorted(freq.items(),
                          key=lambda x: (x[1], x[0]),
                          reverse=True)
        keywords = keywords[:min_size]
        keywords = dict((x, y) for x, y in keywords)

        for k in keywords:
            articleScore = keywords[k] * 1.0 / max(num_words, 1)
            keywords[k] = articleScore * 1.5 + 1
        return dict(keywords)
    else:
        return dict()

def split_sentences(text):
    """Split a large string into sentences
    """
    import nltk.data
    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

    sentences = tokenizer.tokenize(text)
    sentences = [x.replace('\n', '') for x in sentences if len(x) > 10]
    return sentences

def length_score(sentence_len):
    return 1 - math.fabs(ideal - sentence_len) / ideal

def title_score(title, sentence):
    if title:
        title = [x for x in title if x not in stopwords]
        count = 0.0
        for word in sentence:
            if (word not in stopwords and word in title):
                count += 1.0
        return count / max(len(title), 1)
    else:
        return 0

def sentence_position(i, size):
    """Different sentence positions indicate different
    probability of being an important sentence.
    """
    normalized = i * 1.0 / size
    if (normalized > 1.0):
        return 0
    elif (normalized > 0.9):
        return 0.15
    elif (normalized > 0.8):
        return 0.04
    elif (normalized > 0.7):
        return 0.04
    elif (normalized > 0.6):
        return 0.06
    elif (normalized > 0.5):
        return 0.04
    elif (normalized > 0.4):
        return 0.05
    elif (normalized > 0.3):
        return 0.08
    elif (normalized > 0.2):
        return 0.14
    elif (normalized > 0.1):
        return 0.23
    elif (normalized > 0):
        return 0.17
    else:
        return 0
'''
def pinChannel(username,url):
    print(url)
    try:
        value=hashlib.sha224(url.encode('utf-8')).hexdigest()
        sender_id = ''
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
        data={'sub':[value]}
        users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
        if(len(users.each())):#check if entry exists
            data=users.val()[sender_id]
            if "pin" in data.keys():
                lis=data["pin"]
            else:
                lis=[]
            lis.append(value)
            data['pin']=lis
            print(data)
            db.child("users").child(sender_id).update(data,user['idToken'])
        else:
            print(data)
            db.child("users").child(sender_id).set(data,user['idToken'])
    except:
        refresh(user)
        subChannel(sender_id,value)

#inactive
def unpinChannel(username,url):
    try:
        sender_id=''
        value=hashlib.sha224(url.encode('utf-8')).hexdigest()
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
        data={}
        print(sender_id)
        users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    data=users.val()[sender_id]
            #print(data)
            if "pin" in data.keys():
                #print("here")
                lis=data["pin"]
                #print(lis)
                #print(value)
                if value in lis:
                    lis.remove(value)
                    data["pin"]=lis
                    db.child("users").child(sender_id).update(data,user['idToken'])
    except:
        refresh(user)
        unsubChannel(sender_id,value)


def subChannel(username,value):
    try:
        sender_id = ''
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
        data={'sub':[value]}
        users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
        if(len(users.each())):#check if entry exists
            data=users.val()[sender_id]
            if "sub" in data.keys():
                lis=users.val()[sender_id]['sub']
            else:
                lis=[]
            lis.append(value)
            data['sub']=lis
            db.child("users").child(sender_id).update(data,user['idToken'])
        else:
            db.child("users").child(sender_id).set(data,user['idToken'])
    except:
        refresh(user)
        subChannel(sender_id,value)
#inactive
def unsubChannel(username,value):
    try:
        sender_id=''
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
        data={}
        users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    data=users.val()[sender_id]
            if "sub" in data.keys():
                lis=data["sub"]
                if value in lis:
                    lis.remove(value)
                    data['sub']=lis
                    db.child("users").child(sender_id).update(data,user['idToken'])
    except:
        refresh()
        unsubChannel(sender_id,value)


# source , last_updated . hourly limit ( day leaks inactive)
def subscribe_model(source):

    file_name = "sum.pickle"
    load_stopwords('en')

    articles_per_source = dict()

    if os.path.isfile(file_name):
        article_file = open(file_name,"rb")
        articles_per_source = pickle.load(article_file)
        article_file.close()

    article_file = open(file_name,"wb")

    # Redundant as of TODO:
    MEMO_flag = False
    if source in articles_per_source.keys():
        MEMO_flag = True

    article_links = []

    now = datetime.datetime.now()
    current_time = now.hour*60 + now.minute

    '''     TODO: CACHING
    if source in articles_per_source.keys():
        n_articles = len(articles_per_source[source])
        if (current_time-articles_per_source[source][0]) < 60 :
            pickle.dump(articles_per_source,article_file)
            article_file.close()
            n_articles = len(articles_per_source[source])
            print(n_articles," in this hash")
            random.shuffle(articles_per_source[source])
            return articles_per_source[source][-1*min(n_articles-1,5):]
    '''

    links = newspaper.build(source,memoize_articles=MEMO_flag)  #turn this True while in serev

    for article in links.articles[:min(max_article_addition,len(links.articles))]:
        article_links.append([summary(article.url)])
    #articles_per_source[source] = [current_time]
    if source not in articles_per_source.keys():
        articles_per_source[source] = []
    articles_per_source[source] += article_links
    n_articles = len(articles_per_source[source])
    articles_per_source[source] = articles_per_source[source][:min(n_articles,50)]

    pickle.dump(articles_per_source,article_file)
    article_file.close()

    random.shuffle(articles_per_source[source])

    return articles_per_source[source][-1*min(n_articles-1,3):]

def generate_summaries(url,sentences):
    if "http" not in url:
        url="http://"+url
    links_to_articles = subscribe_model(url)
    available = len(links_to_articles)
    results = []
    for article_url in links_to_articles[:min(available,max_articles)]:
        summar=summary(article_url)
        headline = summar[0]
        publish_date = summar[1]
        top_image_url  = summar[2]
        summaries = summar[3]
        concate_news = ""
        for bullets in summaries:
            concate_news +=  bullets
            concate_news += "\n"
        sum_keys = sorted(SUMMARIES.keys())
        if len(sum_keys) > max_local_summaries :
            del SUMMARIES[0]
        if not len(sum_keys):
            hash_index = 0
        else:
            hash_index = sum_keys[-1]
        results.append([headline,top_image_url,publish_date,concate_news])
        SUMMARIES[hash_index+1] = concate_news
    return results

def generate_feed(username):
        data={}
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
            users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
            if(len(users.each())):
                lis=users.val()[sender_id]
                if 'sub' in lis.keys():
                    lis=lis['sub']
                else:
                    return {}
                subl={}
                try:
                    articles_per_source = db.child("sources").get(user['idToken']).val()
                    Uarticle = db.child("article").get(user['idToken']).val()
                except:
                    refresh(user)
                    generate_feed(username)
                result={}
                for i in lis:
                    li=[]
                    if i!=None:
                        if i in articles_per_source.keys():
                            lent=len(articles_per_source[i])
                            hashes=articles_per_source[i][-min(lent,4):]
                        for hashe in hashes:
                            try:
                                li.append(Uarticle[hashe])
                            except:
                                print(hashe)
                                pass
                    result[i]=li
                return result
            else:
                return {}
        else:
            return {}

def browser(source):
    try:
        articles_per_source = db.child("sources").get(user['idToken']).val()
        Uarticle = db.child("article").get(user['idToken']).val()
    except:
        refresh(user)
        generate_feed(username)
    li=[]
    if source!=None:
        if source in articles_per_source.keys():
            lent=len(articles_per_source[source])
            hashes=articles_per_source[source][-min(lent,9):]
            for hashe in hashes:
                try:
                    li.append(Uarticle[hashe])
                except:
                    print(hashe)
    return li


def show_saved(username):
        data={}
        users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
        if(len(users.each())):#check if entry exists
       	    lis=users.val()
            sender_id=str(lis[username])
            users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
            if(len(users.each())):
                lis=users.val()[sender_id]
                if 'pin' in lis:
                    lis=lis['pin']
                    print(lis)
                else:
                    return []
                try:
                    articles_per_source = db.child("sources").get(user['idToken']).val()
                    Uarticle = db.child("article").get(user['idToken']).val()
                except:
                    refresh(user)
                    generate_feed(username)
                li=[]
                print("++"*20)
                print(lis)
                for hashe in lis:
                    print("====",hashe)
                    if hashe!=None:
                            li.append(Uarticle[hashe])
                    result=li
                    print(li)
                return result
            else:
                return []
        else:
            return []

def extra(username):
    #print("hello")
    users=db.child("id").order_by_key().equal_to(username).get(user['idToken'])
    if(len(users.each())):#check if entry exists
        lis=users.val()
        sender_id=str(lis[username])
    users=db.child("users").order_by_key().equal_to(sender_id).get(user['idToken'])
    if(len(users.each())):#check if entry exists
        lis=users.val()[sender_id]['sub']
        #print(lis)
        return lis

def addUser(sender_id,value):
    try:
        data = {sender_id:value}
        db.child("id").child(value).set(sender_id,user['idToken'])
    except:
        refresh(user)
        addUser(sender_id,value)

if __name__ == "__main__":
    print(subscribe_model(input()))
