#!/usr/bin/env python

#import tornado.options
#from tornado.options import define, options

import urllib2
import urllib
import json
import datetime
from optparse import OptionParser

#from sqlalchemy import create_engine
#from sqlalchemy.orm import sessionmaker
from sqlalchemy import Date
from sqlalchemy.sql.expression import cast

from pprint import pprint
from unsourced.models import Article,ArticleURL,Tag
from unsourced.util import parse_config_file
from unsourced import db

from tornado.options import define, options


def get_arts_OLD(num_days=1):
    date_from = datetime.date.today() - datetime.timedelta(days = num_days-1)
    date_to = datetime.date.today()

    phrases = ("scientists have", "scientists say",  "paper published", "research suggests", "latest research", "researchers", "the study")


    search = '(%s) %s..%s' % (
            " OR ".join(['"'+s+'"' for s in phrases]),
            date_from.strftime('%Y%m%d'),
            date_to.strftime('%Y%m%d'))

    print search
    params = {'search': search,
            'limit': 1000,
            'output': 'js',
            }
    url = "http://journalisted.com/api/findArticles?" + urllib.urlencode(params)
    print url
    foo = json.load(urllib2.urlopen(url), encoding='utf-8')
    articles = foo['results']
    return articles




def get_arts(num_days=1):
    articles = []
    date_from = datetime.date.today() - datetime.timedelta(days = num_days-1)
    date_to = datetime.date.today()

    phrases = ("scientists have", "scientists say",  "paper published", "research suggests", "latest research", "researchers", "the study")

    for phrase in phrases:
        search = '"%s" %s..%s' % (phrase,
            date_from.strftime('%Y%m%d'),
            date_to.strftime('%Y%m%d'))

        print search
        params = {'search': search,
            'limit': 1000,
            'output': 'js',
            }
        url = "http://journalisted.com/api/findArticles?" + urllib.urlencode(params)
        print url
        foo = json.load(urllib2.urlopen(url), encoding='utf-8')
        print "got %d" % (len(foo['results']),)
        articles += foo['results']
        # TODO: dedupe the list!
    return articles



def main():
    parser = OptionParser()
    parser.add_option("-n", type="int", dest="num_days", default=1)

    (opts, args) = parser.parse_args()

    session = db.create_session()


    found_arts = get_arts(opts.num_days)

    n_had = 0
    n_dupes = 0
    n_added = 0

    print "found %d articles" % (len(found_arts),)
    for src in found_arts:
        # TODO: JL should serve up urls...
        urls = [src['permalink']]

        got = session.query(ArticleURL).filter(ArticleURL.url.in_(urls)).all()
        if len(got)>0:
            n_had += 1
#            print "got %s" % (src['permalink'],)
            # TODO: add any newly-discovered urls
            continue

        # kludge to reduce duplicates (should be handled by journalisted)
        title = src['title']
        day = datetime.datetime.strptime(src['pubdate'],'%Y-%m-%d %H:%M:%S').date()
        got = session.query(Article).\
            filter(Article.headline == title).\
            filter(cast(Article.pubdate, Date) == day).\
            filter(cast(Article.pubdate, Date) == day).\
            all()
        if len(got)>0:
            n_dupes += 1
            print "skip duplicate '%s' %s" % (title,src['permalink'])
            continue

        print "load %s" % (src['permalink'],)
        url_objs = [ArticleURL(url=u) for u in urls]
        art = Article(src['title'],src['permalink'],src['pubdate'],url_objs)
        for u in url_objs:
            session.add(u)
        session.add(art)
        n_added += 1

    print "%d new, %d already had, %d dupes skipped" % (n_added,n_had,n_dupes)

    print "committing"
    session.commit()


if __name__ == "__main__":
    main()

