#!/usr/bin/env python

from datetime import datetime
import urllib2
from optparse import OptionParser
import logging
import csv
import socket
import dateutil.parser
import os
import sys
import re

from urllib2helpers import CacheHandler

import site
here = os.path.dirname(os.path.abspath(__file__))
site.addsitedir(os.path.join(here,"../.."))

import metareadability



def tests_from_csv(filename):
    # url, headline, journo/byline, pubdate
    reader = csv.reader(open(filename, 'rb'))
    rows = [ row for row in reader ]
    n = 1
    try:
        for row in rows:
            row[1] = row[1].decode('utf-8')
            row[2] = row[2].decode('utf-8')
            if row[3].strip() == '':
                row[3] = None
            else:
                row[3] = dateutil.parser.parse(row[3])
            n+=1
    except:
        logging.error("error in %s:%d"%(filename,n))
        raise
    logging.info("loaded %d tests from '%s'",len(rows),filename)
    return rows



def BYLINE_ONLY_compare_result(got, expected, url):
    errs = []
    # byline
    if got[1] != expected[1]:
        errs.append(" byline: got '%s', expected '%s'" % (got[1],expected[1]))
    if errs:
        logging.warning("failed %s" % (url,))
        [ logging.warning(" %s"%(err,)) for err in errs ]
        return False
    else:
        logging.debug("matched %s" % (url,))
        return True


def norm(txt):
    if txt is None:
        return u''
    return metareadability.util.normalise_text(txt)


def compare_result(got, expected, url):

    errs = []

    #headline
    if norm(got[0]) != norm(expected[0]):
        errs.append("title: got '%s', expected '%s'" % (got[0],expected[0]))
    #pubdate
    a = None if got[2] is None else got[2].date()
    b = None if expected[2] is None else expected[2].date()
    if a != b:
        errs.append(" date: got '%s', expected '%s'" % (got[2],expected[2]))

    # byline
    if norm(got[1]) != norm(expected[1]):
        errs.append(" byline: got '%s', expected '%s'" % (got[1],expected[1]))
 

    if errs:
        logging.warning("failed %s" % (url,))
        [ logging.warning(" %s"%(err,)) for err in errs ]
        return False
    else:
        logging.debug("matched %s" % (url,))
        return True


def main():
    parser = OptionParser(usage="%prog: [options] <csvfiles>")
    parser.add_option('-v', '--verbose', action='store_true')
    parser.add_option('-d', '--debug', action='store_true')
    parser.add_option('-u', '--url', help="only test urls containing URL (can appear anywhere in url)")
    parser.add_option('-c', '--cache', action='store_true', help="cache downloaded files in .cache dir")
    (options, args) = parser.parse_args()
 
    log_level = logging.WARNING
    if options.debug:
        log_level = logging.DEBUG
    elif options.verbose:
        log_level = logging.INFO

    logging.basicConfig(level=log_level, format='%(message)s')

    if options.cache:
        logging.info("Using .cache")
        opener = urllib2.build_opener(CacheHandler('.cache'))
    else:
        opener = urllib2.build_opener()
    urllib2.install_opener(opener)
    socket.setdefaulttimeout(5)


    if len(args)<1:
        parser.error("no input csv files")
    test_data = []
    for infile in args:
        test_data.extend(tests_from_csv(infile))

    good = 0
    bad = 0
    skipped = 0

    for row in test_data:
        url = row[0]
        expected = row[1:]

        if options.url and options.url not in url:
            continue

        kwargs = {}
        if url.endswith(".pdf"):
            logging.info("SKIP pdf: %s" %(url,))
            skipped += 1
            continue
#        print "fetching", url
        try:
            resp = urllib2.urlopen(url)
            html = resp.read()
            content_type = resp.info().getheader('Content-Type','')
            m = re.compile(r';\s*charset\s*=\s*([^;]*)', re.I).search(content_type)
            if m:
                kwargs['encoding'] = m.group(1)
        except urllib2.URLError, e:
            logging.info("URLError (%s) on %s" %(e,url))
            skipped += 1
            continue

        got = metareadability.extract(html,url,**kwargs)
#        print "got '%s' (expected %s) [ %s ]" %(got[1],expected[1],url)
#        continue
        logging.debug(got)

        if compare_result(got, expected, url):
            good += 1
        else:
            bad += 1

    summary = "finished: %d good, %d bad, %d skipped" % (good, bad, skipped)

    if bad > 0:
        logging.warn(summary)
        sys.exit(1)
    else:
        logging.info(summary)
        sys.exit(0)


if __name__ == '__main__':
    main()

