# encoding=utf-8
import re
import datetime
import socket
import logging
import pytz
import uuid
import psycopg2
import urlparse
from bs4 import BeautifulSoup as bs
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
from scrapy.conf import settings
from scrapy.exceptions import CloseSpider
from sina_weibo.utils import getCookies
from sina_weibo.items import InformationItem, FollowsItem
from sina_weibo.handlers import PGHandler
from sina_weibo.postgres import db
from parse_user_detil import get_user_detail, get_uid_by_uname,parse_follows

class MediaSpider(Spider):
    '''爬取媒体汇'''
    name = "weibo_media"
    domain = "http://weibo.cn"
    target = 'weibo'
    hostname = socket.gethostname()
    jobtime = datetime.datetime.now()
    time_record_static = datetime.datetime.now()
    time_record_dynamic = datetime.datetime.now()
    cell_id = None
    job_id = None
    user_id = None
    error_found = False
    ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0'
    cookies = {}
    handle_httpstatus_list = [302]

    #媒体汇url，以地区为分类
    start_urls = [
        'http://weibo.cn/v2star/?cat=2&sorttype=area'
    ]
    time_record_static = datetime.datetime.now()
    time_record_dynamic = datetime.datetime.now()

    account = ''
    password = ''

    def __init__(self, *args, **kwargs):
        super(Spider, self).__init__(self.name, *args, **kwargs)
        dispatcher.connect(self.spider_opened, signals.spider_opened)
        dispatcher.connect(self.spider_closed, signals.spider_closed)

        meta_conn_string = ' '.join(["host=", settings['META_DB']['host'], "dbname=", settings['META_DB']['dbname'], \
                                     "user=", settings['META_DB']['user'], "password=", settings['META_DB']['password'],
                                     "port=", settings['META_DB']['port']])
        self.meta_conn = psycopg2.connect(meta_conn_string)
        self.meta_conn.autocommit = True
        self.meta_session = db(self.meta_conn)
        data_conn_string = ' '.join(["host=", settings['DATA_DB']['host'], "dbname=", settings['DATA_DB']['dbname'], \
                                     "user=", settings['DATA_DB']['user'], "password=", settings['DATA_DB']['password'],
                                     "port=", settings['DATA_DB']['port']])
        self.data_conn = psycopg2.connect(data_conn_string)
        self.data_session = db(self.data_conn)
        resource = self.meta_session.getOne(
            "select account, password, ua_mobile, download_delay from crawler_resource where host = '%s' and target = '%s'" % (
            self.hostname, self.target))


        self.account = resource[0]
        self.password = resource[1]
        self.cell_name = self.name
        self.cookies = getCookies(self.account, self.password)
        self.cell_name = self.name

    def spider_opened(self, spider):
        # get stats
        handler = PGHandler(self.name)
        handler.setLevel(settings.get('LOG_LEVEL'))
        logger = logging.getLogger()
        logger.addHandler(handler)
        self.dstats = spider.crawler.stats.get_stats()
        self.collect_stats(status='running')

    def spider_closed(self, spider):
        self.data_conn.close()
        self.collect_stats(status='finished')


    def parse(self, response):
        ''' 解析媒体汇url, 以便爬取用户和用户关注的人 '''
        #获取每个地区对应的url
        urls = response.xpath("/html/body/div[5]/a/@href").extract()
        for url in urls:
            # 请求每个url对应的页面
            self.time_record_dynamic = datetime.datetime.now()
            if (self.time_record_dynamic - self.time_record_static).seconds > 43200:
                self.cookies = self.getCookies()
                self.time_record_static = datetime.datetime.now()
                self.time_record_dynamic = datetime.datetime.now()

            yield Request(url = urlparse.urljoin(self.start_urls[0], url), callback=self.parse_each_area)

    def parse_each_area(self, response):
        ''' 解析每个地区对应的页面 '''
        soup = bs(response.body, 'html.parser')
        # 获取每个媒体汇的用户id
        tables = soup.find_all('div', class_='c')[2].find_all('table')
        for table in tables:
            uid = table.tr.find_all("td", valign="top")[1].find_all('a')[0]['href'].split('/')[2]
            # 爬个人信息
            yield get_user_detail(uid=uid, cookies=self.cookies)

        #下页
        np = response.css('#pagelist > form:nth-child(1) > div:nth-child(1) > a:nth-child(1)::text')
        if np and u'下页' == np[0].extract():
            url =  response.urljoin(response.css('#pagelist > form:nth-child(1) > div:nth-child(1) > a:nth-child(1)::attr(href)')[0].extract())
            yield Request(url=url,callback=self.parse_each_area)

    def collect_stats(self, status):

        response_count_3xx = None
        response_count_4xx = None
        response_count_5xx = None

        job_stats = {}
        job_stats['host'] = self.hostname
        #         job_stats['user_id'] = self.user_id
        #         job_stats['cell_id'] = self.cell_id
        job_stats['cell_name'] = self.name
        job_stats['item_count'] = self.dstats.get('item_scraped_count', 0)
        if self.dstats.get('start_time', datetime.datetime.now()):  # when job starts in waiting queue
            job_stats['run_time'] = pytz.utc.localize(self.dstats.get('start_time'))  # when job starts running
        if self.dstats.get('finish_time'):
            job_stats['end_time'] = pytz.utc.localize(self.dstats.get('finish_time'))
        if status == 'running':
            job_stats['status'] = 'running'
        elif status == 'finished':
            job_stats['status'] = self.dstats.get('finish_reason', None)
            if self.error_found or job_stats['item_count'] == 0:
                job_stats['status'] = 'failed'

        job_stats['image_count'] = self.dstats.get('image_count')
        job_stats['image_downloaded'] = self.dstats.get('image_downloaded')
        job_stats['request_count'] = self.dstats.get('downloader/request_count')
        job_stats['response_bytes'] = self.dstats.get('downloader/response_bytes')
        job_stats['response_count'] = self.dstats.get('downloader/response_count')
        job_stats['response_count_200'] = self.dstats.get('downloader/response_status_count/200')
        for key, value in self.dstats.iteritems():
            if 'downloader/response_status_count/3' in key:
                job_stats['response_count_3xx'] = int(response_count_3xx or 0) + value
            elif 'downloader/response_status_count/4' in key:
                job_stats['response_count_4xx'] = int(response_count_4xx or 0) + value
            elif 'downloader/response_status_count/5' in key:
                job_stats['response_count_5xx'] = int(response_count_5xx or 0) + value
        job_stats['load_time'] = datetime.datetime.now()

        try:
            with self.meta_conn:
                if not self.job_id:
                    self.job_id = uuid.uuid1().hex
                    job_stats['job_id'] = self.job_id
                    job_stats['start_time'] = self.jobtime
                    self.meta_session.Insert(settings['STATS_TABLE'], job_stats)
                else:
                    wheredict = {}
                    wheredict['job_id'] = self.job_id
                    self.meta_session.Update(settings['STATS_TABLE'], job_stats, wheredict)

        except psycopg2.Error, e:
            logging.warn('Failed to refresh job stats: %s' % e)


