# encoding=utf-8
import re
import datetime
import socket
import logging
import pytz
import uuid
import psycopg2
import urlparse
from scrapy import FormRequest
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
from scrapy.conf import settings
from sina_weibo.utils import getCookies
from sina_weibo.items import InformationItem, FollowsItem
from sina_weibo.handlers import PGHandler
from sina_weibo.postgres import db
from parse_user_detil import get_user_detail, get_uid_by_uname

class CelebritySpider(Spider):
    '''爬取媒体汇'''
    name = "weibo_famous"
    domain = "http://weibo.cn"
    target = 'weibo'
    hostname = socket.gethostname()
    jobtime = datetime.datetime.now()
    time_record_static = datetime.datetime.now()
    time_record_dynamic = datetime.datetime.now()
    cell_id = None
    job_id = None
    user_id = None
    error_found = False
    ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0'
    cookies = {}
    handle_httpstatus_list = [302]

    start_urls = [
        'http://data.weibo.com/top/influence/famous', # 名人影响力榜, influence
        'http://data.weibo.com/top/hot/media',  # 名人人气榜
    ]

    account = ''
    password = ''

    def __init__(self, *args, **kwargs):
        super(Spider, self).__init__(self.name, *args, **kwargs)
        dispatcher.connect(self.spider_opened, signals.spider_opened)
        dispatcher.connect(self.spider_closed, signals.spider_closed)

        meta_conn_string = ' '.join(["host=", settings['META_DB']['host'], "dbname=", settings['META_DB']['dbname'], \
                                     "user=", settings['META_DB']['user'], "password=", settings['META_DB']['password'],
                                     "port=", settings['META_DB']['port']])
        self.meta_conn = psycopg2.connect(meta_conn_string)
        self.meta_conn.autocommit = True
        self.meta_session = db(self.meta_conn)
        data_conn_string = ' '.join(["host=", settings['DATA_DB']['host'], "dbname=", settings['DATA_DB']['dbname'], \
                                     "user=", settings['DATA_DB']['user'], "password=", settings['DATA_DB']['password'],
                                     "port=", settings['DATA_DB']['port']])
        self.data_conn = psycopg2.connect(data_conn_string)
        self.data_session = db(self.data_conn)
        resource = self.meta_session.getOne(
            "select account, password, ua_mobile, download_delay from crawler_resource where host = '%s' and target = '%s'" % (
            self.hostname, self.target))

        self.account = resource[0]
        self.password = resource[1]
        self.cell_name = self.name
        self.cookies = getCookies(self.account, self.password)
        self.cell_name = self.name

    def spider_opened(self, spider):
        # get stats
        handler = PGHandler(self.name)
        handler.setLevel(settings.get('LOG_LEVEL'))
        logger = logging.getLogger()
        logger.addHandler(handler)
        self.dstats = spider.crawler.stats.get_stats()
        self.collect_stats(status='running')

    def spider_closed(self, spider):
        self.data_conn.close()
        self.collect_stats(status='finished')

    def parse(self, response):
        '''解析各个排行榜的一级分类及url'''
        for url in response.css('.hottabs_list > a'):
            fir_category_url = urlparse.urljoin(response.url, url.xpath('./@href').extract()[0])
            # 获取一级分类
            fir_category = url.xpath('./text()').extract()[0]
            yield Request(url=fir_category_url,meta={'fir_category': fir_category}, callback=self.parse_sec_category)

    def parse_sec_category(self, response):
        '''通过各个一个分类寻找对应的二级分类，如名人人气榜含有：体育 娱乐 财经 传媒等二级分类'''
        fir_category = response.meta['fir_category']
        sec_category_div = None
        if response.xpath('//*[@id="pl_hot_secondtab"]/div[1]'):# 名人人气榜
            sec_category_div = response.xpath('//*[@id="pl_hot_secondtab"]/div[1]/div[@node-type="second_tab"]/a[@suda-uatrack="key=apex_top&value=all"]')
        elif response.xpath('//*[@id="pl_influence_changeTab"]/div[2]/div[1]/div/ul'): # 名人影响力榜
            sec_category_div = response.xpath('//*[@id="pl_influence_changeTab"]/div[2]/div[1]/div/ul/li/a[@suda-uatrack="key=apex_top&value=all"]')
        elif response.xpath('//*[@id="pl_content_groupTab"]/div[1]/div[2]/div/ul[1]'):# 娱乐微群榜
            sec_category_div = response.xpath('//*[@id="pl_content_groupTab"]/div[1]/div[2]/div/ul[1]/li/a[@suda-uatrack="key=apex_top&value=all"]')

        if not sec_category_div:
            return

        # 请求各个二级分类下的用户榜单
        for url in sec_category_div:
            url_sec_category = urlparse.urljoin(response.url, url.xpath('./@href')[0].extract())
            # 获取二级分类
            sec_category = url.xpath('./text()')[0].extract()
            yield Request(url=url_sec_category, meta={'fir_category': fir_category, 'sec_category':sec_category}, callback=self.parse_uid)

    def parse_uid(self, response):
        ''' 解析用户uid '''
        fir_category = response.meta['fir_category']
        sec_category = response.meta['sec_category']

        for span in response.css('.zw_name'):
            uid = span.css('a::attr(href)')[0].extract().split('/')[-1]
            if not uid.isdigit():
                uid = get_uid_by_uname(uid, self.cookies)
            user = InformationItem()
            user['user_id'] = uid
            user['fir_category'] = fir_category
            user['sec_category'] = sec_category
            yield  get_user_detail(user=user,cookies=self.cookies, uid=None)

        # 判断是否有下一页
        if not response.css('.W_pages_comment > a'):
            return

        # 下一页的榜单用户是通过请求下列url返回的json
        url_json = {'hot': 'http://data.weibo.com/top/ajax/hot',
                    'qun': 'http://data.weibo.com/top/ajax/qunclass',
                    'influence': 'http://data.weibo.com/top/ajax/influence'
        }

        for key in url_json.keys():
            if key in response.url:
                url = url_json[key]
        # 获取最大的页数
        max_page = response.css('.W_pages_comment > a')[-2].xpath('./text()')[0].extract()
        # 获取发送请求使用的数据
        query_string = response.css('.W_pages_comment > a')[-2].xpath('./@action-data')[0].extract()
        data = urlparse.parse_qs(query_string)

        # 请求下一页
        for page in range(2, int(max_page)+1):
            data['page'] = str(page)
            yield FormRequest(url=url, formdata=data, meta={'fir_category': fir_category, 'sec_category':sec_category},callback=self.parse_next_page)

    def parse_next_page(self, response):
        ''' 解析请求返回的json数据内包含的用户id'''
        fir_category = response.meta['fir_category']
        sec_category = response.meta['sec_category']

        for url in re.findall(r'<span class="zw_name">\w+<a.*?href="(.*?)" target="_blank" suda-uatrack="key=apex_top&value=all">.*?</span>',response.body.replace('\\', ''), re.S):
            uid = url.split('/')[-1]
            if not uid.isdigit():
                uid = get_uid_by_uname(uid, self.cookies)
            user = InformationItem()
            user['user_id'] = uid
            user['fir_category'] = fir_category
            user['sec_category'] = sec_category
            # 解析用户信息
            yield  get_user_detail(user=user,cookies=self.cookies, uid=None)


    def collect_stats(self, status):
        response_count_3xx = None
        response_count_4xx = None
        response_count_5xx = None

        job_stats = {}
        job_stats['host'] = self.hostname
        #         job_stats['user_id'] = self.user_id
        #         job_stats['cell_id'] = self.cell_id
        job_stats['cell_name'] = self.name
        job_stats['item_count'] = self.dstats.get('item_scraped_count', 0)
        if self.dstats.get('start_time', datetime.datetime.now()):  # when job starts in waiting queue
            job_stats['run_time'] = pytz.utc.localize(self.dstats.get('start_time'))  # when job starts running
        if self.dstats.get('finish_time'):
            job_stats['end_time'] = pytz.utc.localize(self.dstats.get('finish_time'))
        if status == 'running':
            job_stats['status'] = 'running'
        elif status == 'finished':
            job_stats['status'] = self.dstats.get('finish_reason', None)
            if self.error_found or job_stats['item_count'] == 0:
                job_stats['status'] = 'failed'

        job_stats['image_count'] = self.dstats.get('image_count')
        job_stats['image_downloaded'] = self.dstats.get('image_downloaded')
        job_stats['request_count'] = self.dstats.get('downloader/request_count')
        job_stats['response_bytes'] = self.dstats.get('downloader/response_bytes')
        job_stats['response_count'] = self.dstats.get('downloader/response_count')
        job_stats['response_count_200'] = self.dstats.get('downloader/response_status_count/200')
        for key, value in self.dstats.iteritems():
            if 'downloader/response_status_count/3' in key:
                job_stats['response_count_3xx'] = int(response_count_3xx or 0) + value
            elif 'downloader/response_status_count/4' in key:
                job_stats['response_count_4xx'] = int(response_count_4xx or 0) + value
            elif 'downloader/response_status_count/5' in key:
                job_stats['response_count_5xx'] = int(response_count_5xx or 0) + value
        job_stats['load_time'] = datetime.datetime.now()

        try:
            with self.meta_conn:
                if not self.job_id:
                    self.job_id = uuid.uuid1().hex
                    job_stats['job_id'] = self.job_id
                    job_stats['start_time'] = self.jobtime
                    self.meta_session.Insert(settings['STATS_TABLE'], job_stats)
                else:
                    wheredict = {}
                    wheredict['job_id'] = self.job_id
                    self.meta_session.Update(settings['STATS_TABLE'], job_stats, wheredict)

        except psycopg2.Error, e:
            logging.warn('Failed to refresh job stats: %s' % e)