# -*- coding: UTF-8 -*-

import sys
import os
import json
sys.path.append("..")
import html_downloader
from bkupdate import outputer, html_parser, url_manager
from urllib.parse import unquote
from argument_parser import ArgumentParser
from elasticsearch import Elasticsearch
import logging
from datetime import date
import socket
socket.setdefaulttimeout(300)


class SpiderMain:
    def __init__(self, interrupt=False):
        self.args = ArgumentParser()
        self.count = self.args.keyword_start
        self.urls = url_manager.UrlManager()
        self.downloader = html_downloader.HtmlDownloader()
        self.parser = html_parser.HtmlParser()
        self.outputer = outputer.HtmlOutputer()
        self.title_to_id = dict()
        self.interrupt = interrupt
        # 初始化ES客户端
        # self.es = Elasticsearch(hosts=self.args.es_url, timeout=120)

        # 爬虫更新中断后保存之前爬取的词条
        if self.interrupt:
            self.interrupt_previous_titles = set()
            self.load_interrupt_titles()
            print("Load %d titles." % len(self.interrupt_previous_titles))
        # self.url_to_update_time = dict()
        # logging.info("Load data from ES...")
        # self.load_previous_update_time(args)
        # logging.info("Data loaded.")

    def load_interrupt_titles(self):
        for file_dir in os.listdir(self.args.update_interrupt_dir):
            print("Load previous titles of dir %s" % file_dir)
            for file_name in os.listdir(os.path.join(self.args.update_interrupt_dir, file_dir)):
                self.interrupt_previous_titles.add(file_name.strip('.html'))

    def load_previous_update_time(self, args):
        """
        加载上次更新时间
        :param args:
        :return:
        """
        body = {
            "query": {
                "exists": {
                    "field": "update_time"
                }
            },
            "_source": [
                "id",
                "key_url",
                'update_time'
            ],
            "size": 10000
        }
        data = self.es.search(index=args.es_index_name, body=body, scroll='2m')
        sid = data['_scroll_id']
        scroll_size = len(data['hits']['hits'])
        self.load_batch_update_time(data)
        cnt = 0
        while scroll_size > 0:
            data = self.es.scroll(scroll_id=sid, scroll='2m')
            self.load_batch_update_time(data)
            sid = data['_scroll_id']
            scroll_size = len(data['hits']['hits'])
            cnt += 1
            if cnt % 100 == 0:
                logging.info("Read {} items.".format(cnt*10000))

    def load_batch_update_time(self, data):
        """
        批量更新函数
        :param data:
        :return:
        """
        for data in data['hits']['hits']:
            data = data['_source']
            if data['key_url']:
                self.url_to_update_time[data['key_url']] = data['update_time']

    def craw(self):
        while self.urls.has_new_url():
            page_url = self.urls.get_new_url()
            try:
                html_cont, item_url = self.downloader.download_update(page_url)
                item_url = item_url.split('?fromtitle')[0]
                update_time, title = self.parser.parse_update_time(html_cont)
                # if item_url in self.url_to_update_time:
                #     if update_time > self.url_to_update_time[item_url]:
                #         self.outputer.collect_data(html_cont, item_url, title)
                # else:
                #     self.outputer.collect_data(html_cont, item_url, title)
                if self.interrupt and title in self.interrupt_previous_titles:
                    self.urls.add_old_url(page_url)
                    self.count += 1
                    print("Keyword already exist %d Url: %s" % (self.count, unquote(page_url)))
                    continue
                if update_time >= '2018-01-01 00:00:00':
                    self.outputer.collect_data(html_cont, item_url, title)
                self.urls.add_old_url(page_url)
                self.count += 1
                # if self.count % 1000000 == 0:
                #     logging.debug('Visited %d items.' % self.count)
                print("Keyword Success %d Url:%s" % (self.count, unquote(page_url)))
            except:
                self.urls.add_fail_url(page_url)
                print("Failed")


if __name__ == '__main__':
    args = ArgumentParser()
    logging.basicConfig(
        level=logging.INFO,
        filename=args.update_output_log_path,
        filemode='a',
        format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
    )
    spider = SpiderMain(interrupt=True)
    # logging.info("The number of entities with baike urls is {}.".format(len(spider.url_to_update_time)))
    spider.craw()
    spider.outputer.log_file_handler.close()