from urllib.request import Request, urlopen
import base
import json
import fn
import tdx
import sys
import re
import sqllite_gn as stock_gn
import sqllite_hy_list as stock_hy_list

# 实例化 通达信 行业类
tdx_hy = tdx.hy()

class Spider(base.BaseClass):

    url = 'http://basic.10jqka.com.cn/300269/concept.html#ifind'

    def __fetch_content(self, code):
        url = "http://basic.10jqka.com.cn/%s/concept.html#ifind" % (code)
        # 包装头部
        firefox_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}

        r = Request(url, headers=firefox_headers)
        html = urlopen(r)

        # 获取数据
        data = html.read()
        page = data.decode("gbk", "ignore")

        if page:
            return page
        else:
            return []

    # 概念入库
    #
    # code int 股票编码
    # zxg int 设置自选股 0为否 1为是
    def gn_op(self, code):
        _time = fn.get_time()
        res = stock_gn.check_row(code, _time)

        data ={}
        data['stock'] = code
        # 获取符合更新的数据
        if res:
            content = self.__fetch_content(code)
            rule = '';
            rule += '<td class="gnName".+?>(.+?)</td>';
            s = re.findall(rule, content, re.S)
            res = ''
            for i in s:
                res += fn.filter_tags(i) + ' '

            data['info'] = res
            data['expire_at'] = int(_time + 172800)
            stock_gn.save(data)
            print(code + '更新数据')
        elif not res:
            # 插入数据
            data['info'] = ''
            data['expire_at'] = _time
            res = stock_gn.save(data)
            if res == 1:
                print(code + '插入数据')
            else:
                print(code + '无须更新')

    # 从stock_gn里获取code
    def get_code_from_gn(self):
        rows = stock_gn.getCode()
        for row in rows:
            code = row.get('code')
            print(code + '采集中..')
            self.gn_op(code)
            # print(code + '采集完毕..')

    # 从行业 stock_hy_list获取code
    def __get_code(self):
        rows = stock_hy_list.getCode()
        for row in rows:
            code = row[0]
            print(code + '采集中..')
            self.gn_op(code)
            # print(code + '采集完毕..')

    def go(self):
        self.__get_code()


# spider = Spider()
# spider.go()
