# -*- coding: UTF-8 -*-

"""
collector.ths - 采集同花顺数据

官网：http://www.10jqka.com.cn
数据中心：http://data.10jqka.com.cn/
====================================================================
"""

import requests
from bs4 import BeautifulSoup
from requests_html import HTMLSession
from zb.crawlers.utils import get_header
import re
import pandas as pd


headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
           'Accept-Encoding': 'gzip, deflate',
           'Accept-Language': 'en-US,en;q=0.5',
           'Connection': 'keep-alive',
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 '
                         '(KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER'}


def http_requests(url):
    response = requests.get(url, headers=get_header())
    if response.status_code == 200:
        html = BeautifulSoup(response.text, 'lxml')
        return html
    else:
        raise ValueError("返回状态码（%s）不是200，url采集失败"
                         % str(response.status_code))


def zao_pan():
    """获取同花顺早盘必读信息"""
    url = "http://stock.10jqka.com.cn/zaopan/"
    response = requests.get(url, headers=get_header())
    html = BeautifulSoup(response.text, 'lxml')

    # 名人名言
    wisdom = html.find('div', {'class': "select-content"}).text.strip()

    # 昨日收盘
    yesterday = html.find('div', {'class': 'yestoday'}).text.strip()
    yesterday = yesterday.replace("&nbsp&nbsp", "|")

    # 今日关注
    content = html.find('div', {'class': "content-main-fl fl"}).text.strip()
    content = re.sub('[ \u3000]', "\n", content)

    res = [wisdom, yesterday, content]

    return "\n\n".join(res)


# 板块数据
# --------------------------------------------------------------------
def get_ths_plates():
    """获取同花顺所有概念列表

    :return: pd.DataFrame
        ['code', 'kind', 'name', 'url']
    """
    plate_kinds = {
        'gn': "概念板块",
        'dy': "地域板块",
        'thshy': "同花顺行业",
        'zjhhy': "证监会行业"
    }
    plates = []
    for kind in plate_kinds.keys():
        url = "http://q.10jqka.com.cn/%s/" % kind
        response = requests.get(url, headers=headers)
        html = BeautifulSoup(response.text, "lxml")

        results = html.find("div", {"class": "category boxShadow m_links"}).find_all("a")

        for a in results:
            url = a['href']
            plate = {
                "name": a.text,
                "code": url.strip("/").split('/')[-1],
                "url": url,
                "kind": plate_kinds[kind]
            }
            plates.append(plate)

    return pd.DataFrame(plates)


def get_plate_fund_flow(kind):
    """获取同花顺最新的行业/概念资金流

    :param kind: str
        指明需要获取的资金流类型，可选值 ['hyzjl', 'gnzjl']
        hyzjl - 行业资金流
        gnzil - 概念资金流

    :return: pd.DataFrame
        ['序号', '行业', '行业指数', '涨跌幅', '流入资金(亿)', '流出资金(亿)',
        '净额(亿)', '公司家数', '领涨股', '涨跌幅', '当前价(元)']
    """
    if kind not in ['hyzjl', 'gnzjl']:
        raise ValueError("kind 必须是 ['hyzjl', 'gnzjl'] 中的一个值")

    url_template = "http://data.10jqka.com.cn/funds/{kind}/field/" \
                   "tradezdf/order/desc/page/{page}/ajax/1/"

    i = 1
    results = []
    session = HTMLSession()
    while 1:
        url = url_template.format(page=i, kind=kind)
        response = session.get(url, headers=get_header())
        response.html.render()
        html = BeautifulSoup(response.html.text, 'lxml')

        # table = html.find('table', {'class': "m-table J-ajax-table"}).text.strip()
        table = html.text.strip()
        cells = table.split("\n")[:-2]
        total_pages = int(table.split("\n")[-2].split("/")[1])

        col_nums = 11
        row_nums = int(len(cells) / col_nums)
        col_names = cells[0: 11]
        for x in range(1, row_nums):
            results.append(cells[x * col_nums: (x + 1) * col_nums])

        # 尾页退出
        if i >= total_pages:
            break
        else:
            i += 1
    return pd.DataFrame(results, columns=col_names)


def get_plate_shares(plate_code, kind='gn'):
    """从同花顺网站获取板块中所有股票的行情

    :param plate_code: str
        板块代码，可以在这里查找对应的板块代码：http://q.10jqka.com.cn/gn/
    :param kind: str
        板块类型，可选值：['gn', 'thshy']
        gn      概念板块
        thshy   同花顺行业板块
    :return: pd.DataFrame
        ['序号', '代码', '名称', '现价', '涨跌幅(%)', '涨跌', '涨速(%)',
        '换手(%)', '量比', '振幅(%)', '成交额', '流通股', '流通市值', '市盈率']
    """
    url_template = "http://q.10jqka.com.cn/{kind}/detail/order/desc/" \
                   "page/{page}/ajax/1/code/{code}"

    kind_values = ['gn', 'thshy']
    if kind not in kind_values:
        raise ValueError("kind参数的取值必须在 %s 中" % kind_values)

    i = 1
    results = []
    session = HTMLSession()
    while 1:
        url = url_template.format(kind=kind, page=i, code=plate_code)
        response = session.get(url, headers=get_header())
        response.html.render()
        html = BeautifulSoup(response.html.text, 'lxml')

        # table = html.find('table', {'class': "m-table m-pager-table"}).text.strip()
        table = html.text.strip()
        cells = table.split("\n")[:-1]
        total_pages = int(table.split("\n")[-1].split("/")[1])
        del cells[14]

        col_nums = 14
        row_nums = int(len(cells) / col_nums)
        col_names = [x.strip() for x in cells[0: 14]]
        for x in range(1, row_nums):
            results.append(cells[x * col_nums + 2: (x + 1) * col_nums - 1])

        # 尾页退出
        if i > total_pages:
            break
        else:
            i += 1
    return pd.DataFrame(results, columns=col_names)


# 个股F10
# --------------------------------------------------------------------

class ThsF10(object):
    def __init__(self, code):
        self.code = code
        self.base_url = "http://basic.10jqka.com.cn/" \
                        "{code}/".format(code=code)

    def get_company_info(self):
        """

        页面案例：http://basic.10jqka.com.cn/600682/company.html

        :return:
        """
        base_url = self.base_url
        url = base_url + "company.html"
        html = http_requests(url)

        # 公司基本资料
        table1 = html.find('table', {"class": "m_table"})
        table1_info = [x for x in table1.text.strip().split('\n')
                       if x != ""]
        table2 = html.find('table', {"class": "m_table ggintro"})
        table2_info = [re.sub("[\t\u3000 &nbsp▼▲]", '', x) for x
                       in table2.text.strip().split('\n')
                       if x != ""]
        detail_info = table1_info + table2_info

        # 董、监、高
        m_tab_content = html.find_all('div', {'class': "m_tab_content"})
        m_tab_content_infos = []
        for table in m_tab_content:
            info = [re.sub("[ ]", '', x) for x
                    in table.text.strip().split('\n') if x != ""]
            m_tab_content_infos.append(info)

        # 发行相关
        bd_pr = html.find('div', {'class': "bd pr"}).table
        pr_info = [x for x in bd_pr.text.strip().split('\n')
                   if x != '' and x != " "]
        del pr_info[-2]
        pr_info[-1] = pr_info[-1].replace("收起▲", "")

        return {
            "详细情况": detail_info,
            "董监高": m_tab_content_infos,
            "发行相关": pr_info
        }

    def get_concept_info(self):
        base_url = self.base_url
        url = base_url + "concept.html"
        html = http_requests(url)

        def _clear(data):
            # 清理多余的数据
            while 1:
                try:
                    i = data.index('展开')
                except ValueError:
                    break
                del data[i - 1:i + 1]
            return data

        # 概念
        gn_tables = html.find_all('table', {'class': "gnContent"})
        gn_infos = []
        for table in gn_tables:
            gn = [x.strip() for x in table.text.strip().split("\n")
                  if x.strip() != ""]
            gn = _clear(gn)
            gn_infos.append(gn)

        # 题材要点
        tcyd = html.find_all('div', {'class': "gntc"})[2]
        tcyd_info = [x.strip(" \t \xa0>") for x in tcyd.text.strip().split("\n")
                     if x.strip(" \t \xa0>") != ""]
        tcyd_info = _clear(tcyd_info)

        return {
            "概念题材": gn_infos,
            "题材要点": tcyd_info
        }
