import scrapy
import os
import time
import random
import json
from copy import deepcopy
from scrapy.http import Request
from urllib.parse import urljoin
from pyquery import PyQuery as pq
from urllib.parse import urlencode

from loophole.settings import BASE_DIR
from loophole.items import RepositoryItem, ExploitDbItem


class ExploitDbSpider(scrapy.Spider):
    name = 'exploit_db'
    # allowed_domains = ['exploit-db.com/']
    start_urls = ["https://www.exploit-db.com/"]

    custom_settings = {
        # 设置管道下载
        'ITEM_PIPELINES': {
            'loophole.pipelines.ExploitDbMysqlPipeline': 300,
        },
    }

    headers = {
        'X-Requested-With': 'XMLHttpRequest',
        'Referer': 'https://www.exploit-db.com/',
    }
    start = 0
    length = 120
    page = 1
    param = {
        'draw': '1',
        'columns[0][data]': 'date_published',
        'columns[0][name]': 'date_published',
        'columns[0][searchable]': 'true',
        'columns[0][orderable]': 'true',
        'columns[0][search][value]': '',
        'columns[0][search][regex]': 'false',
        'columns[1][data]': 'download',
        'columns[1][name]': 'download',
        'columns[1][searchable]': 'false',
        'columns[1][orderable]': 'false',
        'columns[1][search][value]': '',
        'columns[1][search][regex]': 'false',
        'columns[2][data]': 'application_md5',
        'columns[2][name]': 'application_md5',
        'columns[2][searchable]': 'true',
        'columns[2][orderable]': 'false',
        'columns[2][search][value]': '',
        'columns[2][search][regex]': 'false',
        'columns[3][data]': 'verified',
        'columns[3][name]': 'verified',
        'columns[3][searchable]': 'true',
        'columns[3][orderable]': 'false',
        'columns[3][search][value]': '',
        'columns[3][search][regex]': 'false',
        'columns[4][data]': 'description',
        'columns[4][name]': 'description',
        'columns[4][searchable]': 'true',
        'columns[4][orderable]': 'false',
        'columns[4][search][value]': '',
        'columns[4][search][regex]': 'false',
        'columns[5][data]': 'type_id',
        'columns[5][name]': 'type_id',
        'columns[5][searchable]': 'true',
        'columns[5][orderable]': 'false',
        'columns[5][search][value]': '',
        'columns[5][search][regex]': 'false',
        'columns[6][data]': 'platform_id',
        'columns[6][name]': 'platform_id',
        'columns[6][searchable]': 'true',
        'columns[6][orderable]': 'false',
        'columns[6][search][value]': '',
        'columns[6][search][regex]': 'false',
        'columns[7][data]': 'author_id',
        'columns[7][name]': 'author_id',
        'columns[7][searchable]': 'false',
        'columns[7][orderable]': 'false',
        'columns[7][search][value]': '',
        'columns[7][search][regex]': 'false',
        'columns[8][data]': 'code',
        'columns[8][name]': 'code.code',
        'columns[8][searchable]': 'true',
        'columns[8][orderable]': 'true',
        'columns[8][search][value]': '',
        'columns[8][search][regex]': 'false',
        'columns[9][data]': 'id',
        'columns[9][name]': 'id',
        'columns[9][searchable]': 'false',
        'columns[9][orderable]': 'true',
        'columns[9][search][value]': '',
        'columns[9][search][regex]': 'false',
        'order[0][column]': '9',
        'order[0][dir]': 'desc',
        'start': start,
        'length': length,
        'search[value]': '',
        'search[regex]': 'false',
        'author': '',
        'port': '',
        'type': '',
        'tag': '',
        'platform': '',
        '_': '1587968469299',
    }
    main_url = start_urls[0] + "?" + urlencode(param)

    def start_requests(self):
        yield Request(url=self.main_url, callback=self.parse, headers=self.headers)
        self.start += self.length

    def parse(self, response):
        print("==========当前抓取第{}页==========".format(self.page))
        item = ExploitDbItem()
        result_str = response.body_as_unicode()
        result_dict = json.loads(result_str)
        records_total = result_dict["recordsTotal"]
        source = "https://www.exploit-db.com"
        for obj in result_dict["data"]:
            title = obj["description"][1]
            date = obj["date_published"]
            cve = obj["code"][0]["code_type"] + "-" + obj["code"][0]["code"] if obj["code"] else ""
            link = source + "/exploits/" + obj["id"]

            # 下载链接
            doc = pq(obj["download"])
            a = doc('a')
            href = a.attr('href')
            download_link = urljoin(source, href)

            download_id = obj["id"]
            author = obj["author_id"][-1]
            type = obj["type_id"]
            platform = obj["platform_id"]
            verified = obj["verified"]

            # # 我的
            # item["title"] = title
            # item["date"] = date
            # item["cve"] = cve
            # item["link"] = link
            # item["source"] = source
            # item["type"] = type
            # item["download_id"] = download_id
            # item["author"] = author
            # item["platform"] = platform
            # item["verified"] = verified

            # you_sir的
            item['name'] = title
            item['add_time'] = date
            # item['infotype'] = type
            item['infotype'] = 0
            item['language'] = platform

            yield Request(url=download_link,
                          meta={'item': deepcopy(item)},
                          headers=self.headers,
                          callback=self.parse_detail)
            time.sleep(random.random())
        self.page += 1
        self.start += self.length
        self.param["start"] = self.start
        next_url = self.start_urls[-1] + '?' + urlencode(self.param)
        records_total = 5
        if self.page <= records_total:
            yield Request(url=next_url, callback=self.parse, headers=self.headers)

    def parse_detail(self, response):
        """处理下载页"""
        item = response.meta.get('item')

        # 我的
        # download_id = item.get('download_id')
        # content = response.text
        # file_dir = os.path.join(BASE_DIR, 'download')
        # if not os.path.exists(file_dir):
        #     os.makedirs(file_dir)
        # with open(os.path.join(file_dir, '{}.txt'.format(download_id)), 'w+', encoding='utf-8') as f:
        #     f.write(content)

        # you_sir的
        content = response.text
        item['urlinfo'] = content

        print(item)
        yield deepcopy(item)
