# scrapy benign
# scrapy crawl completelyfreesoftware
import scrapy
from scrapy.selector import Selector
import re
import subprocess
# from malware.items import FileDownloadItem


FILE_STORE='G:/benignware5/'
CATAGORY = 'we'

class PortablefreewareSpider(scrapy.Spider):
    name = "tencentsoftware"
    allowed_domains = ["qq.com"]
    start_urls = ["https://pc.qq.com/category/c0-200.html"]
    res = []

    def parse(self, response):
        # print(response.body)
        sel = Selector(text=response.body,type="html")
        links = sel.xpath("//div/a/@href").re('https://.*\.exe')

        for link in links:
            print(link)
            p = subprocess.Popen(["wget", link, "-P", FILE_STORE],shell=True)
            p.wait()

        # item = FileDownloadItem()
            # item['file_urls'] = self.res
        # yield item

        if response.url.find('category') == -1:
            return
        else:
            # if response.url.find('-') == -1:
            #     nextpage = 1
            # else:
            nextpage = int(re.findall( '-(\d+).html', response.url)[0]) + 1
        if nextpage < 240:
            # if nextpage == 1:
            # print('next is c0-1.html')
            # yield scrapy.Request(url ='https://pc.qq.com/category/c0-1.html',
            #                            callback=self.parse)
        # else:
            yield scrapy.Request(url=re.sub( '-(\d+).html', '-' + str(nextpage) + '.html', response.url),
                                     callback=self.parse)


