import scrapy
from scrapy.selector import Selector
import re
import subprocess
from malware.items import FileDownloadItem


FILE_STORE='F:/malware2/'
# CATAGORY = ['Worm']
# CATAGORY = ['Trojan-Downloader', 'Trojan-Ransom', 'Backdoor', 'AdWare', 'Worm']
CATAGORY = ['AdWare']

class MalekalSpider(scrapy.Spider):
    name = "malekal"
    allowed_domains = ["malwaredb.malekal.com"]
    start_urls = ["http://malwaredb.malekal.com/index.php?page=2000"]#50,51,52,85
    #start_urls = ["file:///Users/emma/Work/CrawlMalware/malware/output.html"]
    res = []

    def __init__(self):
        import os
        files= os.listdir(FILE_STORE)
        for fn in files:
            self.res.append(fn)
        print(len(self.res))

    def parse(self, response):
        sel = Selector(text=response.body,type="html")
        links = sel.xpath("//div[@align='center']/table/tr/td/a/@href").extract()
        lines = response.xpath("//tr")

        for index, line in enumerate(lines):
            # print(line.extract())
            link = line.xpath('.//@href').re('.*php\?file.*')
            label = line.xpath('.//td/text()').re('kaspersky: .*')

            if len(label) == 0 or len(link) == 0:
                continue
            link = link[0]
            link = 'http://malwaredb.malekal.com' + link[1:]
            label = label[0]
            print(link, label)
            for cata in CATAGORY:
                if label.find(cata) != -1:
                    p = subprocess.Popen(["wget", link, "-P", FILE_STORE + cata +'/', '--limit-rate=100k'])
                    p.wait()
        '''
        for link in links:
            if link.lstrip('./') in self.res:
                continue
            if link.find('files.php?file=') != -1:
                self.res.append(link.lstrip('./'))
                link = response.urljoin(link)
                p = subprocess.Popen(["wget", link, "-P", FILE_STORE])
                p.wait()
        item = FileDownloadItem()
        #item['file_urls'] = self.res
        yield item
        '''
        if response.url.find('page') == -1:
            yield scrapy.Request(response.url+"/index.php?page=2", callback = self.parse)
        else:
            nextpage = int(re.findall('page=(\d+)', response.url)[0])+1
            if nextpage < 3000:
                yield scrapy.Request(url = re.sub('page=\d+','page='+str(nextpage), response.url), callback=self.parse)

