# -*- coding: utf-8 -*-
#
# Project: ArticleCollect (www.malwarepatrol.net)
# author : tx
# Update : 2017-12-05
#

# from w3lib.url import urljoin
import scrapy
from ArticleCollect.items import ArticlecollectItem
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import time
import re

class ForcepointSpider(CrawlSpider):
    name = 'malwarepatrol'
    allowed_domains = ['www.malwarepatrol.net']

    start_urls = ["https://www.malwarepatrol.net/blog/"]

    rules = [
        Rule(LinkExtractor(allow=('')), callback='parse_item', follow=False),
    ]


    def parse_item(self, response):
        if not response.xpath('//article[@id]//h2/a[@href]'):
            return
        for blog_url in response.xpath('//article[@id]//h2[@class="entry-title"]/a/@href').extract():
            yield scrapy.Request(url=blog_url, headers=response.headers, dont_filter=True, callback=self.parse_content)


    def parse_content(self, response):
        item = ArticlecollectItem()
        item['url'] = response.url
        item['spider_time'] = time.time()
        item['html'] = None
        item['title'] = None
        item['content'] = None
        item['publisher'] = None
        item['publish_time'] = None
        item['article_id'] = None
        item['publisher_href'] = None
        item['img_urls'] = None
        item['publisher_id'] = None

        html = response.xpath('/html').extract_first()
        item['html'] = html if html else None

        title = response.xpath('//h1[@class="entry-title"]/text()').extract_first()
        item['title'] = title.encode('utf-8') if title else None

        article_id = response.xpath('//artilc[@id]/@id').extract_first()
        item['article_id'] = article_id.encode('utf-8') if article_id else None

        item['publisher'] = "Malware Company"
        item['publisher_href'] = "https://www.malwarepatrol.net/company/"

        content = response.xpath('//div[@class="entry-content"]').extract_first()
        item['content'] = content if content else None

        item['publish_time'] = time.strftime("%Y-%m-%d", time.localtime())

        yield item


def format_date(value):
    if not value:
        return None
    str = value
    restr = r'.*\ .*,\ .*'        # time_value = "November 10, 2015"
    pattern = re.compile(restr,re.VERBOSE | re.IGNORECASE |re.DOTALL)
    ret = pattern.findall(str)
    if ret:
        try:
            strtime = ret[0].replace(',', '')
            timesp = time.strptime(strtime, "%B %d %Y")
            timesf = time.strftime("%Y-%m-%d", timesp)
            return timesf
        except:
            timesf = time.strftime("%Y-%m-%d", time.localtime())
            return timesf
    timesf = time.strftime("%Y-%m-%d", time.localtime())
    return timesf