""" [Malwarebytes] Web Scraper """

import os
import logging
from scrapy import Request
from .abstract_crawler import AbstractWebCrawler


class MalwarebytesCrawler(AbstractWebCrawler):
    """ [Malwarebytes] Web Scraper """

    # Spider Properties
    name = "web_malwarebytes"

    # Crawler Properties
    resource_link = 'https://blog.malwarebytes.com/'
    resource_label = 'malwarebytes'

    # TODO Move it to the super class
    custom_settings = {
        'ITEM_PIPELINES': {
            'scrapy_crawlers.pipelines.ElasticIndexPipeline': 500
        }
    }

    # Sets all css selectors to none
    links_to_articles_query = '#content > div > div.large-card > div.content > h2 > a::attr(href)'
    links_to_pages_query = None
    extract_title_query = '#content > div:nth-child(1) > h1::text'
    extract_datetime_query = '#content > div:nth-child(1) > p.small > span::text'
    extract_content_query = '#articleBody'

    api_url = 'https://blog.malwarebytes.com/page/'
    api_page = 1

    def __init__(self):
        super().__init__()  # Calls __init__ method from base class
        self.start_urls = [self.api_url + str(self.api_page)]

    @classmethod
    def parse(self, response):
        """ References Parser """

        logging.info('Parsing page on %s', response.url)

        # follow links to articles
        for href in self.links_to_articles(response):
            yield response.follow(href, self.parse_article)

        # break scraping if page size is 0
        if len(response.body) == 0:
            return

        # follow pages
        self.api_page = self.api_page + 1
        yield Request(self.api_url + str(self.api_page), callback=self.parse)
