from crawler_cluster.worker import Worker
from aiohttp import ClientSession, ClientTimeout, TCPConnector
import random


class HTTPWorker(Worker):
    user_agents = [
        'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36']

    def __init__(self, max_consecutive_errors: int = 4, nav_timeout=30_000):
        self.max_consecutive_errors = max_consecutive_errors
        self._nav_timeout = nav_timeout
        self._session = None
        self._last_url = None
        self._last_response = None

    @property
    def url(self) -> str:
        """URL of current page."""
        return self._last_response.url

    @property
    def default_nav_timeout(self) -> float:
        """Default navigation timeout in seconds."""
        return self._nav_timeout / 1000

    async def launch(self) -> bool:
        """Open HTTP client session. Return True if opened successfully."""
        self._session = ClientSession(timeout=ClientTimeout(
            nav_timeout), connector=TCPConnector(ssl=False))
        return not self._session.closed

    async def fetch(url, **kwargs):
        """Send a GET request. Return response object."""
        resp = await self._session.get(url, headers=self._headers, **kwargs)
        self._last_resp = resp
        return resp

    @property
    def _headers(self):
        return {'User-Agent': random.choice(user_agents)}

    async def _get_html(self) -> str:
        return await self._last_response.text()
