from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from time import time
from tqdm import tqdm
import requests
import re
import feedparser
from typing import Tuple
import json

options = webdriver.FirefoxOptions()
options.headless = True
profile = webdriver.FirefoxProfile()
profile.set_preference('network.cookie.cookieBehavior', 2)
driver = webdriver.Firefox(options=options, firefox_profile=profile)

content_cache_file = 'content_cache.json'
blacklist_file = 'blacklisted_urls.json'
lat_lon_cache_file = 'lat_lon_cache.json'


class InvalidArticleException(Exception):
    def __init__(self, *args):
        super().__init__(*args)


def get_content_text(url: str) -> str:
    driver.get(url)
    try:
        return driver.find_element_by_name('articleBody').text  # type: str
    except NoSuchElementException:
        raise InvalidArticleException(f'Unable to read article at url={url}: no articleBody element!')


def get_content_html(url: str) -> str:
    driver.get(url)
    try:
        return driver.find_element_by_name('articleBody').get_attribute('innerHTML')  # type: str
    except NoSuchElementException:
        raise InvalidArticleException(f'Unable to read article at url={url}: no articleBody element!')


def get_lat_lon(place_name: str) -> Tuple[float, float]:
    response = requests.get(f'https://nominatim.openstreetmap.org/search?q={place_name}&format=json&polygon=0').json()[0]
    return float(response['lat']), float(response['lon'])


class LocationFormatException(Exception):
    def __init__(self, *args):
        super().__init__(*args)


def get_location(body_text: str) -> str:
    match = re.match('([a-zA-Z0-9., ]+)[–—―]', body_text)
    if match is None:
        raise LocationFormatException('Could not find location in text')
    return match.group(1).strip().title()


if __name__ == '__main__':
    start_time = time()

    try:
        text_by_url = json.load(open(content_cache_file))
    except FileNotFoundError:
        text_by_url = {}

    try:
        blacklist = json.load(open(blacklist_file))
    except FileNotFoundError:
        blacklist = []

    with open('feeds', 'r') as file:
        for feed_url in tqdm(file.read().splitlines()):
            for entry in tqdm(feedparser.parse(feed_url).entries):
                if entry.link in text_by_url:
                    tqdm.write(f'Already cached a copy of {entry.link}')
                    continue

                if entry.link in blacklist:
                    tqdm.write(f'Skipping (previously got parsing error): {entry.link}')
                    continue

                tqdm.write(f'Reading from {entry.link}')
                try:
                    text_by_url[entry.link] = {
                        'content': get_content_text(entry.link),
                        'html': get_content_html(entry.link),
                        'headline': entry.title,
                        'blurb': entry.description,
                    }
                    json.dump(text_by_url, open(content_cache_file, 'w'))
                except InvalidArticleException:
                    tqdm.write('Parsing error: invalid article format')
                    blacklist.append(entry.link)
                    json.dump(blacklist, open(blacklist_file, 'w'))

    print('Took {} seconds'.format(time() - start_time))
