from bs4 import BeautifulSoup
from urllib.request import urlopen
from urllib.parse import urljoin
import urllib.request
import time
import sys
import logging
"""
If there is a problem with recursion limit, set a new limit
Extensions are to prevent the program from entering not. html files so as to it won't return errors
To crawl through websites, enter a index page url
Examples of websites are given below
"""
sys.setrecursionlimit(150)

EXTENSIONS = ('.txt', '.jpg', '.png', '.pdf')

# url ='https://automatetheboringstuff.com/'
# url = 'https://piotr-cv-project.herokuapp.com/'
# url = "https://www.pythonforbeginners.com/"
url = "http://0.0.0.0:8000/"
# url = "https://halome.nu/"
# url = "http://quirktools.com/screenfly/"
# url = "http://www.smallwebsites.co/"

start_time = time.time()


def site_map(index_url):
    """
    Saves site map to the .json file
    :param index_url: Url of index page of given website
    :return: Returns n-element dictionary. n is equal to number of accessible sub-websites of index_url
    {
    url: {
    'title': title,
    'links': {set of links}
        }
    url: {...}
    }
    """
    set_of_urls = set()

    set_of_urls = get_urls(index_url, set_of_urls, index_url)
    result = get_titles_and_links(set_of_urls, index_url)

    return result


def get_urls(changing_url, set_of_urls, main_url):
    """
    Getting data of all accessible urls
    :param changing_url: Mutable Url used in finding next urls by recursion method
    :param set_of_urls: Empty list which is being filled with urls while the function is working
    :param main_url: Immutable url used in verification of the beginning of links
    :return: Filled list set_of_urls with every accessible link
    """
    if changing_url in set_of_urls:
        return set_of_urls

    try:
        content = urlopen(changing_url).read()
        soup = BeautifulSoup(content, "html.parser")
        set_of_urls.add(changing_url)
        links = {
            urljoin(changing_url, link.get('href')) for link in soup.find_all('a')
            if not urljoin(changing_url, link.get('href')).endswith(EXTENSIONS)
        }
        links = links - set_of_urls             # Optimization by removing unnecessary, earlier checked links
        for link in links:
            if link.startswith(main_url) and link not in set_of_urls:
                try:
                    set_of_urls.union(get_urls(link, set_of_urls, main_url))
                except TypeError:
                    pass
        return set_of_urls
    except urllib.error.HTTPError or ValueError:
        logging.warning('HttpError, can\'t access the website')
        pass


def get_titles_and_links(set_of_urls, main_url):
    """
    This function generates titles and urls to associated urls and adds it to url value in dictionary
        Example of one element generated by one url:
        {
        url: {
        'title': title,
        'links': {set of links}
            }
        url: {...}
        }
    :param set_of_urls: Filled list with urls based on which title and links are fetched from associated website
    :param main_url: Immutable url used in verification of the beginning of links
    :return: Returns n-element dictionary. n is equal to length of set_of_urls list
    """
    target = {one_url: {} for one_url in set_of_urls}
    for one_url in set_of_urls:
        content = urlopen(one_url).read()

        soup = BeautifulSoup(content, "html.parser")

        title = soup.title.string
        if title:
            target[one_url] = {'title': title}

        links = {
            urljoin(one_url, link.get('href')) for link in soup.find_all('a')
            if main_url in urljoin(one_url, link.get('href'))
        }

        if links is None:
            target[one_url]['links'] = 'set()'
        else:
            target[one_url]['links'] = links

    return target


if __name__ == '__main__':
    site_map(url)
