import re
from urllib.parse import urlparse, urldefrag
from bs4 import BeautifulSoup
import csv

domains = ("ics.uci.edu", "cs.uci.edu", "informatics.uci.edu", "stat.uci.edu")

def scraper(url, resp):
    links = extract_next_links(url, resp)
    return [link for link in links if is_valid(link)]

def extract_next_links(url, resp):
    # Implementation requred.
    # We need to do things here
    # Check if response is good. Please check my logic as this was done at like 1 AM LOL
    if resp:
        if resp.status >= 300: #100-299 should be able to load a page with content. All others will be invalid 
            return list()

        #list of urls found on page
        found_urls = []

        #handle tokenization of html content
        html_content = resp.raw_response.content
        soup = BeautifulSoup(html_content, 'lxml')
        url_tokens = tokenize(soup.get_text().lower()) # Get all text and make it lowercase to ignore case

        #save token results in csv
        with open('results.csv', 'a+') as results_csv:
            writer = csv.writer(results_csv)
            writer.writerow([url, len(url_tokens)])
            for token in url_tokens:
                writer.writerow([url, token])

        #start getting the links on a page
        links = soup.find_all('a', href=True)
        # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
        #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
        #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>] From BS4 Documentation
        
        # get just href
        for link in links:
            href = link.get('href')
            if href is not None:
                found_urls.append(href)
        
        return found_urls
    return list()

def is_valid(url):
    try:
        url_defrag = urldefrag(url).url
        parsed = urlparse(url_defrag)

        if parsed.scheme not in set(["http", "https"]):
            return False

        #Seeing if the current url is in the list of allowed domains
        domain_allowed = False
        for domain in domains:
            if domain in parsed.netloc:
                domain_allowed = True
        #Checking separately as it includes specific path
        if "today.uci.edu/department/information_computer_sciences" in parsed.netloc + parsed.path:
            domain_allowed = True

        #avoiding not allowed urls
        if not domain_allowed:
            return False
        
        #avoid calendars 
        if re.match(r"^.*calendar.*$", parsed.path.lower()):
            return False
        if "wics.ics.uci.edu/events/" in url: #also a calendar
            return False

        #avoid too long url (set arbitrarily at 300 for now)
        if len(url) > 300:
            return False

        return not re.match(
            r".*\.(css|js|bmp|gif|jpe?g|ico"
            + r"|png|tiff?|mid|mp2|mp3|mp4"
            + r"|wav|avi|mov|mpeg|ram|m4v|mkv|ogg|ogv|pdf"
            + r"|ps|eps|tex|ppt|pptx|doc|docx|xls|xlsx|names"
            + r"|data|dat|exe|bz2|tar|msi|bin|7z|psd|dmg|iso"
            + r"|epub|dll|cnf|tgz|sha1"
            + r"|thmx|mso|arff|rtf|jar|csv"
            + r"|rm|smil|wmv|swf|wma|zip|rar|gz)$", parsed.path.lower())

    except TypeError:
        print ("TypeError for ", parsed)
        raise


# Ethan's implementation from Assingment 1. Feel free to change
def tokenize(text):
    tokens = []
    if len(text) > 0:
        # found https://stackoverflow.com/questions/4893506/fastest-python-method-for-search-and-replace-on-a-large-string
        # string generated by copying stopwords to file and printing string with | as delimiter.
        pattern = re.compile(r"\b(a|about|above|after|again|against|all|am|an|and|any|are|aren't|as|at|be|because|been|before|being|below|between|both|but|by|can't|cannot|could|couldn't|did|didn't|do|does|doesn't|doing|don't|down|during|each|few|for|from|further|had|hadn't|has|hasn't|have|haven't|having|he|he'd|he'll|he's|her|here|here's|hers|herself|him|himself|his|how|how's|i|i'd|i'll|i'm|i've|if|in|into|is|isn't|it|it's|its|itself|let's|me|more|most|mustn't|my|myself|no|nor|not|of|off|on|once|only|or|other|ought|our|ours|ourselves|out|over|own|same|shan't|she|she'd|she'll|she's|should|shouldn't|so|some|such|than|that|that's|the|their|theirs|them|themselves|then|there|there's|these|they|they'd|they'll|they're|they've|this|those|through|to|too|under|until|up|very|was|wasn't|we|we'd|we'll|we're|we've|were|weren't|what|what's|when|when's|where|where's|which|while|who|who's|whom|why|why's|with|won't|would|wouldn't|you|you'd|you'll|you're|you've|your|yours|yourself|yourselves)\b")
        rep = ''
        re.sub(pattern, rep, text)
        try:
            matches = re.findall(r'\w{2,}', re.sub(pattern, rep, text))
            tokens.extend(matches)
        except UnicodeDecodeError:
            print("Ran into decoding error.")
    return tokens