import requests
from bs4 import BeautifulSoup
import validators
import queue
from crawler import Spoofer
from threading import Thread

"""
    Author:
        - Scot Matson

    Description: Basic web crawler that collects web pages for later parsing.
                 Used to collect large data sets; to be used for learning
                 about MapReduce implementations.

                 Due to the large volume of web traffic this may potentially
                 produce, care should be used not to inadventently produce
                 a DDoS attack or violate a company's security policies.

    Version:     Alpha v0.1
    Change Log:  Data loss is high, but not something I'm worried about
                 at this time. Initial implementation is focused on
                 ability to crawl K-defined pages without error.

    NOTES:
        I am unable to access sjsu.edu today, I may have been blacklisted. (whoops!)
        Careful while crawling, don't use large data sets. We need to implement a
        rotating domain and IP spoofing apparently.
"""

page_number = 0
concurrent = 100

def crawl_web(seed_url, number_of_pages):
    """
        Author: Scot Matson

        Contributors:
            - [David Barban]

        Master process for web crawling operations.
    """
    seed = seed_url
    MAX_PAGES = number_of_pages
    global page_number
    q = queue.Queue()
    q.put(seed)
    visited = set() # Prevent revisiting

    while not q.empty() and page_number < MAX_PAGES:
        # Thread page scraping process
        for i in range(concurrent):
            t = Thread(target=doWork(visited, q))
            t.daemon = True
            t.start()

'''
def doWork(visited, q):
    """
        Author: David Barban

        Contributors:
            - [Your name here]

        Master process for web crawling operations.
    """
    #Abstract page scraping further for threading
    url = q.get()
    if url not in visited:
        scrape_page(url, q)
        visited.add(url)
    else:
        #print('Seen:',url)
        pass
'''

def scrape_page(url, q):
    """
        Author:
            - Scot Matson

        Contributors:
            - [Your name here]

        Scrapes 'href' information from a web page
    """
    # Set user-agent to address cases in which sites are blocking
    # traffic unassociated with a web browser
    global page_number

    # Dynamically generate a user-agent
    user_agent = Spoofer.UserAgent()
    # compose header list
    header_list = {'User-Agent' : user_agent.get_user_agent(),"Accept-Encoding": "gzip, deflate"}
    #timeout = 0.050
    timeout = 5000.0
    #try:
    # Requests for web requests
    r = requests.get(url, headers=header_list, timeout=timeout)
    plain_text = r.text

    # BeautifulSoup for HTML parsing
    soup = BeautifulSoup(plain_text, 'html5lib')

    # Store page and increment counter
    save_page(url, soup.prettify())
    print('['+str(page_number)+']','Stored:',url)
    page_number += 1

    # This is where the actual scraping occurs
    # Probably should put this into its own function
    for line in soup.find_all('a'):
        href = str(line.get('href'))
        if (validators.url(href)):
            # Add scraped pages to the Queue
            q.put(line.get('href'))
        else:
            # Likely malformed. Okay to dump for now.
            # print('Malformed:',url)
            pass
    #except:
        # To maintain efficiency, we are going to burn any page that
        # loads slowly, timesout, or has too many redirects.
        # print('Timeout:',url)
    #    pass

def save_page(url, source_code):
    """
        Author:
            - Scot Matson

        Contributors:
            - [Your name here]

        Save data to a file.
        Notes:
            Probably doesn't belong in this file, it deserves its own I/O class.
            Is this something worth doing?
    """
    global page_number
    file_path = 'web_pages' # For now, just use a local directory
    file_name = '/page_' + str(page_number)
    file_ext  = '.html'

    html_file = file_path+file_name+file_ext
    fh = open(html_file, 'a')
    fh.write('<!--'+url+'-->')
    fh.write(source_code)
    fh.close()
