'''
Created on Dec 13, 2016

@author: ToOro
'''
import time
import urlparse
import threading
import multiprocessing
import sys

from technique.web_crawling.mongo_cache import MongoCache
from mongo_queue import MongoQueue
from technique.web_crawling.util.common import Downloader
from technique.web_crawling.util.common import normalize
from alexa_cb import AlexaCallback

SLEEP_TIME = 1

def threaded_crawler(seed_url, delay=5, cache=None, scrape_callback=None, user_agent='wswp',
                     proxies=None, num_retries=1, max_threads=10, timeout=60):
    """Crawl using multiple threads
    """
    # the queue of URL's that still need to be crawled
    crawl_queue = MongoQueue()
    crawl_queue.clear()
    crawl_queue.push(seed_url)
    D = Downloader(cache=cache, delay=delay, user_agent=user_agent, proxies=proxies,
                   num_retries=num_retries, timeout=timeout)
    
    def process_queue():
        while True:
            # keep track that are processing url
            try:
                url = crawl_queue.pop()
            except KeyError:
                # currently no urls to process
                break
            else:
                html = D(url)
                if scrape_callback:
                    try:
                        links = scrape_callback(url, html) or []
                    except Exception as e:
                        print 'Error in callback for : {}: {}'.format(url, e)
                    else:
                        for link in links:
                            #add this new links to queue
                            crawl_queue.push(normalize(seed_url, link))
                crawl_queue.complete(url)
    
    # wait for all download threads to finish
    threads = []
    while threads or crawl_queue:
        for thread in threads:
            if not thread.is_alive():
                threads.remove(thread)
        while len(threads) < max_threads and crawl_queue.peek():
            # can start some more threads
            thread = threading.Thread(target=process_queue)
            thread.setDaemon(True) # set daemon so main thread can exist when receives ctrl-c
            thread.start()
            threads.append(thread)
        time.sleep(SLEEP_TIME)
        
def process_crawler(args, **kwargs):
    #num_cpus = multiprocessing.cpu_count()
    num_cpus = 1
    # pool = multiprocessing.Pool(processes=num_cpus)
    print 'Starting {} processes'.format(num_cpus)
    processes = []
    for i in range(num_cpus):
        p = multiprocessing.Process(target=threaded_crawler, args=[args], kwargs=kwargs)
        # parsed = pool.apply_async(threaded_link_crawler, args, kwargs)
        p.start()
        processes.append(p)
    # wait for processes to complete
    for p in processes:
        p.join()