import urlparse
import mechanize
import argparse
import threading

parser = argparse.ArgumentParser()
parser.add_argument('Root',help="Enter the URL you want to visit e.g. http://www.yahoo.com")
args = parser.parse_args()
url = args.Root

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[31m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

urls = [url]
visited = [url]
broken_links = []

def initializing_browser():
    br = mechanize.Browser()
    br.set_handle_robots(False)
    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/45.0.2454101')]
    br.set_handle_refresh(True)
    return br

def crawler(urls):
    br = initializing_browser()
    try:
        br.open(urls[0])
        visited.append(urls[0])
        print bcolors.OKGREEN +visited[len(visited)-1] + bcolors.ENDC
        urls.pop(0)
        for link in br.links():
            newurl = urlparse.urljoin(link.base_url,link.url)
            if newurl not in visited and urlparse.urlparse(url).hostname in newurl and newurl not in broken_links and newurl not in urls:
                urls.append(newurl)

    except Exception as e:
        print bcolors.WARNING + urls[0] + bcolors.ENDC
        print e
        broken_links.append(urls[0])
        urls.pop(0)

#initializing_browser()
#crawler(urls)
def main():
    while len(urls)>0:
        try:
            crawler(urls)
        except:
            pass
    print bcolors.OKBLUE + "Closing Thread " + threading.currentThread().getName() + bcolors.ENDC

threads = []

if __name__=="__main__":
    for i in range(10):
        t = threading.Thread(target=main)
        threads.append(t)
        t.start()
