import urllib
import urlparse
from bs4 import BeautifulSoup

url = "http://www.example.com/"

remaining = [url]

visited = [url]

while len(remaining) > 0:
    try:
        htmltext = urllib.urlopen(remaining[0]).read()
    except:
        print remaining[0]

    soup = BeautifulSoup(htmltext, 'lxml')

    remaining.pop(0)
    print len(remaining)

    for tag in soup.findAll('a', href=True):
        tag['href'] = urlparse.urljoin(url, tag['href'])
        if url in tag['href'] and tag['href'] not in visited:
            remaining.append(tag['href'])
            visited.append(tag['href'])


for i in visited:
    print i;
