import sys
import re
import urllib2
import urlparse
#tocrawl = set([sys.argv[1]])
tocrawl = set("http://www.htl.rennweg.at/")
crawled = set([])
keywordregex = re.compile('<meta\sname=["\']keywords["\']\scontent=["\'](.*?)["\']\s/>')
linkregex = re.compile('^(http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[p;/~\+#])?')


def func_google():
#while 1:
#    try:
  #      crawling = tocrawl.pop()
    #    print crawling
    #except KeyError:
      #  raise StopIteration
    crawling = "http://www.htl.rennweg.at/"
    url = urlparse.urlparse(crawling)
    try:
        response = urllib2.urlopen(crawling)
    except:
        print("Fehler")#continue
    msg = response.read()
    startPos = msg.find('<title>')
    if startPos != -1:
        endPos = msg.find('</title>', startPos+7)
        if endPos != -1:
            title = msg[startPos+7:endPos]
            print title
    keywordlist = keywordregex.findall(msg)
    if len(keywordlist) > 0:
        keywordlist = keywordlist[0]
        keywordlist = keywordlist.split(", ")
        print keywordlist
    links = linkregex.findall(msg)
    print(links)
    crawled.add(crawling)
    for link in links:#(links.pop(0) for _ in xrange(len(links))):
        if link.startswith('/'):
            link = 'http://' + url[1] + link
        elif link.startswith('#'):
            link = 'http://' + url[1] + url[2] + link
        elif not link.startswith('http'):
            link = 'http://' + url[1] + '/' + link
        if link not in crawled:
            tocrawl.add(link)
            print(link)
    linkmap = {}
    
    for it in tocrawl:
        if not it in linkmap:
            linkmap[it] = 1
        else:
            linkmap[it] += 1
        for it,it2 in linkmap.items():
            print("key",it,"value",it2)
            
func_google()
            
