from collections import deque
import httplib
import json
import logging
import os.path
import sys
import time


logging.basicConfig(
    format='%(asctime)s %(message)s',
    filename='crawl.log',
    level=logging.DEBUG)


def log_exception_hook(type, value, tb):
    logging.error("Exception", exc_info=(type, value, tb))
    print "%s: %s\n%s" % (type, value, tb)
sys.excepthook = log_exception_hook


url_format = "/api/%(type)s/%(id)s/?api_key=%(apikey)s&format=json&field_list=%(fields)s"
apikey = "YOURKEYHERE"
mario_id = 177


def save_image(filename, data):
    logging.debug("SAVE %s", filename)
    with open(filename, "wb") as f:
        f.write(data)

def load_json(filename):
    with open(filename, "rb") as f:
        return json.loads(f.read())

def load(name):
    try:
        return load_json("%s.json" % name)
    except:
        return []

def fetch(type, id):
    filename = "%s/%s.json" % (type, id)
    return load_json(filename)

def fetch_character(id):
    return fetch("character", id)


def get_image(url):
    time.sleep(1.1)
    url = url + "?api_key=" + apikey
    conn = httplib.HTTPConnection("www.giantbomb.com")
    headers = {
        "User-Agent": "sixdegreesmario bot 2.0",
    }
    conn.request("GET", url, None, headers)
    r = conn.getresponse()
    logging.debug("GET %s %s %s", url, r.status, r.reason)
    if r.status != 200:
        print url
        print r.status, r.reason
        return None
    return r.read()

def p(s):
    print s.encode(sys.stdout.encoding, 'replace')


done = load("done")
todo = load("todo")
seen = done + todo
seen_games = {id: (name, prev) for type, id, name, degree, prev in seen if type == "game"}
seen_characters = {id: (name, prev, degree) for type, id, name, degree, prev in seen if type == "character"}

rev_games = {}
rev_characters = {}
for type, sid, sname, degree, prev in seen:
    if type == "game":
        rev_games.setdefault(prev, []).append((sname, sid))
    elif type == "character":
        rev_characters.setdefault(prev, []).append((sname, sid))

def fill_tree(important_chars):
    tree_chars = set(important_chars)
    tree_games = set()
    tree_edges = list()
    for important_char_id in important_chars:
        char_id = important_char_id
        while True:
            game_id = seen_characters[char_id][1]
            if game_id is None:
                break
            tree_edges.append((char_id, game_id))
            if game_id in tree_games:
                break
            tree_games.add(game_id)
            char_id = seen_games[game_id][1]
            tree_edges.append((game_id, char_id))
            if char_id in tree_chars:
                break
            tree_chars.add(char_id)
    return tree_chars, tree_games, tree_edges

def isingames(id,min=2):
    return id in rev_games and len(rev_games[id])>=min

def has_descr(id):
    try:
        data = fetch_character(id)
    except:
        return False
    r=data["results"]
    return r["description"] is not None

def find(maxdeg=6):
    c=[]
    for type, id, name, degree, prev in seen:
        if type != "character": continue
        if degree>maxdeg: continue
        if not isingames(id,1): continue
        if not has_descr(id): continue
        c.append(id)
    return fill_tree(c)

cnodes, gnodes, edges = find(6)

folder = "cimg"

#kind = "medium"
#important_characters = [177]

kind = "icon"
important_characters = cnodes

for id in important_characters:

    #if not isingames(id,3): continue
    #if not has_descr(id): continue

    filename = "%s/%s-%s.png" % (folder, kind, id)
    if os.path.isfile(filename): continue
    try:
        cdata = fetch_character(id)
    except:
        p(r["name"])
        print "no character data <------"
        continue
    r = cdata["results"]
    if "image" not in r:
        p(r["name"])
        print "no image data <------"
        continue
    rimage = r["image"]
    if rimage is None:
        p(r["name"])
        print "image data is None <------"
        continue
    if kind+"_url" not in rimage:
        p(r["name"])
        print "no image of type <------", kind
        continue
    url = rimage[kind+"_url"]
    p(r["name"])
    img = get_image(url)
    save_image(filename, img)
