
# coding: utf-8

from multiprocessing import Pool
from pathlib import Path
from tqdm import tqdm

import math
import requests
import time
import sys
import json
import pandas
from pprint import pprint
from collections import Counter

# n is not the number but the length of each short list after splitting, totalBatches is the cnt
def split_list(lst_long, n):
    lst_splitted = []
    totalBatches = math.ceil(len(lst_long) / n)
    for i in range(totalBatches):
        lst_short = lst_long[i*n:(i+1)*n]
        lst_splitted.append(lst_short)
    return lst_splitted

#singleCount is the progress made in an operation, totalCount is the target amount, currentCount is total progress so far
#This function can be simplified to two inputs: Progress and target. Just update the progress after an operation
def show_work_status(singleCount, totalCount, currentCount=0):
    currentCount += singleCount
    percentage = currentCount / totalCount * 1. * 100
    status = '>' * int(percentage) + ' ' * (100 - int(percentage))
    sys.stdout.write('\rStatus: [{0}] {1:.2f}%'.format(status, percentage)) # {1: .2f} is float precision
    sys.stdout.flush()
    if percentage >= 100:
        print('\n')



def worker(lst_user_id_temp):
    dic_temp = {}
    for user_id in lst_user_id_temp:
        base_url = 'http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/'
        params = {
            'key' : 'C4373AA8FFA97C04CD349DB842864903',
            'steamid' : user_id.strip(),
            'format' : 'json'
        }
        r = requests.get(base_url, params = params)
        user_inventory = r.json().get('response').get('games')
        dic_temp.update({user_id.strip():user_inventory})
        time.sleep(.5)
    return dic_temp


userPath = Path.cwd() / 'W1' / 'steam_user_id.txt'
with userPath.open() as f:
    lst_user_id = [l.strip() for l in f.readlines() if l.strip() != '']

p = Pool(5)
total_count = len(lst_user_id)
current_count = 0
show_work_status(0, total_count, current_count)

dic_master = {}
for i in tqdm(split_list(lst_user_id, 50)):
    lst_temp_dic = p.map(worker, split_list(i, 25))   # p.map() would map let the elements of a list to pass through a same function and combine the results
    for j in lst_temp_dic:
        dic_master.update(j)
    time.sleep(5)
# assume every call is done in 0 seconds, then based on the rate limit(calls/min), calculate the forced idle time to make sure it's safe.

savingPath = Path.cwd() / 'Data' / 'crawled_user_inventory.txt'
with open(savingPath, 'w') as f:
    for user_id, user_inventory in dic_master.items():
        f.write(json.dumps({user_id: user_inventory}))
        f.write('\n')

r = requests.get('http://steamspy.com/api.php?request=all')
dic_app_user = r.json()

len(dic_app_user)

pathAppDetail = Path.cwd() / 'data' / 'appDetail.txt'
lstAppId = dic_app_user.keys()
with open(pathAppDetail, 'w') as f:
    for app_id in tqdm(lstAppId):
        urlAppDetail = 'http://store.steampowered.com/api/appdetails?appids={0}'.format(app_id)
        for i in range(3):
            try:
                r = requests.get(urlAppDetail)
                result = r.json()
                break
            except:
                time.sleep(5)
                pass
        f.write(json.dumps(result)+'\n')
        time.sleep(1.52)

with open(pathAppDetail, 'r') as f:
    dic_steam_app = {'initial_price':{}, 'name':{}, 'score':{}, 'windows':{}, 'mac':{}, 'linux':{}, 'type':{}, 'release_date':{}, 'recommendation':{}, 'header_image':{}}
    lst_raw_string = f.readlines()[:3000]
    for raw_string in tqdm(lst_raw_string):
        search_obj = re.search(pattern=r'\{"(\d+)":', string=raw_string)
        id_str = search_obj.group(1)
        app_data = json.loads(raw_string)[id_str].get('data')
        if app_data:
            steam_id = app_data.get('steam_appid')
            initial_price = app_data.get('price_overview', {}).get('initial')
            if app_data.get('is_free') == True:
                initial_price = 0
            app_name = app_data.get('name')
            critic_score = app_data.get('metacritic', {}).get('score')
            app_type = app_data.get('type')
            for platform, is_supported in app_data.get('platforms', {}).items():
                if is_supported:
                    dic_steam_app[platform].update({steam_id:1})
            if not app_data.get('release_date', {}).get('coming_soon'):
                release_date = app_data.get('release_date', {}).get('date')
                if release_date:
                    try:
                        release_date = datetime.strptime(release_date, '%b %d, %Y').date()
                    except:
                        try:
                            release_date = datetime.strptime(release_date, '%d %b, %Y').date()
                        except:
                            release_date = None
            recommendation = app_data.get('recommendations', {}).get('total')
            header_image = app_data.get('header_image')
            dic_steam_app['initial_price'].update({steam_id: initial_price})
            dic_steam_app['name'].update({steam_id:app_name})
            dic_steam_app['score'].update({steam_id:critic_score})
            dic_steam_app['type'].update({steam_id:app_type})
            dic_steam_app['release_date'].update({steam_id:release_date})
            dic_steam_app['recommendation'].update({steam_id:recommendation})
            dic_steam_app['header_image'].update({steam_id:header_image})

df_app_info = pd.DataFrame(dic_steam_app)
df_app_info.to_pickle(Path.cwd() / 'data' / 'df_app_info.pkl')
engine = sqlalchemy.create_engine('mysql+pymysql://root:591743372a//@127.0.0.1/game_recommendation?charset=utf8mb4')
df_app_info.to_sql('tbl_app_info_test', engine, if_exists='replace')