import json
import pytz
import urllib, re
from dateutil import parser
from dateutil.relativedelta import relativedelta
import datetime
import os.path
import requests
from bs4 import BeautifulSoup
BASE_URL = "https://www.espn.com"

## General functions
def retry_request(url, headers={}):
    """Get a url and return the request, try it up to 3 times if it fails initially"""
    session = requests.Session()
    session.mount("http://", requests.adapters.HTTPAdapter(max_retries=3))
    res = session.get(url=url, allow_redirects=True, headers=headers)
    session.close()
    return res

def get_soup(res):
    return BeautifulSoup(res.text, "lxml")

def get_new_json(url, headers={}):
    print(url)
    res = retry_request(url, headers)
    if res.status_code == 200:
        return res.json()
    else:
        print("ERROR:", res.status_code)
        return {"error_code": res.status_code, "error_msg": "URL Error"}

def get_new_html_soup(url, headers={}):
    print(url)
    res = retry_request(url, headers)
    if res.status_code == 200:
        return get_soup(res)
    else:
        print("ERROR: ESPN", res.status_code)
        return {"error_code": res.status_code, "error_msg": "ESPN Error"}

## Get constants
def get_date_leagues():
    return ["mlb","nba","ncb","ncw","wnba","nhl"]

def get_week_leagues():
    return ["nfl","ncf"]

def get_ncb_groups():
    return [50,55,56,100]

def get_ncw_groups():
    return [50,55,100]

def get_ncf_groups():
    return [80,81]

def get_leagues():
    """ Return a list of supported leagues """
    return get_week_leagues() + get_date_leagues()

def get_html_boxscore_leagues():
    return ["nhl"]

def get_no_scoreboard_json_leagues():
    """ Scoreboard json isn't easily available for some leagues, have to grab the game_ids from sportscenter_api url """
    return ["wnba", "nhl"]

def get_sport(league):
    if league in ["nba","wnba","ncb","ncw"]:
        return "basketball"
    elif league in ["mlb"]:
        return "baseball"
    elif league in ["nfl","ncf"]:
        return "football"
    elif league in ["nhl"]:
        return "hockey"

## Get urls
def get_sportscenter_api_url(sport, league, dates):
    return "https://sportscenter.api.espn.com/apis/v1/events?sport={}&league={}&dates={}".format(sport, league, dates)

def get_date_scoreboard_url(league, date, group=None):
    """ Return a scoreboard url for a league that uses dates (nonfootball)"""
    if league in get_date_leagues():
        if league == "nhl":
            return "{}/{}/scoreboard?date={}".format(BASE_URL, league, date)
        else:
            if group == None:
                return "{}/{}/scoreboard/_/date/{}?xhr=1".format(BASE_URL, league, date)
            else:
                return "{}/{}/scoreboard/_/group/{}/date/{}?xhr=1".format(BASE_URL, league, group, date)
    else:
        raise ValueError("League must be {} to get date scoreboard url".format(get_date_leagues()))

def get_week_scoreboard_url(league, season_year, season_type, week, group=None):
    """ Return a scoreboard url for a league that uses weeks (football)"""
    if league in get_week_leagues():
        if group == None:
            return "{}/{}/scoreboard/_/year/{}/seasontype/{}/week/{}?xhr=1".format(BASE_URL, league, season_year, season_type, week)
        else:
            return "{}/{}/scoreboard/_/group/{}/year/{}/seasontype/{}/week/{}?xhr=1".format(BASE_URL, league, group, season_year, season_type, week)
    else:
        raise ValueError("League must be {} to get week scoreboard url".format(get_week_leagues()))

def get_game_url(url_type, league, espn_id):
    valid_url_types = ["recap", "boxscore", "playbyplay", "conversation", "gamecast"]
    if url_type not in valid_url_types:
        raise ValueError("Unknown url_type for get_game_url. Valid url_types are {}".format(valid_url_types))
    return "{}/{}/{}?gameId={}&xhr=1".format(BASE_URL, league, url_type, espn_id)

def get_current_scoreboard_urls(league, offset=0):
    """ Return a list of the current scoreboard urls for a league 
    For date leagues optional offset is in days
    For week leagues optional offseet is in weeks """
    urls = []
    if league in get_date_leagues():
        date_str = (datetime.datetime.now() + relativedelta(days=+offset)).strftime("%Y%m%d")
        if league == "ncb":
            for group in get_ncb_groups():
                urls.append(get_date_scoreboard_url(league, date_str, group))
        elif league == "ncw":
            for group in get_ncw_groups():
                urls.append(get_date_scoreboard_url(league, date_str, group))
        else:
            urls.append(get_date_scoreboard_url(league, date_str))
        return urls
    elif league in get_week_leagues():
        # need to add timezone to now to compare with timezoned entry datetimes later
        dt = datetime.datetime.now(pytz.utc) + relativedelta(weeks=+offset)
        # guess the league season_year
        if dt.month > 2:
            guessed_season_year = dt.year
        else:
            guessed_season_year = dt.year - 1
        calendar = get_calendar(league, guessed_season_year)
        for season_type in calendar:
            if 'entries' in season_type:
                for entry in season_type['entries']:
                    if dt >= parser.parse(entry['startDate']) and dt <= parser.parse(entry['endDate']):
                        if league == "ncf":
                            for group in get_ncf_groups():
                                urls.append(get_week_scoreboard_url(league, guessed_season_year, season_type['value'], entry['value'], group))
                        else:
                            urls.append(get_week_scoreboard_url(league, guessed_season_year, season_type['value'], entry['value']))
        return urls
    else:
        raise ValueError("Unknown league for get_current_scoreboard_urls")

def get_all_scoreboard_urls(league, season_year):
    """ Return a list of all scoreboard urls for a given league and season year """
    urls = []
    if league in get_date_leagues():
        start_datetime, end_datetime = get_season_start_end_datetimes(league, season_year)
        while start_datetime < end_datetime:
            if league == "ncb":
                for group in get_ncb_groups():
                    urls.append(get_date_scoreboard_url(league, start_datetime.strftime("%Y%m%d"), group))
            elif league == "ncw":
                for group in get_ncw_groups():
                    urls.append(get_date_scoreboard_url(league, start_datetime.strftime("%Y%m%d"), group))
            else:
                urls.append(get_date_scoreboard_url(league, start_datetime.strftime("%Y%m%d")))
            start_datetime += relativedelta(days=+1)
        return urls
    elif league in get_week_leagues():
        calendar = get_calendar(league, season_year)
        for season_type in calendar:
            if 'entries' in season_type:
                for entry in season_type['entries']:
                    if league == "ncf":
                        for group in get_ncf_groups():
                            urls.append(get_week_scoreboard_url(league, season_year, season_type['value'], entry['value'], group))
                    else:
                        urls.append(get_week_scoreboard_url(league, season_year, season_type['value'], entry['value']))
        return urls
    else:
        raise ValueError("Unknown league for get_all_scoreboard_urls")

## Get stuff from URL or filenames
def get_league_from_url(url):
    return url.split('.com/')[1].split('/')[0]

def get_date_from_scoreboard_url(url):
    league = get_league_from_url(url)
    if league == "nhl":
        return url.split("?date=")[1].split("&")[0]
    else:
        return url.split('/')[-1].split('?')[0]

def get_data_type_from_url(url):
    """ Guess and return the data_type based on the url """
    data_type = None
    valid_data_types = ["scoreboard", "recap", "boxscore", "playbyplay", "conversation", "gamecast"]
    for valid_data_type in valid_data_types:
        if valid_data_type in url:
            data_type = valid_data_type
            break
    if data_type == None:
        raise ValueError("Unknown data_type for url. Url must contain one of {}".format(valid_data_types))
    return data_type

def get_filename_ext(filename):
    if filename.endswith(".json"):
        return "json"
    elif filename.endswith(".html"):
        return "html"
    else:
        raise ValueError("Uknown filename extension for {}".format(filename))

## Get requests helpers
def get_season_start_end_datetimes_helper(url):
    # TODO use cached replies if scoreboard url is older than 1 year
    scoreboard = get_url(url)
    return parser.parse(scoreboard['content']['sbData']['leagues'][0]['calendarStartDate']), parser.parse(scoreboard['content']['sbData']['leagues'][0]['calendarEndDate'])

def get_season_start_end_datetimes(league, season_year):
    """ Guess a random date in a leagues season and return its calendar start and end dates, only non football adheres to this format"""
    if league == "mlb":
        return get_season_start_end_datetimes_helper(get_date_scoreboard_url(league, str(season_year) + "0415"))
    elif league == "nba":
        return get_season_start_end_datetimes_helper(get_date_scoreboard_url(league, str(season_year - 1) + "1101"))
    elif league == "ncb" or league == "ncw":
        return get_season_start_end_datetimes_helper(get_date_scoreboard_url(league, str(season_year - 1) + "1130"))
    elif league == "wnba":
        # hardcode wnba start end dates, assumed to be April 20 thru Oct 31
        return datetime.datetime(season_year,4,20, tzinfo=pytz.timezone("US/Eastern")).astimezone(pytz.utc), datetime.datetime(season_year,10,31, tzinfo=pytz.timezone("US/Eastern")).astimezone(pytz.utc)
    elif league == "nhl":
        # hardcode nhl start end dates, assumed to be Oct 1 thru June 30
        return datetime.datetime(season_year-1,10,1, tzinfo=pytz.timezone("US/Eastern")).astimezone(pytz.utc), datetime.datetime(season_year,6,30, tzinfo=pytz.timezone("US/Eastern")).astimezone(pytz.utc)
    else:
        raise ValueError("League must be {} to get season start and end datetimes".format(get_date_leagues()))

def create_filename_ext(league, data_type):
    """ Get a filename extension (either .html or .json depending on league and data_type) """
    if league in get_html_boxscore_leagues() and data_type == "boxscore":
        return "html"
    else:
        return "json"

def get_filename(cached_json_path, league, data_type, url):
    """ Build a full filename with directories for given league, data_type and url"""
    # add slash if necessary to cached_json_path
    if cached_json_path[-1] != "/":
        cached_json_path += "/"
    dir_path = cached_json_path + "/" + league + "/" + data_type + "/"
    # create a league directory and data_type directory in cached_json if doesn't already exist
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    # create filename with / replaced with |
    filename = dir_path + url.replace('/','|')
    ext = "." + create_filename_ext(league, data_type)
    if filename.endswith(ext) == False:
        filename = filename + ext
    return filename

def get_cached(filename):
    """ Return cached json if it exists """
    data = None
    if os.path.isfile(filename):
        ext = get_filename_ext(filename)
        if ext == "json":
            with open(filename) as json_data:
                data = json.load(json_data)
        elif ext == "html":
            data = BeautifulSoup(open(filename), "lxml") 
    return data

## Get requests
def get_teams(league):
    """ Returns a list of teams with ids and names """
    teams = []
    if league == "ncf":
        # espn's college football teams page only lists fbs
        # need to grab teams from standings page instead if want all the fbs and fcs teams
        for division in ["fbs","fcs-i-aa"]:
            url = BASE_URL + "/college-football/standings/_/view/" + division
            print(url)
            soup = get_soup(retry_request(url))
            selector = ".hide-mobile"
            team_divs = soup.select(selector)
            for team_div in team_divs:
                teams.append({'id': team_div.find("a")['href'].split('/')[-2], 'name': team_div.text})
    else:
        url = BASE_URL + "/" + league + "/teams"
        print(url)
        soup = get_soup(retry_request(url))
        if league == "wnba":
            selector = "div.pl3"
        else:
            selector = "div.mt3"
        regions = soup.select('div.mt7')
        for region in regions:
        	state = ''
        	for var in region.select('div.headline') :
        		state = var.text
	        team_divs = region.select(selector)
	        for team_div in team_divs:
	        	teams.append({'state':state, 'id': team_div.find("a")['href'].split('/')[-2], 'name': team_div.find("h2").text})
    return teams

def get_player_id(linkPlain):
	pos = linkPlain.find('/player/_/id/')
	if pos == -1 :
		return -1
	nxt = linkPlain[pos+13:]
	posSlash = nxt.find('/')
	id = nxt[:posSlash]
	return id

def get_player_stat(league,id):
	url = 'https://www.espn.com/'+league+'/player/stats/_/id/'+str(id)
	print(url)
	soup = get_soup(retry_request(url))
	divSoup = soup.select("div.ResponsiveTable.pt4")

	divLen = len(divSoup)
	ret = []
	for k in range(divLen) :
		div = divSoup[k]
		titleSoup = div.select("div.Table__Title")
		title = titleSoup[0].text
		tableSoup = div.select("table.Table")

		# Season scrape
		trSoup = tableSoup[0].select("tr.Table__TR")
		n = len(trSoup)
		seasonList = []
		career = []
		for i in range(1,n-1):
			tr = trSoup[i]
			tdSoup = tr.select("td.Table__TD")
			seasonList.append({'season':tdSoup[0].text,'team':tdSoup[1].text})
		#Stat scrape
		trSoup = tableSoup[1].select('tr.Table__TR')
		n = len(trSoup)
		thList = []
		for i in range(n) :
			tr = trSoup[i]
			if i == 0 : # Table Head
				thSoup = tr.select("th.Table__TH")
				for th in thSoup:
					thList.append(th.text)
			else : # TD
				stat = []
				tdSoup = tr.select("td.Table__TD")
				for td in tdSoup:
					stat.append(td.text)
				if i == n-1 :
					career = stat
				else :
					seasonList[i-1]['stat'] = stat
		print(seasonList)
		print(career)
		print(thList)
		ret.append({'seasonList':seasonList,'career':career,'thList':thList})
	return ret

def get_team_info(league,team_id):
	url = BASE_URL + "/" + league + "/team/stats/_/name/"+team_id
	soup = get_soup(retry_request(url))
	leadersSoup = soup.select("h3.di")
	leaders = []
	for leader in leadersSoup :
		leaders.append(leader.contents[0])

	url = BASE_URL + "/" + league + "/team/roster/_/name/"+team_id
	soup = get_soup(retry_request(url))
	tableSoup = soup.select("tr.Table__TR")
	members = []
	for tr in tableSoup :
		step = 0
		divSoup = tr.select("div.inline")
		player = {}
		done = False
		for div in divSoup :
			tot = str(div)
			step += 1
			if '--headshot' in tot:
				continue
			if "player/_/id/" in tot:
				done = True
				player['id'] = get_player_id(tot)
				player['name'] = div.contents[0].text
			if step == 3 :
				player['pos'] = div.text
		if done == True :
			members.append(player)
	injury_url = 'https://www.espn.com/'+league+'/team/injuries/_/name/'+team_id
	soup = get_soup(retry_request(injury_url))
	injuriesSoup = soup.select("div.ContentList__Item")
	injuries = []
	for var in injuriesSoup :
		id = get_player_id(str(var))
		if id == -1 :
			continue
		injuries.append(id)
	return leaders, members, injuries

def get_standings(league, season_year, college_division=None):
    standings = {"conferences": {}}
    if league in ["nhl","nfl","mlb","nba","wnba","ncf","ncb","ncw"]:
        if league == "ncf" and college_division == None:
            # default to fbs
            college_division = "fbs"
        if college_division:
            valid_college_divisions = ["fbs", "fcs", "fcs-i-aa", "d2", "d3"]
            if college_division == "fcs":
                college_division = "fcs-i-aa"
            if college_division in valid_college_divisions:
                url = "{}/{}/standings/_/season/{}/view/{}".format(BASE_URL, league, season_year, college_division)
            else:
                raise ValueError("College division must be none or {}".format(",".join(valid_college_divisions)))
        elif league in ["wnba"]:
            url = "{}/{}/standings/_/season/{}/group/conference".format(BASE_URL, league, season_year)
        else:
            url = "{}/{}/standings/_/season/{}/group/division".format(BASE_URL, league, season_year)
        print(url)
        soup = get_soup(retry_request(url))
        standings_divs = soup.find_all("div", class_="standings__table")

        for i in range(len(standings_divs)):
            conference_name = standings_divs[i].find("div", class_="Table__Title").text
            standings["conferences"][conference_name] = {"divisions": {}}
            division = "" # default blank division name
            teams_table = standings_divs[i].find("table", class_="Table--fixed-left")
            trs = teams_table.find_all("tr")
            for tr in trs:
                if "subgroup-headers" in tr["class"]:
                    division = tr.text # replace default blank division name
                    standings["conferences"][conference_name]["divisions"][division] = {"teams": []}
                elif tr.text != "":
                    if division == "" and standings["conferences"][conference_name]["divisions"] == {}:
                        standings["conferences"][conference_name]["divisions"][division] = {"teams": []}
                    team = {}
                    team_span_tag = tr.find("td", class_="Table__TD").find("span", class_="hide-mobile")
                    team_a_tag = team_span_tag.find("a")
                    if team_a_tag is None:
                        # some teams are now defunct with no espn links
                        team["name"] = team_span_tag.text.strip()
                        team["abbr"] = ""
                    else:
                        team["name"] = team_a_tag.text
                        if league in ["ncf","ncb","ncw"]:
                            team["abbr"] = team_a_tag["href"].split("/id/")[1].split("/")[0].upper()
                        else:
                            team["abbr"] = team_a_tag["href"].split("/name/")[1].split("/")[0].upper()
                    standings["conferences"][conference_name]["divisions"][division]["teams"].append(team)

    return standings
                
def get_calendar(league, date_or_season_year):
    """ Return a calendar for a league and season_year"""
    if league in get_week_leagues():
        url = get_week_scoreboard_url(league, date_or_season_year, 2, 1)
    elif league in get_date_leagues():
        url = get_date_scoreboard_url(league, date_or_season_year)
    # TODO use cached replies for older urls
    return get_url(url)['content']['calendar']

def get_url(url, cached_path=None):
    """ Retrieve an ESPN JSON data or HTML BeautifulSoup, either from cache or make new request """
    data_type = get_data_type_from_url(url)
    league = get_league_from_url(url)
    if data_type == "scoreboard":
        # for wnba and nhl we'll use a different api to retrieve game_ids and basic game data
        if league in get_no_scoreboard_json_leagues():
            url = get_sportscenter_api_url(get_sport(league), league, get_date_from_scoreboard_url(url))
    return get_cached_url(url, league, data_type, cached_path)

def get_cached_url(url, league, data_type, cached_path, headers={}):
    """ get_url helper if want to specify the league and datatype (for non espn.com links) """
    if cached_path:
        filename = get_filename(cached_path, league, data_type, url)
        data = get_cached(filename)
    else:
        data = None
    if data == None:
        ext = create_filename_ext(league, data_type)
        if ext == "json":
            data = get_new_json(url, headers)
            # dont cache if got an ESPN internal 500 error
            if cached_path and "error_code" not in data:
                with open(filename, 'w') as f:
                    json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True)
        elif ext == "html":
            data = get_new_html_soup(url, headers)
            if cached_path and "error_code" not in data:
                with open(filename, 'w') as f:
                    f.write(data.prettify())
    return data

def get_score(league, date):
	url = 'https://www.espn.com/'+league+'/scoreboard/_/date/' + date
	print(url)
	page = requests.get(url)
	soup = BeautifulSoup(page.content, 'html.parser')
	scoreSoup = soup.find_all("th")
	print(scoreSoup)

def get_fighter(id):
	url = 'https://www.espn.com/mma/fighter/_/id/'+id
	print(url)
	soup = get_soup(retry_request(url))
	bioSoup = soup.select('div.PlayerHeader__Bio')
	player = {}
	for bio in bioSoup:
		liSoup = bio.select('li')
		for li in liSoup :
			ttuSoup = li.select('div.ttu')
			for ttu in ttuSoup:
				player[ttu.text] = li.text[len(ttu.text):]
	nameSoup = soup.select('span.truncate.min-w-0')
	fullName = ''
	for name in nameSoup :
		if len(fullName) :
			fullName += ' '
		fullName += name.text
	player['name'] = fullName
	player['id'] = id
	statSoup = soup.select('div.StatBlockInner__Value')
	step = 0
	statIns = ['W-L-D', '(T)KO', 'SUB']
	for stat in statSoup :
		player[statIns[step]] = stat.text
		step += 1
	return player

def get_fighter_list():
	alphaList = 'abcdefghijklmnopqrstuvwxyz'
	ret = []
	for alpha in alphaList :
		url = 'http://www.espn.com/mma/fighters?search=' + alpha
		soup = get_soup(retry_request(url))
		hrefSoup = soup.select('a')
		for href in hrefSoup :
			tot = str(href)
			key = '<a href="/mma/fighter/_/id/'
			pos = tot.find(key)
			if pos == -1 :
				continue
			nxt = tot[pos+len(key):]
			posSlash = nxt.find('/')
			id = nxt[:posSlash]
			ret.append(get_fighter(id))
	return ret

def ppjson(data):
    print(json.dumps(data, indent=2, sort_keys=True))

def get_game_state(league, id):
	url = 'https://www.espn.com/'+league+'/boxscore?gameId='+id
	soup = get_soup(retry_request(url))
	

print(get_current_scoreboard_urls('nfl'))
#print(get_date_scoreboard_url('nba','20200114'))
#get_score('nba','20210118')
#print(get_fighter_list())
#print(get_fighter('3165122'))
#print(get_date_scoreboard_url('nba','20200114'))
#print(get_player_stat('mlb','36724'))
#print(get_team_info("nba","bos"))
#https://github.com/erilu/web-scraping-NBA-statistics
"""
url = "https://www.espn.com/nba/player/stats/_/id/3975"
url = 'https://www.espn.com/nba/player/stats/_/id/3917376'
f = urllib.request.urlopen(url)
player_source = f.read().decode('utf-8')
# extract career stats using this regex
stats_regex = ('\[\"Career\",\"\",(.*?)\]\},\{\"ttl\"\:\"Regular Season Misc Totals\"')
career_info = re.findall(stats_regex, player_source)
print(career_info)
"""