from bs4 import BeautifulSoup as bs
import requests
import re
import numpy as np
from datetime import datetime, date, timedelta
import pandas as pd
from urllib.parse import urljoin
from collections import defaultdict
import time
import pickle
import os
import glob
import argparse

'''Eastside Sierras'''
sierras_cathedral_range_url = 'https://www.mountain-forecast.com/subranges/cathedral-range/locations'
sierras_other_url = 'https://www.mountain-forecast.com/subranges/2662/locations'
sierras_carson_url = 'https://www.mountain-forecast.com/subranges/carson-range/locations'
'''Cascades'''
cascade_range_url = 'https://www.mountain-forecast.com/subranges/cascade-range-3/locations'
'''Southern California'''
san_bernadino_url = 'https://www.mountain-forecast.com/subranges/san-bernardino-mountains/locations'
san_jacinto_url = 'https://www.mountain-forecast.com/subranges/san-jacinto-mountains/locations'
'''Wyoming'''
teton_range_url = 'https://www.mountain-forecast.com/subranges/teton-range-1/locations'
wind_river_range_url = 'https://www.mountain-forecast.com/subranges/wind-river-range-1/locations'
'''Hawaii'''
hawaii_url = 'https://www.mountain-forecast.com/subranges/hawaiian-islands/locations'


def load_urls(urls_filename):
	""" Returns dictionary of mountain urls saved a pickle file """

	full_path = os.path.join(
		'G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/Mountain-Forecast', urls_filename)

	with open(full_path, 'rb') as file:
		urls = pickle.load(file)
	return urls


def dump_urls(mountain_urls, urls_filename):
	""" Saves dictionary of mountain urls as a pickle file """

	full_path = os.path.join(
		'G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/Mountain-Forecast', urls_filename)

	with open(full_path, 'wb') as file:
		pickle.dump(mountain_urls, file)


def get_urls_by_elevation(url):
	""" Given a mountain url it returns a list or its urls by elevation """

	base_url = 'https://www.mountain-forecast.com/'
	full_url = urljoin(base_url, url)

	time.sleep(1)  # Delay to not bombard the website with requests
	page = requests.get(full_url)
	soup = bs(page.content, 'html.parser')

	try:
		elevation_items = soup.find('ul', attrs={
			'class': 'b-elevation__container'}).find_all('a', attrs={'class': 'js-elevation-link'})
	except AttributeError as e:
		elevation_items = soup.find('div', attrs={
			'class': 'b-elevation__container'}).find_all('a', attrs={'class': 'js-elevation-link'})

	return [urljoin(base_url, item['href']) for item in elevation_items]


def get_mountains_urls(urls_filename='mountains_urls.pickle', url='https://www.mountain-forecast.com/countries/United-States?top100=yes'):
	""" Returs dictionary of mountain urls

	If a file with urls doesn't exists then create a new one using "url" and return it
	"""
	if not any(slash in urls_filename for slash in ['/', '\\', '\\\\']):
		urls_filename = os.path.join(os.path.dirname(
			os.path.realpath(__file__)), urls_filename)
		print(urls_filename)
	if os.path.isfile(urls_filename):
		print('trying to load urls from {}'.format(urls_filename))
		mountain_urls = load_urls(urls_filename)
	else:
		directory_url = url

		page = requests.get(directory_url)
		soup = bs(page.content, 'html.parser')
		mtn_range = soup.find('h1').get_text()
		if mtn_range:
			if 'with' in mtn_range:
				print('Retrieving urls for the {}'.format(
					mtn_range.split(' with')[0]))
			else:
				print('Retrieving urls for the {}'.format(mtn_range.strip()))
		mountain_items = soup.find(
			'ul', attrs={'class': 'b-list-table'}).find_all('li')
		mountain_urls = {item.find('a').get_text(): get_urls_by_elevation(
			item.find('a')['href']) for item in mountain_items}
		print('dumping urls to {}'.format(urls_filename))
		dump_urls(mountain_urls, urls_filename)

	return mountain_urls


def clean(text):
	""" Returns a string with leading and trailing spaces removed """
	return re.sub('\W', '', text)
	# return re.sub('\s+', ' ', text).strip()  # Is there a way to use only REGEX?


def wind_speed(text):
	return re.search('\d', text).group()


def wind_direction(text):
	return re.search('[a-zA-Z]', text).group()


def save_data(rows, fname=None, savedir=''):
	""" Saves the collected forecasts into a CSV file

	If the file already exists then it updates the old forecasts
	as necessary and/or appends new ones.
	"""

	column_names = ['mountain', 'date', 'elevation', 'time', 'wind', 'summary', 'rain', 'snow',
					'max_temperature', 'min_temperature', 'chill', 'freezing_level', 'sunrise', 'sunset']

	today = date.today()
	# dataset_name = os.path.join('G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/Mountain-Forecast', '{:02d}{}_mountain_forecasts.csv'.format(today.month, today.year))  # i.e. 042019_mountain_forecasts.csv
	if savedir == '':
		if fname is None:
			dataset_name = os.path.join('G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/Mountain-Forecast',
										'{:02d}{}_mountain_forecasts.csv'.format(today.month, today.year))
		else:
			dataset_name = os.path.join('G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/Mountain-Forecast',
										'{:02d}{}{}_mountain_forecasts.csv'.format(today.month, today.year, fname))
	else:
		if not os.path.exists(savedir):
			os.makedirs(savedir)
		if fname is None:
			dataset_name = os.path.join(
				savedir, '{:02d}{}_mountain_forecasts.csv'.format(today.month, today.year))
		else:
			dataset_name = os.path.join(savedir, '{:02d}{}{}_mountain_forecasts.csv'.format(
				today.month, today.year, fname))
	try:
		new_df = pd.DataFrame(rows, columns=column_names)
		new_df['wind speed'] = new_df['wind'].apply(wind_speed)
		new_df['wind direction'] = new_df['wind'].apply(wind_direction)
		# new_df['wind speed'] = new_df['wind'].apply(
		# 	lambda x: int(x.split(' ')[0]))
		# new_df['wind direction'] = new_df['wind'].apply(
		# 	lambda x: x.split(' ')[1])
		old_df = pd.read_csv(dataset_name, dtype=object)
		old_df[6:8].replace('-', np.NaN, inplace=True)
		old_df[12:14].replace('-', np.NaN, inplace=True)
		old_cols = old_df.columns.values
		if 'wind speed' not in old_cols:
			old_df['wind speed'] = old_df['wind'].apply(wind_speed)
			old_df['wind direction'] = old_df['wind'].apply(wind_direction)
			# old_df['wind speed'] = old_df['wind'].apply(lambda x: int(x.split(' ')[0]))
			# old_df['wind direction'] = old_df['wind'].apply(lambda x: x.split(' ')[1])
		new_df.set_index(column_names[:4], inplace=True)
		old_df.set_index(column_names[:4], inplace=True)

		old_df.update(new_df)
		only_include = ~old_df.index.isin(new_df.index)
		combined = pd.concat([old_df[only_include], new_df], sort=False)

		combined.drop_duplicates(inplace=True)
		combined[2:4].replace('-', np.NaN, inplace=True)
		combined[8:10].replace('-', np.NaN, inplace=True)
		# combined[6:8].replace({'-',np.NaN},inplace=True)
		# combined[12:14].replace({'-',np.NaN},inplace=True)
		print('saving to {}'.format(dataset_name))
		combined.to_csv(dataset_name)

	except FileNotFoundError:
		new_df.replace({'-', np.NaN}, inplace=True)
		new_df[2:4].replace('-', np.NaN, inplace=True)
		new_df[8:10].replace('-', np.NaN, inplace=True)
		print('saving to {}'.format(dataset_name))
		new_df.to_csv(dataset_name, index=False)


def aggregate_sources():
	today = date.today()
	forecasts = glob.glob('G:/My Drive/GC Inventory Glory/Personal Work Folders/Robbie/Py/**/' +
						  '*mountain_forecasts.csv', recursive=True)
	so = pd.DataFrame
	scat = pd.DataFrame
	scar = pd.DataFrame
	casc = pd.DataFrame
	gorg = pd.DataFrame
	jac = pd.DataFrame
	wy = pd.DataFrame
	ha = pd.DataFrame
	print(forecasts)

	mtn_sources = {'sierra_others': so, 'sierra_cathedrals': scat, 'sierra_carsons': scar, 'cascades': casc,
				   'san bernadino': gorg, 'san jacinto': jac, 'wyoming': wy,
				   'hawaii': ha}
	for f in forecasts:
		date_string = datestring_from_file(f)
		if not str(today.year) in date_string:
			continue
		print('aggregating {}'.format(f))
		for k in mtn_sources.keys():
			if k in f:
				print('k: {} f: {}'.format(k, f))
				if mtn_sources[k].empty:
					mtn_sources[k] = pd.read_csv(f)
					print('{} is the first of {} '.format(f, k))
					break
				mtn_sources[k] = pd.concat(
					[pd.read_csv(f), mtn_sources[k]], sort=False).drop_duplicates()
				print('aggregated {} with {}'.format(f, k))
				break

	for k, v in mtn_sources.items():
		if not v.empty:
			v['date'] = pd.to_datetime(
				v['date'], format='%Y-%m-%d').dt.to_period(freq='M')
			dates = pd.unique(v['date'])
			if len(dates) > 1:
				for d in dates:
					# .to_csv(os.path.join(os.getenv("forecast_folder"),'{:02d}{}{}_mountain_forecasts.csv'.format(today.month, today.year,k)),index=False)
					v.loc[v['date'] == d, :].to_csv(os.path.join(os.getenv(
						"forecast_folder"), '{}{}_mountain_forecasts.csv'.format(d, k)), index=False)
			v.to_csv(os.path.join(os.getenv("forecast_folder"), 'Historical',
								  '{}{}_mountain_forecasts.csv'.format(max(dates).year, k)), index=False)


def archive_forecasts(testing=False):

	out = os.getenv("forecast_folder")
	hist = os.path.join(out, 'Historical')
	csvs = glob.glob(out+'/*.csv')
	today = datetime.today().date()
	if testing:
		csvs = csvs[:10]
	for file in csvs:
		if mod_time_from_path(file).date() == today:
			continue
		if testing:
			print(file, os.path.join(hist, os.path.basename(file)))
		else:
			try:
				os.rename(file, os.path.join(hist, os.path.basename(file)))
			except FileExistsError:
				os.remove(file)
				continue


def scrape(mountains_urls):
	""" Does the dirty work of scraping the forecasts for each mountain"""

	rows = []

	for mountain_name, urls in mountains_urls.items():

		for url in urls:

			# Request Web Page
			page = requests.get(url)
			soup = bs(page.content, 'html.parser')

			# Get data from header
			forecast_table = soup.find('table', attrs={
				'class': 'forecast__table forecast__table--js'})  # Default unit is metric
			days = forecast_table.find(
				'tr', attrs={'data-row': 'days'}).find_all('td')

			# Get rows from body
			times = forecast_table.find(
				'tr', attrs={'data-row': 'time'}).find_all('td')
			# 'img'  # Use "img" instead of "td" to get direction of wind
			winds = forecast_table.find(
				'tr', attrs={'data-row': 'wind'}).find_all('td')
			summaries = forecast_table.find(
				'tr', attrs={'data-row': 'summary'}).find_all('td')
			rains = forecast_table.find(
				'tr', attrs={'data-row': 'rain'}).find_all('td')
			snows = forecast_table.find(
				'tr', attrs={'data-row': 'snow'}).find_all('td')
			max_temps = forecast_table.find(
				'tr', attrs={'data-row': 'max-temperature'}).find_all('td')
			min_temps = forecast_table.find(
				'tr', attrs={'data-row': 'min-temperature'}).find_all('td')
			chills = forecast_table.find(
				'tr', attrs={'data-row': 'chill'}).find_all('td')
			freezings = forecast_table.find(
				'tr', attrs={'data-row': 'freezing-level'}).find_all('td')
			sunrises = forecast_table.find(
				'tr', attrs={'data-row': 'sunrise'}).find_all('td')
			sunsets = forecast_table.find(
				'tr', attrs={'data-row': 'sunset'}).find_all('td')
			# Iterate over days
			# for i, day in enumerate(days):
			for day in days:
				# print(day.__dict__)
				current_day = clean(day.get_text())
				elevation = url.rsplit('/', 1)[-1]
				num_cols = int(day['colspan'])
				# num_cols = int(day['data-columns'])

				if current_day != '':  # What if day is empty in the middle? Does it affect the count?

					# date = str(date(date.today().year, date.today().month, int(current_day.split(' ')[1])))  # Avoid using date format. Pandas adds 00:00:00 for some reason. Figure out better way to format
					try:
						date_ = datetime.strptime('{}{}'.format(
							date.today().year, current_day), '%Y%a%d').date()
					except:
						date_ = datetime.strptime('{}{}'.format(
							date.today().year, current_day), '%Y%A%d').date()
					# Iterate over forecast
					# for j in range(i, i + num_cols):
					for j in range(0, num_cols):

						time_cell = clean(times[j].get_text())
						wind = clean(winds[j].get_text())
						# wind = clean(winds[j]['alt'])
						summary = clean(summaries[j].get_text())
						rain = clean(rains[j].get_text())
						snow = clean(snows[j].get_text())
						max_temp = clean(max_temps[j].get_text())
						min_temp = clean(min_temps[j].get_text())
						chill = clean(chills[j].get_text())
						freezing = clean(freezings[j].get_text())
						sunrise = clean(sunrises[j].get_text())
						sunset = clean(sunsets[j].get_text())

						rows.append(np.array([mountain_name, date_, elevation, time_cell, wind, summary,
											  rain, snow, max_temp, min_temp, chill, freezing, sunrise, sunset]))

			time.sleep(1)  # Delay to not bombard the website with requests

	return rows


def scrape_forecasts():
	""" Call the different functions necessary to scrape mountain weather forecasts and save the data """

	start = time.time()
	print('\nGetting Mountain URLS')
	# mountains_urls = get_mountains_urls(urls_filename = '10_mountains_urls.pickle', url = 'https://www.mountain-forecast.com/countries/United-States')

	mountains_urls = get_mountains_urls(
		urls_filename='100_mountains_urls.pickle')
	print('URLs for {} Mountains collected\n'.format(len(mountains_urls)))

	print('Scraping forecasts...\n')
	forecasts = scrape(mountains_urls)

	print('Saving forecasts...\n')
	save_data(forecasts)

	print('All done! The process took {} seconds\n'.format(
		round(time.time() - start, 2)))


def scrape_list(url_dict={}):
	# url_dict = {'sierra_others':sierras_other_url,'sierra_cathedrals':sierras_cathedral_range_url,'sierra_carsons':sierras_carson_url,'cascades':cascade_range_url}
	"""
	An Extension of scrape_forecasts()
	Call the different functions necessary to scrape mountain weather forecasts and save the data specifically for the Eastside Sierras(Cathedrals)
	"""

	start = time.time()
	print('\nGetting Mountain URLS')
	# url_list = [sierras_other_url,sierras_cathedral_range_url,sierras_carson_url]
	if not url_dict:
		url_dict = {'sierra_others': sierras_other_url,
					'sierra_cathedrals': sierras_cathedral_range_url,
					'sierra_carsons': sierras_carson_url,
					'cascades': cascade_range_url, 'wyoming tetons': teton_range_url,
					'wyoming wind river range': wind_river_range_url, 'san bernadino': san_bernadino_url,
					'san jacinto': san_jacinto_url,
					'hawaii': hawaii_url}
	for k, v in url_dict.items():
		# for l in url_list:
		# fname = l+'.pickle'
		fname = k+'.pickle'
		mountains_urls = get_mountains_urls(urls_filename=fname, url=v)

		print('URLs for {} Mountains collected\n'.format(len(mountains_urls)))

		print('Scraping forecasts...\n')
		forecasts = scrape(mountains_urls)

		print('Saving forecasts...\n')
		'''
		Save data in separate files, as well?
		'''
		save_data(forecasts)
		save_data(forecasts, k, os.getenv("forecast_folder"))
	'''
	# mountains_urls = get_mountains_urls(urls_filename = '10_mountains_urls.pickle', url = 'https://www.mountain-forecast.com/countries/United-States')
	mountains_urls = get_mountains_urls(urls_filename = 'cathedral_range_mountains_urls.pickle', url = cathedral_range_url)
	# mountains_urls = get_mountains_urls(urls_filename = '100_mountains_urls.pickle')
	print('URLs for {} Mountains collected\n'.format(len(mountains_urls)))
	
	print('Scraping forecasts...\n')
	forecasts = scrape(mountains_urls)

	print('Saving forecasts...\n')
	save_data(forecasts)
	'''
	mins = (time.time() - start)/60
	secs = mins/60
	print('All done! The process took {} seconds\n'.format(
		round(time.time() - start, 2)))


def mod_time_from_path(path):
	return seconds_todatetime(os.path.getmtime(path))


def datestring_from_file(f, return_groups=False, pattern='', pattern_index=-1, test=False, find_all=False):
	'''
	f should be a string containing 1 date
	This is intended for searching file path's for dates
	'''
	patterns = [r'(\d{4}-\d{2}-\d{2})', r'(\d{4}[\.-]\d{2}[\.-]\d{2})', r'(\d{4}[\.-]\d{2})',
				r'(\d{2}[.-]\d{2}[.-]\d{4})',
				r'((?:\d{1,2})(?:(?:\/|-|\.)\d{1,2})(?:(?:\/|-|\.)\d{2,4}))',
				r'((?:\d{1,2}(?:-|\/))?\d{4})',
				r'((?:\d{,2}\s)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*(?:-|\.|\s|,)\s?\d{,2}[a-z]*(?:-|,|\s)?\s?\d{2,4})']
	if test:
		print('length of patterns {}'.format(len(patterns)))
		print('passed in arguments: {},return_groups={},pattern={},pattern_index={}'.format(
			f, return_groups, pattern, pattern_index))
	if not f:
		print('{} has to be a date containing string'.format(f))
		return None
	if not pattern:
		if pattern_index == -1:
			pattern_index += 1
		if pattern_index >= len(patterns):
			if test:
				print('pattern_index is too large')
			return None
		if test:
			print(pattern_index)
		pattern = patterns[pattern_index]
	if test:
		print('arguments at search: {},return_groups={},pattern={},pattern_index={}'.format(
			f, return_groups, pattern, pattern_index))
	if find_all:
		return re.findall(pattern, os.path.basename(f))
	dates = re.search(pattern, os.path.basename(f))
	if dates:
		if test:
			print(dates.groups())
		if return_groups:
			return dates.groups()
		return dates.group(0)
	pattern_index += 1
	return datestring_from_file(f, return_groups=return_groups, pattern_index=pattern_index)


def main(start_now=False, **kwargs):
	print('running')
	if start_now:
		if kwargs.get('aggregate', None):
			aggregate_sources()
		scrape_list()


def process_args():
	parser = argparse.ArgumentParser()
	parser.add_argument('-s', '--start_now', action='store_true')
	parser.add_argument('-a', '--aggregate', action='store_true')
	# parser.add_argument('--dry', action='store',type=str)
	args = parser.parse_args()
	return vars(args)


if __name__ == '__main__':
	'''
	Should probably expand this to a command line tool if it were to be something more
	https://www.mountain-forecast.com/mountain_ranges
	^^^ is the list of all Mountain Ranges with Forecasts
	Could present as a list or allow a specific peak or range to be searched and retrieved
	'''
	start_now = False
	main(**process_args())
	# if 'start' in arg_dict.keys():
	#     start_now = arg_dict['start']
	# main(start_now)
	# aggregate_sources()
	# scrape_list()

	# scrape_forecasts()
