import os
import time as t
import argparse
import re
import json
import bs4
from bs4 import BeautifulSoup
from robobrowser import RoboBrowser
from dateutil.rrule import *
from dateutil.parser import *
from datetime import *

def main():

	# parse command line arguments
	parser = argparse.ArgumentParser(
		description=("Parse information out of HTML course search results"
		"from the UVic website. Relies on the exact file structure generated by"
		"'Scraper.py'. Parses most recent term by default."))

	parser.add_argument(
		"--indir",
		help="Input directory (default ./html/)", 
		default="./html/"
	)

	parser.add_argument(
		"--outdir", 
		help="Output directory (default ./json/)",
		default="./json/"
	)

	parser.add_argument(
		"--term",
		help=("First month of the term to parse as YYYYMM, "
		"e.g. 201901. Second term = 01, Summer Session = 05, First Term = 09.")
	)

	parser.add_argument(
		"--unprocessed",
		help="Skip processing of the scraped information into a more useful format?",
		type=bool, 
		default=False
	)

	parser.add_argument(
		"--all", 
		help="When true, parse everything in the input directory.",
		type=bool,
		default=False
	)
	args = parser.parse_args()

	# get the directory containing the most recently scraped course search results
	revision = max([int(revision) for revision in os.listdir(args.indir)])
	revision_dir = os.path.join(args.indir, str(revision))

	# get a list of folders containing html search results for each term
	term_dirs = []

	if args.all:
		term_dirs.extend([os.path.join(revision_dir, term) for term in os.listdir(revision_dir)])

	elif args.term:
		term_dirs.append(os.path.join(revision_dir, args.term))

	else:
		most_recent_term = max([int(term) for term in os.listdir(revision_dir)])
		term_dirs.append(os.path.join(revision_dir, str(most_recent_term)))

	# reverse term_dirs so we start with the more recent terms
	term_dirs.reverse()

	# parse data for each term
	for term_dir in term_dirs:

		# extract the term from the current file path (I know this sucks)
		term = os.path.split(term_dir)[1]
		course_files = [os.path.join(term_dir, course_file) for course_file in os.listdir(term_dir)]
		courses = []

		for course_file in course_files:
			print(f"Parsing '{course_file}'")

			# parse course data
			with open(course_file) as file:
				try:
					html = file.read()
				except Exception as e:
					print("Error while reading course data. ", e)

			courses += parseSearchResult(html, term)

		# reformat course data
		courses = courses if args.unprocessed else convert(courses)

		# save the data in json format
		os.makedirs(args.outdir, exist_ok=True)
		filename = os.path.join(args.outdir, f"{term}.json")
		with open(filename, 'w') as file:
			try:
				file.write(json.dumps(courses, indent=4))
				print(f"Saved course data as '{filename}.'")
			except Exception as e:
				print("Error while saving course data. ", e)

	# end of main function
	return

'''
Converts the weekday abbreviations used by UVic (M, T, W, R, F, S, U). UVic will combine
these abbreviations to indicate multiple days e.g. TWR means Tuesday, Wednesday, and
Thursday.
'''
def convertWeekAbbr(abbreviation):

	weekday_consts = {
		'M': MO,
		'T': TU,
		'W': WE,
		'R': TH,
		'F': FR,
		'S': SA,
		'U': SU
	}

	return [weekday_consts[char] for char in abbreviation if char in weekday_consts]

'''
Converts scraped course data into a more useful format e.g. by using RRULEs
'''
def convert(courses):

	# browser for following catalog entry links
	browser = RoboBrowser(parser='html.parser')

	for course in courses:

		# remove term string since it can be generated from the term
		del course['term_string']
		print(f"reformatting '{course['course_title']}'")

		for i, section in enumerate(course['sections']):

			# associate calendar entry with course rather than section
			# check for fucked links like "https://www.uvic.cahttp://web.uvic.ca/soci/courses_spring.php"
			"""
			if i == 0:
				try:
					browser.open(section['catalog_entry'])
					catalog_entry = str(browser.parsed())
					soup = BeautifulSoup(catalog_entry, 'html.parser')
					calendar_entry_link = soup.find('a', text="entry")

					if (calendar_entry_link):
						course['calendar_entry'] = calendar_entry_link["href"]

					course['catalog_entry'] = section['catalog_entry']

					# wait for a second to avoid rate limiting
					t.sleep(1)

				except Exception:
					print(f"invalid link: {section['catalog_entry']}")
			"""

			del section['catalog_entry']

			# create an rrule for each meeting time and remove redundant information
			# the dtstart time will be used as the start time for all recurrences
			for meeting in section['meeting_times']:

				# separate start and end times

				if (not 'TBA' in [
					meeting['time'],
					meeting['days'],
					meeting['start_date'],
					meeting['end_date']]):

					# parse datetime objects from "time" field
					meeting_time_split = meeting["time"].split(" - ")
					meeting["start_time"] = datetime.strptime(meeting_time_split[0], "%I:%M %p")
					meeting["end_time"] = datetime.strptime(meeting_time_split[1], "%I:%M %p")

					# calculate meeting duration in seconds
					timediff = meeting["end_time"] - meeting["start_time"]
					meeting["duration"] = timediff.seconds

					# include start time in "start_date" datetime object
					meeting["start_date"] = datetime.combine(
						datetime.fromisoformat(meeting["start_date"]),
						meeting["start_time"].time()
					).isoformat()

					# convert "end_date" to a date and start/end times to time
					meeting["end_date"] = parse(meeting["end_date"]).date().isoformat()
					meeting["start_time"] = meeting["start_time"].time().isoformat()
					meeting["end_time"] = meeting["end_time"].time().isoformat()

					# create an rrule from the above information
					meeting_rule = rrule(
						WEEKLY,
						wkst=SU,
						byweekday=(convertWeekAbbr(meeting["days"])),
						dtstart=parse(meeting["start_date"]),
						until=parse(meeting["end_date"])
					)
					meeting["rrule"] = str(meeting_rule)

				else:
					meeting["start_time"] = "TBA"
					meeting["end_time"] = "TBA"
					meeting["duration"] = "TBA"
					meeting["rrule"] = "TBA"

				# these fields are now redundant
				del meeting["time"]
				del meeting["days"]
				del meeting["start_date"]
				del meeting["end_date"]

	return courses

'''
Returns a list of dictionaries containing information for each course listed in the given HTML file.

Params:
	html - an HTML document containing search results from UVic's course search webpage.
	term - the term associated with the courses in the HTML document.
'''

def parseSearchResult(html, term):

	# read current html file into beautifulsoup
	soup = BeautifulSoup(html, 'html.parser')

	# create a new data structure for course information
	courses = []

	# find the all the headers for all of the search results
	tables = soup.find("table", class_="datadisplaytable")
	if not tables:
		return courses

	for table_header in tables.find_all("th", class_="ddtitle"):

		# construct a dictionary with default values to store information about this course listing
		course = {
			"course_title": "",
			"subject": "",
			"course_code": "",
			"term": term,
			"term_string": "",
			"levels": "",
			"sections": [
				{
					"crn": "",
					"description": "",
					"section_type": "",
					"section_number": "",
					"reg_start": "",
					"reg_end": "",
					"attributes": "",
					"campus": "",
					"schedule_type_alt": "",
					"method": "",
					"credits": "",
					"catalog_entry": "",
					"meeting_times": [],
				}
			]
		}

		# isolate a string of the course listing header
		header_contents = table_header.a.contents[0]

		# remove dashes in course title if there are too many
		hyphen_count = header_contents.count(' - ')
		if hyphen_count > 3:
			header_contents = header_contents.replace(' - ', '', hyphen_count - 3)

		# split the header by dashes (four elements)
		header_data = header_contents.split(' - ')

		# transform the header data into the database format
		course['course_title'] = header_data[0] # course title
		course['sections'][0]['crn'] = header_data[1] # crn
		course['subject'] = header_data[2].split(' ')[0] # subject eg. chem
		course['course_code'] = header_data[2].split(' ')[1] # course code
		course['sections'][0]['section_type'] = header_data[3][0] # section type (a, b, or t)
		course['sections'][0]['section_number'] = header_data[3][1:] # section number

		# find next <tr> after table header's parent <tr> which contains the rest of the course info
		row_after = table_header.parent.find_next("tr").td

		# get the course description
		description = row_after.contents[0].strip()
		course['sections'][0]['description'] = description if len(description) > 0 else "No Description"

		# get the catalog entry
		catalog_entry_a_tag = row_after.find('a')
		if catalog_entry_a_tag:
			course['sections'][0]['catalog_entry'] = f"https://www.uvic.ca{catalog_entry_a_tag['href']}"

		# create a list of strings from the information displayed below the header and above the catalog link, "middle info"
		middle_info = []
		for string in row_after.stripped_strings:
			middle_info.append(string)

		# remove meeting times table and catalog link from list
		for i in range(len(middle_info)):
			if middle_info[i] == "View Catalog Entry":
				middle_info = middle_info[:i]
				break

		# get the fields that are *maybe* there
		for i in range(len(middle_info) - 1):

			if middle_info[i] == "Registration Dates:":

				# split the given registration date range into two ISO 8601 date strings
				split_dates = middle_info[i + 1].split(" to ")

				if len(split_dates) == 2:
					course['sections'][0]['reg_start'] = datetime.strptime(split_dates[0], "%b %d, %Y").isoformat()
					course['sections'][0]['reg_end'] = datetime.strptime(split_dates[1].strip(), "%b %d, %Y").isoformat()

			elif middle_info[i] == "Associated Term:":
				course['term_string'] = middle_info[i + 1]

			elif middle_info[i] == "Levels:":
				course['levels'] = middle_info[i + 1]

			elif middle_info[i] == "Attributes:":
				course['sections'][0]['attributes'] = middle_info[i + 1]


		# get last four elements of the middle info (credits, instructional method, schedule type, campus)
		if len(middle_info) >= 4:
			course['sections'][0]['campus'] = middle_info[-4]
			course['sections'][0]['schedule_type_alt'] = middle_info[-3]
			course['sections'][0]['method'] = middle_info[-2]
			course['sections'][0]['credits'] = middle_info[-1]



		# get <tr> elements from the "scheduled meeting times" table with a CSS selector
		meeting_time_rows = row_after.select("table > tr")[1:]

		# we will store information about each meeting time for this course in a list of dictionaries
		meetings = []

		# if there are scheduled meeting times (1 per row), add their details to data2
		for row in meeting_time_rows:

			# assign default values to the rowdata array
			rowdata = [""] * 6

			# iterate through all of the data in the current <tr>, except instructors
			tds = row.select("td")
			for i in range(0, len(tds) - 1):

				# get the contents of the tag
				entry = tds[i].string

				# if entry is blank, and <abbr> tag or a special character, replace with TBA
				if isinstance(entry, bs4.element.Tag) or entry == "\xa0":
					entry = "TBA"

				# add the collected information to list of information on the current row
				rowdata[i] = entry

			# parse the instructors column, collecting name/email pairs in JSON format from each <a> tag
			instructor_links = tds[6].find_all("a")
			instructor_data = [{"name": "TBA", "email": "TBA"}]

			if len(instructor_links) != 0:
				instructor_data = []
				for instructor_link in instructor_links:
					instructor_data.append({"name": instructor_link["target"], "email": instructor_link["href"].split(":")[1]})

			# split the given date range ex. "Jan 07, 2019 - Apr 05, 2019" into two ISO 8601 date strings
			split_dates = rowdata[4].split(" - ")
			start_date = ""
			end_date = ""

			if len(split_dates) == 2:
				start_date = datetime.strptime(split_dates[0].strip(), "%b %d, %Y").isoformat()
				end_date = datetime.strptime(split_dates[1].strip(), "%b %d, %Y").isoformat()

			# assign default values to the meeting dictionary
			meeting = {
				"type":			 rowdata[0],
				"time":		 	 rowdata[1],
				"days":		 	 rowdata[2],
				"location": 	 rowdata[3],
				"start_date":	 start_date,
				"end_date":		 end_date,
				"schedule_type": rowdata[5],
				"instructors": 	 instructor_data,
			}

			# add ths meeting time to the list of meeting times for the current course
			meetings.append(meeting)

		# add the current course's meetings to the list of all course meetings
		course['sections'][0]['meeting_times'] = meetings

		# check if another section of this course has been parsed already, if it has been,
		# add this section to that previous course's sections list.
		already_found = False
		for previous_course in courses:
			if previous_course['course_title'] == course['course_title']:
				previous_course['sections'].append(course['sections'][0])
				already_found = True

		# add the current course's data to the list of all course data
		if already_found == False:
			courses.append(course)

	# return from function
	print(f"Identified {len(courses)} course listing(s).")
	return courses

# if this file isn't being imported, run automatically
if __name__ == "__main__":
    main()
