# coding=utf-8
import urllib.request
from urllib.parse import quote
from bs4 import BeautifulSoup
import string
import re

class WikipediaScraper():
	def __init__(self):
		self.url = "https://fr.wikipedia.org/wiki"
		self.sqr_02 = "/Quartier_Thabor_-_Saint-Hélier_-_Alphonse_Guérin"
		self.sqr_05 = "/Quartier_Maurepas_-_Patton"
		self.sqr_06 = "/Quartier_Jeanne_d%27Arc_-_Longs-Champs_-_Atalante_Beaulieu"
		self.squares = [self.sqr_02, self.sqr_05, self.sqr_06]

	def __download_html(self, url):

		url = quote(url, safe=string.printable)
		response = urllib.request.urlopen(url, timeout=20)
		html = response.read().decode('utf-8')
		return html

	def __get_table_obj(self, html):
		bs_obj = BeautifulSoup(html, 'html.parser')
		# metro and bus information are inside <td><a> </a></<td>
		# we need to get the table tags because the attribute we are looking for is present in two different places
		return bs_obj.findAll('table')

	def __get_metro(self, html):
		metros = []
		table_objs = self.__get_table_obj(html)

		for table in table_objs:
			a_objs = table.findAll('a', href=re.compile("^(/wiki/)"))
			for a in a_objs:
				if a.has_attr('title'):
					if '(métro de Rennes)' in a['title']:
						metros.append(a['title'])
			return metros

	def __get_bus(self, html):
		buses = []
		table_objs = self.__get_table_obj(html)

		# The bus number is inside <b> tag
		for table in table_objs:
			# We need to search our <b> tag inside <tr> because there are two table which contain <b> tags
			#td_obj = table.findAll
			td_objs = table.findAll('td')
			for td in td_objs:	
				b_objs = td.findAll('b')
				for b in b_objs:
					buses.append(b.getText())
		return buses

	# Get surface in ha
	def __get_surface(self, html):
		surface = 0
		table_objs = self.__get_table_obj(html)

		for table in table_objs:
			tr_objs = table.findAll('tr')
			for tr in tr_objs:
				td_objs = table.findAll('td')
				for td in td_objs:
					if "km2" in td.getText() and "hab" not in td.getText() and surface is 0:
						surface = re.search('[0-9]+(,[0-9])', td.getText()).group(0)
		return surface

	'''
	This function extracts all the data from wikipedia corresponding to the squares defined in the __init__()
	'''
	def scrape(self):
		metros = []
		buses = []
		surfaces = []
		# get data for square 
		for sqr in self.squares:
			html = self.__download_html((self.url + sqr))
			metro = self.__get_metro(html)
			metros.append(metro)
			bus = self.__get_bus(html)
			buses.append(bus)
			surface = self.__get_surface(html)
			surfaces.append(surface)
			
		return {'surfaces': surfaces, 'buses': buses, 'metros': metros}

	
		
