import time
import requests
import pandas as pd
import numpy as np
import random

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys

from pyvirtualdisplay import Display



#
# # extract all parcel id from dataset downloaded from bc government data website
# pid_df = pd.read_csv('./property-addresses.csv', sep=';')
# pid_lst = pid_df[['SITE_ID']]
#
# # for building with invalid pid, use address to search in the website
#
# pid_df = pid_df.dropna(subset=['CIVIC_NUMBER'])
# pid_df['road'] = pid_df['STD_STREET'].apply(lambda x: x.replace('W ', '') + ' WEST' if 'W ' in x else x)
# pid_df['road'] = pid_df['road'].apply(lambda x: x.replace('E ', '') + ' EAST' if 'E ' in x else x)
# pid_df = pid_df.dropna(subset=['road'])
# pid_df['lon'] = pid_df['Geom'].apply(lambda x: x[x.find('[') + 1 : x.find(']')].split(',')[0])
# pid_df['lat'] = pid_df['Geom'].apply(lambda x: x[x.find('[') + 1 : x.find(']')].split(',')[1])
#
# pid_df['CIVIC_NUMBER'] = pid_df['CIVIC_NUMBER'].astype('int').astype('str')
# pid_df['full_add'] = pid_df['CIVIC_NUMBER'].str.cat(pid_df['road'], sep=' ')
#
#
# pid_df['full_add'] = pid_df['full_add'] + ' vancouver'
#
# pid_df.to_csv('./buildings.csv', index_label=0)

pid_df = pd.read_csv('./buildings.csv', index_col=0)
# pid_df = pid_df.loc[:25000, :]
# pid_df = pid_df.loc[25000:50000, :]
# pid_df = pid_df.loc[50000:75000, :]
pid_df = pid_df.loc[15000:30000, :]

pid_df = pid_df.dropna(subset=['SITE_ID'])



display = Display(visible=0, size=(800, 800))
display.start()


driver = webdriver.Chrome('./chromedriver')

for index, row in pid_df.iterrows():
    if index % 10 == 0:
        driver.quit()
        driver = webdriver.Chrome('./chromedriver')
    pid = row['SITE_ID']
    full_address = row['full_add']
    driver.get("https://www.bcassessment.ca/")
    if pid.isdigit():
        select = Select(driver.find_element_by_id('ddlSearchType'))
        select.select_by_visible_text('PID')
        driver.find_element_by_id('txtPID').send_keys(pid)
        driver.find_element_by_id('btnSearch').send_keys(Keys.ENTER)
    else:
        select = Select(driver.find_element_by_id('ddlSearchType'))
        select.select_by_visible_text('Civic address')
    
        driver.find_element_by_id('rsbSearch').send_keys(full_address)
        element = WebDriverWait(driver, 10).until(
            EC.visibility_of_element_located((By.CSS_SELECTOR, ".ui-menu-item-wrapper"))
        )
        element.click()

    time.sleep(random.uniform(1,3))
    page_source = driver.page_source
    soup = BeautifulSoup(page_source, 'lxml')
    try:
        pid_df.loc[index, 'actualy_address'] = soup.find(id='mainaddresstitle').contents[0]
    except:
        pid_df.loc[index, 'actualy_address'] = np.nan
    try:
        pid_df.loc[index, 'total_value'] = soup.find(id='lblTotalAssessedValue').contents[0]
    except:
        pid_df.loc[index, 'total_value'] = np.nan
    try:
        pid_df.loc[index, 'prev_value'] = soup.find(id='lblPreviousAssessedValue').contents[0]
    except:
        pid_df.loc[index, 'prev_value'] = np.nan
    try:
        pid_df.loc[index, 'built_year'] = soup.find(id='lblYearBuilt').contents[0]
    except:
        pid_df.loc[index, 'built_year'] = np.nan
    try:
        pid_df.loc[index, 'bedroom'] = soup.find(id='lblBedrooms').contents[0]
    except:
        pid_df.loc[index, 'bedroom'] = np.nan
    try:
        pid_df.loc[index, 'bathroom'] = soup.find(id='lblBathRooms').contents[0]
    except:
        pid_df.loc[index, 'bathroom'] = np.nan
    try:
        pid_df.loc[index, 'garage'] = soup.find(id='lblGarages').contents[0]
    except:
        pid_df.loc[index, 'garage'] = np.nan
    try:
        pid_df.loc[index, 'carport'] = soup.find(id='lblCarPorts').contents[0]
    except:
        pid_df.loc[index, 'carport'] = np.nan
    try:
        pid_df.loc[index, 'area'] = soup.find(id='lblLandSize').contents[0]
    except:
        pid_df.loc[index, 'area'] = np.nan
    print(pid_df.loc[index])


