from selenium import webdriver
from bs4 import BeautifulSoup
import requests
import time
import csv

START_URL='https://exoplanets.nasa.gov/discovery/exoplanet-catalog/'
browser = webdriver.Chrome('./python/chromedriver')
browser.get(START_URL)
time.sleep(10)
headers = ["name", "light_years_from_earth", "planet_mass", "stellar_magnitude", "discovery_date", "hyperlink", "planet_type", "planet_radius", "orbital_radius", "orbital_period", "eccentricity"]
planet_data = []
new_planet_data = []


def scrape():
    for i in range(1,443):
        while True:
            time.sleep(2)
            soup = BeautifulSoup(browser.page_source, 'html.parser')
            current_page_number = int(soup.find_all('input', attrs={'class', 'page_num'})[0].get('value'))
            if current_page_number < i:
                browser.find_element_by_xpath('//*[@id="primary_column"]/footer/div/div/div/nav/span[2]/a').click()
            elif current_page_number > i:
                browser.find_element_by_xpath('//*[@id="primary_column"]/footer/div/div/div/nav/span[1]/a').click()
            else: 
                break
        for ul_tags in soup.find_all('ul', attrs={'class', 'exoplanet'}):
            li_tags = ul_tags.find_all('li')
            temp_list = []
            for index, li_tag in enumerate(li_tags):
                if index == 0:
                    temp_list.append(li_tag.find_all('a')[0].contents[0])
                else:
                    try:
                        temp_list.append(li_tag.contents[0])
                    except:
                        temp_list.append('')
            hyper_link_tag = li_tags[0]
            temp_list.append("https://exoplanets.nasa.gov"+hyper_link_tag.find_all("a", href=True)[0]["href"])   
            planet_data.append(temp_list)    
        browser.find_element_by_xpath('//*[@id="primary_column"]/footer/div/div/div/nav/span[2]/a').click()
        print(f'{i} page done of first page !!!')


def scrape_more_data(hyper_link):
    try:
        page = requests.get(hyper_link)
        soup = BeautifulSoup(page.content, 'html.parser')
        temp_list = []
        for tr_tag in soup.find_all('tr', attrs={'class': 'fact_row'}):
            td_tags = tr_tag.find_all('td')
            for td_tag in td_tags:
                try:
                    temp_list.append(td_tag.find_all('div', attrs={'class': 'value'})[0].contents[0])
                except:
                    temp_list.append('')
        new_planet_data.append(temp_list)
        
    except:
        time.sleep(1)
        scrape_more_data(hyper_link)


scrape()

for i,data in enumerate(planet_data):
    scrape_more_data(data[5])
    print(f'{i+1} page done of second page !!!')

final_planet_data = []

for i,data in enumerate(planet_data):
    new_planet_data_elements = new_planet_data[i]
    new_planet_data_elements = [elem.replace('\n', '') for elem in new_planet_data_elements]
    new_planet_data_elements = new_planet_data_elements[:7]
    final_planet_data.append(data + new_planet_data_elements)

with open('final_data.csv', 'w') as f:
    csv_writer = csv.writer(f)
    csv_writer.writerow(headers)
    csv_writer.writerow(final_planet_data)