{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from splinter import Browser\n",
    "\n",
    "def init_browser():\n",
    "    executable_path = {\"executable_path\": \"chromedriver.exe\"}\n",
    "    return Browser(\"chrome\", **executable_path, headless=True)\n",
    "\n",
    "def scrape ():\n",
    "    \"\"\"Scrapes various websites for information about Mars, and returns data in a dictionary\"\"\"\n",
    "    \n",
    "    browser = init_browser()\n",
    "    mars_data = {}\n",
    "\n",
    "    # visit the NASA Mars News site and scrape headlines\n",
    "    nasa_url = 'https://mars.nasa.gov/news/'\n",
    "    browser.visit(nasa_url)\n",
    "    time.sleep(1)\n",
    "    nasa_html = browser.html\n",
    "    nasa_soup = BeautifulSoup(nasa_html, 'html.parser')\n",
    "\n",
    "    news_list = nasa_soup.find('ul', class_='item_list')\n",
    "    first_item = news_list.find('li', class_='slide')\n",
    "    nasa_headline = first_item.find('div', class_='content_title').text\n",
    "    nasa_teaser = first_item.find('div', class_='article_teaser_body').text\n",
    "    mars_data[\"nasa_headline\"] = nasa_headline\n",
    "    mars_data[\"nasa_teaser\"] = nasa_teaser\n",
    "\n",
    "    # visit the JPL website and scrape the featured image\n",
    "    jpl_url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'\n",
    "    browser.visit(jpl_url)\n",
    "    time.sleep(1)\n",
    "    browser.click_link_by_partial_text('FULL IMAGE')\n",
    "    time.sleep(1)\n",
    "    try:\n",
    "        expand = browser.find_by_css('a.fancybox-expand')\n",
    "        expand.click()\n",
    "        time.sleep(1)\n",
    "\n",
    "        jpl_html = browser.html\n",
    "        jpl_soup = BeautifulSoup(jpl_html, 'html.parser')\n",
    "\n",
    "        img_relative = jpl_soup.find('img', class_='fancybox-image')['src']\n",
    "        image_path = f'https://www.jpl.nasa.gov{img_relative}'\n",
    "        mars_data[\"feature_image_src\"] = image_path\n",
    "    except ElementNotVisibleException:\n",
    "        image_path = 'https://www.jpl.nasa.gov/spaceimages/images/largesize/PIA22076_hires.jpg'\n",
    "        mars_data[\"feature_image_src\"] = image_path\n",
    "        \n",
    "    # visit the mars weather report twitter and scrape the latest tweet\n",
    "    mars_weather_url = 'https://twitter.com/marswxreport?lang=en'\n",
    "    browser.visit(mars_weather_url)\n",
    "    time.sleep(1)\n",
    "    mars_weather_html = browser.html\n",
    "    mars_weather_soup = BeautifulSoup(mars_weather_html, 'html.parser')\n",
    "\n",
    "    tweets = mars_weather_soup.find('ol', class_='stream-items')\n",
    "    mars_weather = tweets.find('p', class_=\"tweet-text\").text\n",
    "    mars_data[\"weather_summary\"] = mars_weather\n",
    "\n",
    "    # visit space facts and scrap the mars facts table\n",
    "    mars_facts_url = 'https://space-facts.com/mars/'\n",
    "    browser.visit(mars_facts_url)\n",
    "    time.sleep(1)\n",
    "    mars_facts_html = browser.html\n",
    "    mars_facts_soup = BeautifulSoup(mars_facts_html, 'html.parser')\n",
    "\n",
    "    fact_table = mars_facts_soup.find('table', class_='tablepress tablepress-id-mars')\n",
    "    column1 = fact_table.find_all('td', class_='column-1')\n",
    "    column2 = fact_table.find_all('td', class_='column-2')\n",
    "\n",
    "    facets = []\n",
    "    values = []\n",
    "\n",
    "    for row in column1:\n",
    "        facet = row.text.strip()\n",
    "        facets.append(facet)\n",
    "        \n",
    "    for row in column2:\n",
    "        value = row.text.strip()\n",
    "        values.append(value)\n",
    "        \n",
    "    mars_facts = pd.DataFrame({\n",
    "        \"Facet\":facets,\n",
    "        \"Value\":values\n",
    "        })\n",
    "\n",
    "    mars_facts_html = mars_facts.to_html(header=False, index=False)\n",
    "    mars_data[\"fact_table\"] = mars_facts_html\n",
    "\n",
    "    # scrape images of Mars' hemispheres from the USGS site\n",
    "    mars_hemisphere_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "    hemi_dicts = []\n",
    "\n",
    "    for i in range(1,9,2):\n",
    "        hemi_dict = {}\n",
    "        \n",
    "        browser.visit(mars_hemisphere_url)\n",
    "        time.sleep(1)\n",
    "        hemispheres_html = browser.html\n",
    "        hemispheres_soup = BeautifulSoup(hemispheres_html, 'html.parser')\n",
    "        hemi_name_links = hemispheres_soup.find_all('a', class_='product-item')\n",
    "        hemi_name = hemi_name_links[i].text.strip('Enhanced')\n",
    "        \n",
    "        detail_links = browser.find_by_css('a.product-item')\n",
    "        detail_links[i].click()\n",
    "        time.sleep(1)\n",
    "        browser.find_link_by_text('Sample').first.click()\n",
    "        time.sleep(1)\n",
    "        browser.windows.current = browser.windows[-1]\n",
    "        hemi_img_html = browser.html\n",
    "        browser.windows.current = browser.windows[0]\n",
    "        browser.windows[-1].close()\n",
    "        \n",
    "        hemi_img_soup = BeautifulSoup(hemi_img_html, 'html.parser')\n",
    "        hemi_img_path = hemi_img_soup.find('img')['src']\n",
    "\n",
    "        hemi_dict['title'] = hemi_name.strip()       \n",
    "        hemi_dict['img_url'] = hemi_img_path\n",
    "\n",
    "        hemi_dicts.append(hemi_dict)\n",
    "\n",
    "    mars_data[\"hemisphere_imgs\"] = hemi_dicts\n",
    "\n",
    "    browser.quit()\n",
    "\n",
    "    return mars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
