#!/usr/bin/env python
# Name:
# Student number:
"""
This script scrapes IMDB and outputs a CSV file with highest rated movies.
"""

import csv
from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from bs4 import BeautifulSoup
import pandas as pd
import re

TARGET_URL = "https://www.imdb.com/search/title?title_type=feature&release_date=2008-01-01,2018-01-01&num_votes=5000,&sort=user_rating,desc"
BACKUP_HTML = 'movies.html'
OUTPUT_CSV = 'movies.csv'


def extract_movies(dom):
    """
    Extract a list of highest rated movies from DOM (of IMDB page).
    Each movie entry should contain the following fields:
    - Title
    - Rating
    - Year of release (only a number!)
    - Actors/actresses (comma separated if more than one)
    - Runtime (only a number!)
    """

    # Create objects containing the information needed
    title_and_year_object = dom.find_all('h3')
    ratings_object = dom.find_all('strong')
    minutes_object = dom.find_all("span", {"class" :"runtime"})

    # Delete things we dont need
    del ratings_object[:2]
    del title_and_year_object[-1]

    #m make a list of the text of the object
    years_list   = [item.text[-6:-2] for item in title_and_year_object]
    titles_list  = [item.a.text for item in title_and_year_object]
    ratings_list = [item.text for item in ratings_object]
    minutes_list = [item.text for item in minutes_object]

    # make an object of the stars and directors
    stars_list = []
    stars_object = dom.find_all('p', {"class": ""})
    #loop over the starts
    for star in stars_object:
        #Create a list of each word
        star = star.text.split()
        # remove the director and 'Stars: ' and append to list
        while 'Stars:' in star:
            del star[0]
        star = " ".join(star)
        star = star.split(';')
        stars_list.append(star)

    # Create a dataframe with the collected values.
    df = pd.DataFrame()
    df['Title'] = titles_list
    df['Rating'] = ratings_list
    df['Year'] = years_list
    df['Actors'] = stars_list
    df['Runtime'] = minutes_list
    return df

    
def save_csv(movies):
    """
    Output a CSV file containing highest rated movies.
    """

    #output the df to csv
    movies.to_csv(OUTPUT_CSV)


def simple_get(url):
    """
    Attempts to get the content at `url` by making an HTTP GET request.
    If the content-type of response is some kind of HTML/XML, return the
    text content, otherwise return None
    """
    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None
    except RequestException as e:
        print('The following error occurred during HTTP GET request to {0} : {1}'.format(url, str(e)))
        return None


def is_good_response(resp):
    """
    Returns true if the response seems to be HTML, false otherwise
    """
    content_type = resp.headers['Content-Type'].lower()
    return (resp.status_code == 200
            and content_type is not None
            and content_type.find('html') > -1)


if __name__ == "__main__":

    # get HTML content at target URL
    html = simple_get(TARGET_URL)

    # save a copy to disk in the current directory, this serves as an backup
    # of the original HTML, will be used in grading.
    with open(BACKUP_HTML, 'wb') as f:
        f.write(html)

    # parse the HTML file into a DOM representation
    dom = BeautifulSoup(html, 'html.parser')

    # extract the movies (using the function you implemented)
    movies = extract_movies(dom)

    # write the CSV file to disk (including a header)
    save_csv(movies)
