import requests
import time
from bs4 import BeautifulSoup
from csv import writer


URL = 'https://www.imdb.com/list/ls066222599/?st_dt=&mode=detail&page=1&sort=list_order,asc'

# User agent - google User agent and paste yours into here
urlHeaders = {"Paste User agent"}
page = requests.get(URL, headers=urlHeaders)

soup = BeautifulSoup(page.content, 'html.parser')

# Finds all posts
posts = soup.find_all(class_='lister-item mode-detail')

# Opens Csv file to write into
with open('posts.csv', 'w') as csv_file:
    csv_writer = writer(csv_file)
    headers = ['Title', 'Rating', 'StarRating', 'Genre', 'Runtime', 'Link']
    csv_writer.writerow(headers)

        # Iterates through posts to find specific text then writes into csv file
    for post in posts:
        try:
            Title = post.find(class_='lister-item-header').get_text().replace('\n', ' ')
        # site = 'https://www.imdb.com'
            link = post.find('a')['href']
            href = 'https://www.imdb.com' + link
            genre = post.find(class_='genre').get_text().replace('\n', '').replace(' ', '').replace(',', ', ')
            rating = post.find(class_='certificate').get_text()
            starRating = post.find(class_='ipl-rating-star small').get_text().replace('\n', '')
            runtime = post.find(class_='runtime').get_text()
            csv_writer.writerow([Title, rating, starRating, genre, runtime, href])
        except Exception as ex:
		#Somes post don't include everything but they all have Title, genre, href
            csv_writer.writerow([Title, '', '', genre, '', href])   






