##_____________________________
## import the desired libraries

import bs4 as bs
import datetime as dt
import matplotlib.pyplot as plt
from matplotlib import style
import numpy as np
import os
import pandas as pd
import pandas_datareader.data as web
import pickle
import requests
import time


##_______________________________________
## compiling dataset of s&p 500 companies

def compile_data():
    with open("sp500tickers.pickle", "rb") as f:                    # read the pickle file containing the tickers of the companies 
        tickers = pickle.load(f)

    main_df = pd.DataFrame()                                        # main_df is defined as a pandas dataframe

    for count, ticker in enumerate(tickers):
        df = pd.read_csv('stock_dfs/{}.csv'.format(ticker))         # reading the csv file for each ticker
        df.set_index('Date', inplace=True)

        df.rename(columns={'Adj Close': ticker}, inplace=True)
        df.drop(['Open', 'High', 'Low', 'Close', 'Volume'], 1, inplace=True)    # drop all the columns except adj. close

        if main_df.empty:
            main_df = df
        else:
            main_df = main_df.join(df, how='outer')

        if count % 10 == 0:
            print(count)
    print(main_df.head())                                           # dataframe is printed
    main_df.to_csv('sp500_joined_closes.csv')                       # dataframe is converted to csv file

##compile_data()
