from datetime import date, timedelta
import requests

from bs4 import BeautifulSoup
import pandas as pd
from IPython.display import clear_output
from tqdm import tqdm

TICKER_EDGAR_CIK_CSV_PATH = 'ticker_and_edgar_cik.csv'
TICKER_EDGAR_CIK = None
XLSX_OUTPUT_FILENAME = 'insider_trading_output.xlsx'


def main():
    """Pulls insider trading statistics"""
    #TODO: Switch on arg to determine which csv to use
    ticker_csv = pd.read_csv(TICKER_EDGAR_CIK_CSV_PATH, delimiter=',')
    symbols = [i.upper() for i in ticker_csv.Ticker]

    #TODO: Switch on arg to determine the start date
    start_date = str(date.today() - timedelta(days=14))
    dfs = []
    with tqdm(total=len(symbols)) as pbar:
        for i in range(len(symbols)):
            try:
                lst = [symbols[i]]
                cik = symbol_to_cik(lst)
                page = 0
                beg_url = 'https://www.sec.gov/cgi-bin/own-disp?action=getissuer&CIK=' + str(
                    cik[0]) + '&type=&dateb=&owner=include&start=' + str(page * 80)
                urls = [beg_url]
                df_data = []
                for url in urls:
                    soup = to_soup(url)
                    transaction_report = soup.find('table', {'id': 'transaction-report'})

                    t_chil = [i for i in transaction_report.children]
                    t_cont = [i for i in t_chil if i != '\n']

                    headers = [i for i in t_cont[0].get_text().split('\n') if i != '']
                    data_rough = [i for lst in t_cont[1:] for i in lst.get_text().split('\n') if i != '']
                    data = [data_rough[i:i + 12] for i in range(0, len(data_rough), 12)]
                    last_line = data[-1]
                    for j in data:
                        if start_date > j[1]:
                            break
                        else:
                            if j != last_line:
                                df_data.append(j)
                            else:
                                df_data.append(j)
                                page += 1
                                urls.append('https://www.sec.gov/cgi-bin/own-disp?action=getissuer&CIK=' + str(
                                    cik[0]) + '&type=&dateb=&owner=include&start=' + str(page * 80))

                df = pd.DataFrame(df_data, columns=headers)
                df['Purch'] = pd.to_numeric(df['Acquistion or Disposition'].apply(lambda x: 1 if x == 'A' else 0)
                                            * df['Number of Securities Transacted'])
                df['Sale'] = pd.to_numeric(df['Acquistion or Disposition'].apply(lambda x: 1 if x == 'D' else 0)
                                           * df['Number of Securities Transacted'])
                purch = df['Acquistion or Disposition'] == 'A'
                sale = df['Acquistion or Disposition'] == 'D'
                num_purch = len(df[purch])
                num_sale = len(df[sale])
                total_purch = int(df['Purch'].sum(skipna=True))
                total_sale = int(df['Sale'].sum(skipna=True))
                avg_purch = int(total_purch / num_purch)
                avg_sale = int(total_sale / num_sale)
                ratio = round(num_purch / num_sale, 2)
                new_df = pd.DataFrame({'Symbol': lst[0],
                                       'Purchases': num_purch,
                                       'Sales': num_sale,
                                       'Buy/Sell Ratio': ratio,
                                       'Total Bought': f'{total_purch:,}',
                                       'Total Sold': f'{total_sale:,}',
                                       'Avg Shares Bought': f'{avg_purch:,}',
                                       'Avg Shares Sold': f'{avg_sale:,}'},
                                      index=[0])

                new_df.set_index('Symbol', inplace=True)
                dfs.append(new_df)
                pbar.update(1)
            except:
                pbar.update(1)
                continue

    clear_output(wait=True)
    print('SCAN COMPLETE for period beginning: ' + start_date)
    combo = pd.concat(dfs)

    combo.to_excel(XLSX_OUTPUT_FILENAME, index=True)

    return combo.sort_values('Buy/Sell Ratio', ascending=False).head(100)


def handle_args():
    #TODO: Implement this
    pass


def symbol_to_cik(symbols):
    """Returns a list of CIK numbers corresponding to their given symbol"""
    ticker_cik = pd.read_csv(TICKER_EDGAR_CIK_CSV_PATH, delimiter=',')
    df = pd.DataFrame(ticker_cik)
    df.set_index('Ticker', inplace=True)
    new_symbols = [i.lower() for i in symbols]
    cik = [df.loc[i, 'CIK'] for i in new_symbols]
    return cik


def cik_to_symbol(ciks):
    """Returns a list of symbols corresponding to their given CIK numbers"""
    ticker_cik = pd.read_csv(TICKER_EDGAR_CIK_CSV_PATH, delimiter=',')
    df = pd.DataFrame(ticker_cik)
    df.set_index('CIK', inplace=True)
    df = df[~df.index.duplicated(keep='first')]
    tickers = [df.loc[i, 'Ticker'] for i in ciks]
    new_tickers = [i.upper() for i in tickers]
    return new_tickers


def to_soup(url):
    """Turns URL into Soup object"""
    url_response = requests.get(url)
    webpage = url_response.content
    soup = BeautifulSoup(webpage, 'html.parser')
    return soup


if __name__ == '__main__':
    main()
