import urllib.request
import json
import time
import requests
import pandas as pd
from datetime import datetime, timedelta
import os

API_KEY = ''
HISTORICAL_FILENAME = 'historicalData'
SAVE_FOLDER = 'data'

def convert_unix_to_time_of_open_bar(unix, time_delta_mins, MINS_TO_ROUND):
    ''' input:1612553002, 0, 60 (a time of 2/5/21 @ 1:23pm)
    output: 20210205 130000'''
    now = datetime.fromtimestamp(unix) + timedelta(minutes=time_delta_mins)
    rounded_mins = int(((now.minute + now.second / 60) // MINS_TO_ROUND) * MINS_TO_ROUND)
    return (datetime(now.year, now.month, now.day, now.hour, rounded_mins) + timedelta(minutes=0)).strftime('%Y%m%d %H%M%S') # %Y%m%d%H%M%S'

# print(convert_unix_to_time_of_open_bar(1612553002, 0, 60))
# exit()

def datetime_to_unix(datetime_input_as_yyyymmdd):
    # from datetime import datetime
    time_s = str(datetime_input_as_yyyymmdd)
    dt = datetime(int(time_s[0:4]), int(time_s[4:6]), int(time_s[6:8]), int(time_s[8:10]), int(time_s[10:12]), int(time_s[12:14]))
    return dt.timestamp()

def save_csv(data, fileName, folderName='', header=False, write_append='w'):
    import pandas as pd
    if folderName[-1:] != '\\' and len(folderName) > 0 and folderName[-4:] != '.csv':
        folderName += '\\'
    filePath = str(folderName) + str(fileName) + '.csv'
    if folderName[-4:] == '.csv':
        filePath = folderName
    else:
        import os
        if not os.path.exists(str(folderName) + "/"):
            os.makedirs(str(folderName) + "/")
    df = pd.DataFrame(data)
    df.to_csv(filePath, index=False, header=header, mode=write_append)

def get_html_recur(url, attempt, attemptLimit=30):
    if attempt == attemptLimit:
        print('could not get data for', url)
        return -1
    try:
        response = requests.get(url, timeout=10) # urlopen(url, timeout=12)
        html = response#.json()
        response.close()
        # print('success')
        return html
    except:
        time.sleep(5)
        print('failed to get data for', url, ' trying again')
        return get_html_recur(url, (attempt + 1), attemptLimit)


# def datetime_UTC_to_CST(datetime_input_as_yyyymmdd, timezone='UTC'):
#     import pytz
#     time_s = str(datetime_input_as_yyyymmdd)
#     dt = datetime.datetime(int(time_s[0:4]), int(time_s[4:6]), int(time_s[6:8]), 0, 0, 0)
#     print(dt.timestamp())
#     timestamp_CST = pytz.timezone("UTC").localize(dt).astimezone(pytz.timezone('US/Central'))
#     return str(timestamp_CST).replace(' ', '').replace(':', '').replace('-', '')[0:14]

# cur_dt = datetime(2021, 2, 5, 21, 40, 0)
# print(int(convert_unix_to_time_of_open_bar(time.time(), 60, 60).replace(' ', '')), int(cur_dt.strftime('%Y%m%d%H%M%S')))
# exit()


# df = pd.read_clipboard()
# tickers = list(df['Ticker'])
# closes = list(df['Close'])
# cap = list(df['MktCap'])
# dict1 = {tickers[i]: int(float(cap[i]) / float(closes[i])) for i in range(len(tickers))}
# mat = [[coin, dict1[coin]] for coin in dict1.keys()]
# df = pd.DataFrame(mat)
# # df.to_csv
# df.to_clipboard()
# print(tickers)
# exit()

def get_historical_data(interval='hour_or_day'):#, data_points=2):
    ''' '''
    df_desiredCryptos = pd.read_csv('Crypto List(binance).csv')
    coins = list(df_desiredCryptos['Ticker'])
    start_dates = list(df_desiredCryptos['StartDate'])
    coin_start_date = {coins[i]: str('20' + str(start_dates[i])) for i in range(len(coins))}
    coin_to_supply = {coins[i]: int(list(df_desiredCryptos['Supply'])[i]) for i in range(len(coins))}
    try:
        csv = pd.read_csv(SAVE_FOLDER + '\\' + HISTORICAL_FILENAME + interval + '.csv')
    except:
        save_csv([['Ticker', 'Date(YMD)', 'Open', 'High', 'Low', 'Close', 'Vol', 'Time', 'MktCap', 'SocialScore', 'SentimentRatio', 'SocialEngagement']], fileName=HISTORICAL_FILENAME + interval, folderName=SAVE_FOLDER, header=False, write_append='a')
    mat = []
    cur_dt = datetime(2019, 10, 1, 0, 0, 0)
    # while current time is > 60min of opening bar (assuming hourly data), grab next data point
    while int(cur_dt.strftime('%Y%m%d%H%M%S')) < int(convert_unix_to_time_of_open_bar(time.time(), 0, 60).replace(' ', '')):
        open_time = int(datetime_to_unix(cur_dt.strftime('%Y%m%d%H%M%S')))
        cur_dt += + timedelta(minutes=60)
        close_time = int(datetime_to_unix(cur_dt.strftime('%Y%m%d%H%M%S')))
        coins = [coin for coin in coin_start_date.keys() if int(coin_start_date[coin]) < int(cur_dt.strftime('%Y%m%d'))]
        # coins = ['ETH', 'XRP', 'DOGE']
        coins_joined = ','.join(coins)
        data_points = len(coins)
        t1 = time.time()
        url = 'https://api.lunarcrush.com/v2?' + 'data=assets&key=' + API_KEY + '&symbol=' + coins_joined + '&interval=' + interval + '&interval=' + interval + '&start=' + str(open_time) + '&end=' + str(close_time) + '&data_points=' + str(data_points)  # + '&time_series_indicators=' + 'open,high,low,close,average_sentiment,galaxy_score'
        data = get_html_recur(url, 0, attemptLimit=30).json()
        # print(time.time() - t1)
        print('datetime:', cur_dt.strftime('%Y%m%d%H%M%S'))
        for section in data['data']:
            coin = section['symbol']
            # mkt_cap = section['market_cap']
            for row in section['timeSeries']:
                try:
                    print(coin, row)
                    mkt_cap = int(coin_to_supply[coin] * float(row['close']) / 1000)
                    social_metrics, social_engagement = ['reddit_posts_score', 'reddit_comments_score', 'tweet_retweets', 'tweet_favorites'], 0
                    for j in range(4):
                        try:
                            soc = int(row[social_metrics[j]])
                            social_engagement += soc
                        except: pass
                    try:
                        percent_social_sentiment = float(int(row["tweet_sentiment_impact4"]) + int(row["tweet_sentiment_impact5"])) / (int(row["tweet_sentiment_impact1"]) + int(row["tweet_sentiment_impact2"]) + int(row["tweet_sentiment_impact4"]) + int(row["tweet_sentiment_impact5"]))
                    except:
                        percent_social_sentiment = .5
                        try:
                            for row in reversed(mat):
                                if row[0] == coin:
                                    percent_social_sentiment = row[10]
                                    break
                        except:
                            pass
                    # format: 'Ticker', 'Date(YMD)', 'Open', 'High', 'Low', 'Close', 'Vol', 'Time', 'SocialScore', 'MktCap', 'SentimentRatio'
                    # print(convert_unix_to_time_of_open_bar(int(row['time']), 0, 60).split(' ')[0], row['open'], row['high'], row['low'], row['close'],
                    #             ow['volume'], convert_unix_to_time_of_open_bar(int(row['time']), 0, 60).split(' ')[1], row['market_cap'], row['social_score'], percent_social_sentiment)
                    time_ = convert_unix_to_time_of_open_bar(int(row['time']), 0, 60).split(' ')[1]
                    time_ = '000000' if len(time_) < 2 else ('0' + time_ if len(time_) == 5 else time_)
                    mat.append([coin, convert_unix_to_time_of_open_bar(int(row['time']), 0, 60).split(' ')[0], row['open'], row['high'], row['low'], row['close'],
                                row['volume'], time_, mkt_cap, row['social_score'], percent_social_sentiment, social_engagement])
                except:
                    pass
        if len(mat) > 5000:
            save_csv(mat, fileName=HISTORICAL_FILENAME + interval, folderName=SAVE_FOLDER, header=False, write_append='a')
            mat = []

    if len(mat) > 0:
        save_csv(mat, fileName=HISTORICAL_FILENAME + interval, folderName=SAVE_FOLDER, header=False, write_append='a')


    ''' "time": 1612472400, //A unix timestamp (in seconds)
        "open": 147.268845, //Open price for the time period
        "close": 148.173786, //Close price for the time period
        "high": 148.291545, //Higest price fo rthe time period
        "low": 146.99522, //Lowest price for the time period
        "volume": 183534717, //Volume for the time period in USD
        "market_cap": 9825603638, //Total available supply multiplied
        "tweet_sentiment_impact1": 1863, //Sum of social score (engagement) of all tweets classified as sentiment 1 (Very Bearish)
        "tweet_sentiment_impact2": 13743, //Sum of social score (engagement) of all tweets classified as sentiment 2 (Bearish)
        "tweet_sentiment_impact3": 4182, //Sum of social score (engagement) of all tweets classified as sentiment 3 (Neutral)
        "tweet_sentiment_impact4": 132852, //Sum of social score (engagement) of all tweets classified as sentiment 4 (Bullish)
        "tweet_sentiment_impact5": 25476, //Sum of social score (engagement) of all tweets classified as sentiment 5 (Very Bullish)
        "social_score": 178183, //Sum of followers, retweets, likes, reddit karma etc of social posts collected
        "sentiment_relative" MODIFIED: 84, //Percent tweets that are bullish (excluding neutral in the count)
        "social_impact_score": 3.4, //A proprietary score based on the relative trend of social_score
        "correlation_rank": 2.4, //A score based on how the assets social metrics correlate with price and volume
        "galaxy_score": 60.5, //A proprietary score based on technical indicators of price, average social sentiment, relative social activity, and a factor of how closely social indicators correlate with price and volume'''


get_historical_data('hour')










