#!/usr/local/bin/python 

import re, sklearn, sys, os, datetime, getopt, glob, argparse, datetime, json, base64, pickle
import numpy as np
import pandas as pd
from collections import defaultdict
from sklearn.preprocessing import StandardScaler

sys.path.append(".")
sys.path.append("gen")
import utils1;

'Covert to one_hot encoding with prefix for columns'
def makeOneHot(tf1, oheCols=[]):
    ohe = pd.DataFrame();
    for c in oheCols:
        one_hot = pd.get_dummies(tf1[c])
        nc = [f'{c}___{k}' for k in one_hot.columns]
        one_hot.columns = nc
        ohe = pd.concat([ohe, one_hot], axis=1)

    return ohe

def addDiff(tf1, col):
    #col = "MSFT_open"
    if (type(col) == str):
        col = [col]
    for c in col:
        if ( c not in tf1.columns):
            print(f"*WARNING* Column {c} Not FOUND")
            continue
        print(f"+++ Adding {c}")
        tf1[f'{c}___diff1'] = tf1[c] - tf1[c].shift(1)
    return tf1

#-----------------------------------------------------------------------------------
#*** NOTE: DO NOT EDIT THIS FILE - THIS iS CREATED FROM: inv_utils.ipynb
def inJupyter():
    try:    get_ipython; return True
    except: return False

#-----------------------------------------------------------------------------------
def formatConfig(out: dict):
    outj = f"[START]\n{{\n"
    for k,v in out.items():
        kk = f'"{k}"'
        vv = f"'{v}'" if type(v) == str else v
            
        outj += f'{kk:>20}: {vv},\n'
    outj += '"end": 0 \n}\n[END]\n'
    
    return outj

#-----------------------------------------------------------------------------------
def getConfigList(conf, key=""):
    #print(f"Getting {key}")
    ll = conf.get(key, []);
    ret = []
    for l in ll:
        if type(l) == list:
            ret += l
        elif l.startswith("$"):
            ret += getConfigList(conf, l[1:])
        else:
            ret.append(l)

    return ret
#-----------------------------------------------------------------------------------
def getConfigObject(conf, key=""):
    ll = conf.get(key, "");
    if not ll:
        return;
    
    sst = ll[0]
    dec = base64.b64decode(sst)
    ret = pickle.loads(dec, fix_imports=True)

    return ret
#-----------------------------------------------------------------------------------
def getFinalColumns(df, conf):
    inputCols = getConfigList(conf, "inputs")
    ouputCols = getConfigList(conf, "outputs")

    allcols = set(inputCols +ouputCols);
    
    assert allcols.issubset(set(df.columns)), "Hmmm ... columns missing"
    return sorted(list(allcols))
#-----------------------------------------------------------------------------------
'''
This will scale the numeric columns - if this changes - you need to use dataprep
'''
def scaleNumerics(df, conf={}):
    scaleInputCols = conf['scaleInputs']
    scaleOuputCols = conf['scaleOutputs']
    
    dfninps = df[ scaleInputCols ]
    dfnouts = df[ scaleOuputCols ]

    scale   = conf.get('scale', 0)
    dfninpsn, dfnoutsn, scalerX, scalerY = dfninps, dfnouts, None, None
    
    if (scale):
        scalerXStr = conf['scalerXString']
        scalerYStr = conf['scalerYString']
        trnPct     = conf.get('train_pct', 0.9);
        trnCnt     = int(len(df) * trnPct)

        conf["train_pct"   ] = trnPct
        conf["train_count" ] = trnCnt

        if (not scalerXStr):
            scaler  = conf.get('scaler', ["sklearn.preprocessing.MinMaxScaler()"]);
            scalerX = eval(scaler[0]) if type(scaler[0]) == str else scaler
            scalerX = scalerX.fit(dfninps[:trnCnt])
            scalerstr = base64.b64encode(pickle.dumps(scalerX, protocol=None, fix_imports=True))
            scalerstr = scalerstr.decode("utf-8")
            conf["scalerXString"] = [scalerstr]
            #print(f'==>+1 shape: {dfninps.shape} {scalerX.mean_}')
        else:
            scalerXStr = scalerXStr[0]
            decoded    = base64.b64decode(scalerXStr)
            scalerX    = pickle.loads(decoded,fix_imports=True)

        if (not scalerYStr):
            scaler  = conf.get('scaler', ["sklearn.preprocessing.MinMaxScaler()"]);
            scalerY = eval(scaler[0]) if type(scaler[0]) == str else scaler
            scalerY = scalerY.fit(dfnouts[:trnCnt])
            scalerstr = base64.b64encode(pickle.dumps(scalerY, protocol=None, fix_imports=True))
            scalerstr = scalerstr.decode("utf-8")
            conf["scalerYString"] = [scalerstr]
            #print(f'==>+2 shape: {dfninps.shape} {scalerY.mean_}')
        else:
            scalerYStr = scalerYStr[0]
            decoded    = base64.b64decode(scalerYStr)
            scalerY    = pickle.loads(decoded,fix_imports=True)

        di = scalerX.transform(dfninps)
        do = scalerY.transform(dfnoutsn)
        
        dfninpsn = pd.DataFrame(di, columns=scaleInputCols )
        dfnoutsn = pd.DataFrame(do, columns=scaleOuputCols )
        
        #print(f'==>++ shape: {dfninpsn.shape} {scalerX.mean_}')
        #print(f'==>++ shape: {dfnoutsn.shape} {scalerY.mean_}')

    return dfninpsn, dfnoutsn, scalerX, scalerY;
#-----------------------------------------------------------------------------------
def process(config, input_files, output=None):
    conf = utils1.getconfig(config)
    
    n  = len(input_files)
    adfInp = pd.DataFrame();
    for i, file1 in enumerate(input_files):
        print(f"=>Processing {i+1}/{n} {file1} - standby")
        df = pd.read_csv(file1, comment='#')
        
        drps = getConfigList(conf, "dropColumns")
        df.drop(drps, inplace=True, errors="ignore")
        
        # STEP 1: Add diffs
        cdiffs  = getConfigList(conf, 'addDiffs')
        addDiff(df, cdiffs)    #<< 1. Add tis
        df.dropna(inplace=True)
        df.reset_index(inplace=True, drop=True)

        # STEP 2: => One hot encode 
        ohe = None
        ohecols = getConfigList(conf, 'onehotECols')
        if len(ohecols) > 0:
            ohe=makeOneHot(df, conf['onehotECols'])  #< === ADD
            df=pd.concat([df,ohe], axis=1)
         
        # STEP 3: Add
        allCols = [df.columns[0]] + getFinalColumns(df,conf)
        dfunNormalized = df[allCols]
        #Numeric Columns
        dfiNorm, dfoNorm, sX, sY = scaleNumerics(dfunNormalized, conf)
        
        dfNormalized = dfunNormalized.copy()
        dfNormalized[dfiNorm.columns] = dfiNorm;
        dfNormalized[dfoNorm.columns] = dfoNorm;
        
        #FINALLY        
        if (output is not None):
            fi,ext = os.path.splitext(file1)
            nfu  = f'{os.path.basename(fi)}_Orig_{i}{ext}'
            print(f"writing unnormalized to: {nfu}")
            dfunNormalized.to_csv(nfu, index=False)
            
            
            nfn  = f'{os.path.basename(fi)}_Norm_{i}{ext}'
            print(f"writing normalized to. : {nfn}")
            dfNormalized.to_csv  (nfn, index=False)
            
            conf['normalizedFile']   = nfn
            conf['unnormalizedFile'] = nfu
            
        break;

    outj = formatConfig(conf)
    print(outj)
    
    return conf, dfunNormalized, dfNormalized
#-----------------------------------------------------------------------------------
sysargs=None
def addargs():
    sysargs = None
    p = argparse.ArgumentParser(f"{os.path.basename(sys.argv[0])}:")
    p.add_argument('-c', '--config', type=str, default="config.txt", help="Config Files")
    p.add_argument('-o', '--output', type=str, default=0, help="output file")
    p.add_argument('args', nargs=argparse.REMAINDER)
    p.add_argument('input_files',action="store", type=str, nargs='+', help="input file(s)")

    #p.print_help() # always print -help
    try:
        sysargs=p.parse_args(sys.argv[1:])
    except argparse.ArgumentError as exc:
        print(exc.message )
        
    return sysargs
    
#-----------------------------------------------------------------------------------
if __name__ == '__main__':
    if (not inJupyter()):
        t1 = datetime.datetime.now()
        sysargs = addargs()
        ret = process(sysargs.config, sysargs.input_files, sysargs.output)
        t2 = datetime.datetime.now()
        print(f"#All Done in {str(t2-t1)} ***")
    else:
        pass
