#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Mar  2 13:41:07 2019

@author: Kazuki
"""

import numpy as np
import pandas as pd
import os, gc
from glob import glob
from tqdm import tqdm

import sys
sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
import lgbextension as ex
import lightgbm as lgb

from sklearn.externals import joblib
from sklearn.metrics import roc_auc_score

import utils , utils_cat
utils.start(__file__)
#==============================================================================

SUBMIT_FILE_PATH = '../output/0302-2.csv.gz'

COMMENT = 'nejumi + random(f014)'

EXE_SUBMIT = False

SEED = np.random.randint(9999)
print('SEED:', SEED)
#np.random.seed(SEED)

param = {
         'boosting_type': 'gbdt',
         'class_weight': None,
         'colsample_bytree': 0.71,
         'learning_rate': 0.05,
         'max_depth': -1,
         'min_child_samples': 10,
         'min_child_weight': 5,
         'min_split_gain': 0,
#         'n_estimators': n_estimators,
         'n_jobs': -1,
         'num_leaves': 64,
         'objective': 'binary',
#         'random_state': seed,
         'reg_alpha': 0,
         'reg_lambda': 0,
         'subsample': 0.71,
         'subsample_for_bin': 50000,
         'subsample_freq': 1,
         'max_bin': 255,
         'metric': 'auc',
         'nthread': -1,
         'verbose': -1,
#         'seed': seed,
#         'device': 'gpu',
#         'gpu_use_dp': False
         }

NROUND = 19999
NFOLD = 5
VERBOSE_EVAL = 100
ESR = 100

col_drop = [
#        'f003_date_min', 
#        'f003_date_max',
#        'f003_date_max-m-min',
#        'f003_max-m-AvSigVersion',
#        'f003_max-m-Census_OSVersion',
#        'f003_max-m-OsBuildLab'
        ]

USE_PREF = ['f014']

RESULT_DICT = {}
RESULT_DICT['file'] = SUBMIT_FILE_PATH


# =============================================================================
# load
# =============================================================================

files_tr = sorted(glob('../data/train_f*.f'))

# USE_PREF
li = []
for i in files_tr:
    for j in USE_PREF:
        if j in i:
            li.append(i)
            break
files_tr = li

[print(i,f) for i,f in enumerate(files_tr)]

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=30)
               ], axis=1)

col = np.random.choice(X_train.columns, size=int(len(X_train.columns)*0.9), replace=False)
col = list(col)

X_train = pd.concat([X_train[col], 
                     joblib.load('../external/X_train_nejumi.pkl.gz')],
                     axis=1)

y_train = utils.load_target()['HasDetections']

# drop
if len(col_drop) > 0:
    X_train.drop(col_drop, axis=1, inplace=True)


if X_train.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
print('no dup :) ')
print(f'X_train.shape {X_train.shape}')

gc.collect()

CAT = list( set(X_train.columns)&set(utils_cat.ALL))
print(f'CAT: {CAT}')


COL = X_train.columns.tolist()

RESULT_DICT['feature size'] = len(COL)
RESULT_DICT['category feature size'] = len(CAT)


# =============================================================================
# all sample
# =============================================================================


dtrain = lgb.Dataset(X_train, y_train.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)

gc.collect()

#models = []
#for i in range(LOOP):
#    param['seed'] = np.random.randint(9999)
#    model = lgb.train(params=param, train_set=dtrain, 
#                      num_boost_round=NROUND,
#                      )
#    model.save_model(f'../data/lgb{i}.model')
#    models.append(model)

# CV
param['seed'] = np.random.randint(9999)
ret, models = lgb.cv(param, dtrain, NROUND,
                     nfold=NFOLD,
                     stratified=True, shuffle=True,
                     feval=ex.eval_auc,
                     early_stopping_rounds=ESR, 
                     verbose_eval=VERBOSE_EVAL,
                     categorical_feature=CAT,
                     seed=SEED)

for i, model in enumerate(models):
    model.save_model(f'../data/lgb{i}.model')


#models = []
#for i in range(LOOP):
#    model = lgb.Booster(model_file=f'../data/lgb{i}.model')
#    models.append(model)







imp = ex.getImp(models)
imp['split'] /= imp['split'].max()
imp['gain'] /= imp['gain'].max()
imp['total'] = imp['split'] + imp['gain']
imp.sort_values('total', ascending=False, inplace=True)
imp.reset_index(drop=True, inplace=True)


imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
utils.savefig_imp(imp, f'LOG/imp_{__file__}.png', x='total')

RESULT_DICT['nfold'] = NFOLD
RESULT_DICT['seed'] = SEED
RESULT_DICT['eta'] = param['learning_rate']
RESULT_DICT['NROUND'] = NROUND
RESULT_DICT['train AUC'] = ret['auc-mean'][-1]


del dtrain, X_train, y_train; gc.collect()


# =============================================================================
# test
# =============================================================================


files_te = sorted(glob('../data/test_f*.f'))

# USE_PREF
li = []
for i in files_te:
    for j in USE_PREF:
        if j in i:
            li.append(i)
            break
files_te = li

[print(i,f) for i,f in enumerate(files_te)]

X_test = pd.concat([
                pd.read_feather(f) for f in tqdm(files_te, mininterval=30)
               ]+[joblib.load('../external/X_test_nejumi.pkl.gz')], axis=1)[COL]


if X_test.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_test.columns[X_test.columns.duplicated()] }')
print('no dup :) ')
print(f'X_test.shape {X_test.shape}')


y_pred = pd.Series(0, index=X_test.index)

for model in tqdm(models):
    y_pred += pd.Series(model.predict(X_test)).rank()
y_pred /= y_pred.max()

sub = pd.read_csv('../input/sample_submission.csv.zip')
sub['HasDetections'] = y_pred.values


print(f'corr with {utils.SUB_BEST}')
sub_best = pd.read_csv(utils.SUB_BEST)
print('with mybest:', sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )
sub_best['HasDetections'] = np.load(utils.SUB_nejumi)
print('with nejumi:', sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )



print("""
# =============================================================================
# write down these info to benchmark.xlsx
# =============================================================================
""")

[print(f'{k:<25}: {RESULT_DICT[k]}') for k in RESULT_DICT]

print("""
# =============================================================================
""")




# save
#sub.to_csv(SUBMIT_FILE_PATH, index=False, compression='gzip')
utils.to_pkl_gzip(sub[['HasDetections']], SUBMIT_FILE_PATH.replace('.csv.gz', f'_{SEED}.pkl'))

# =============================================================================
# submission
# =============================================================================
if EXE_SUBMIT:
    print('submit')
    utils.submit(SUBMIT_FILE_PATH, COMMENT)




#==============================================================================
utils.end(__file__)
#utils.stop_instance()

