#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 23 16:28:05 2019

@author: Kazuki
"""

import numpy as np
import pandas as pd
import os, gc
from glob import glob
from tqdm import tqdm

import sys
sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
import lgbextension as ex
import lightgbm as lgb
from multiprocessing import cpu_count
from sklearn.externals import joblib

from sklearn.metrics import roc_auc_score

import utils , utils_cat
utils.start(__file__)
#==============================================================================

SEED = np.random.randint(9999)
print('SEED:', SEED)


NFOLD = 5

LOOP = 1

param = {
         'boosting_type': 'gbdt',
         'class_weight': None,
         'colsample_bytree': 0.71,
         'importance_type': 'gain',
         'learning_rate': 0.1,
         'max_depth': -1,
         'min_child_samples': 10,
         'min_child_weight': 5,
         'min_split_gain': 0,
#         'n_estimators': n_estimators,
         'n_jobs': -1,
         'num_leaves': 64,
         'objective': 'binary',
#         'random_state': seed,
         'reg_alpha': 0,
         'reg_lambda': 0,
         'silent': True,
         'subsample': 0.71,
         'subsample_for_bin': 50000,
         'subsample_freq': 1,
         'max_bin': 255,
         'metric': 'auc',
         'nthread': -1,
#         'seed': seed,
#         'device': 'gpu',
#         'gpu_use_dp': False
         }

NROUND = 99999
ESR = 100
VERBOSE_EVAL = 50

USE_PREF = [
        'f009',
        'f010',
        'f011',
        'f012',
#        'f010'
        ]

DROP = [
#        'f002_EngineVersion', 
#        'f002_AppVersion',
#        'f002_AvSigVersion',
#        'f002_Census_OSVersion',
        ]

COL = pd.read_csv('LOG/imp_801_cv.py.csv').head(10).feature.tolist()

# =============================================================================
# load
# =============================================================================


files_tr = sorted(glob('../data/train_f*.f'))

# USE_PREF
li = []
for i in files_tr:
    for j in USE_PREF:
        if j in i:
            li.append(i)
            break
files_tr = li

[print(i,f) for i,f in enumerate(files_tr)]

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=30)
               ], axis=1)[COL]
X_train = pd.concat([X_train, joblib.load('../external/X_train_nejumi.pkl.gz')],
                     axis=1)

y_train = utils.load_target()['HasDetections']


# drop
X_train.drop(DROP, axis=1, inplace=True)


if X_train.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
print('no dup :) ')
print(f'X_train.shape {X_train.shape}')

gc.collect()

CAT = list( set(X_train.columns)&set(utils_cat.ALL))
print(f'CAT: {CAT}')

# =============================================================================
# cv
# =============================================================================

dtrain = lgb.Dataset(X_train, y_train.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)
gc.collect()

model_all = []
nround_mean = 0
loss_list = []
y_preds = []
for i in range(LOOP):
    gc.collect()
    
    param['seed'] = np.random.randint(9999)
    
    ret, models = lgb.cv(param, dtrain, NROUND,
                         nfold=NFOLD,
                         stratified=True, shuffle=True,
                         feval=ex.eval_auc,
                         early_stopping_rounds=ESR, 
                         verbose_eval=VERBOSE_EVAL,
                         categorical_feature=CAT,
                         seed=SEED+i)
    
    y_pred = ex.eval_oob(X_train, y_train.values, models, SEED+i, stratified=True, shuffle=True)
    y_preds.append(y_pred)
    
    model_all += models
    nround_mean += len(ret['auc-mean'])
    loss_list.append( ret['auc-mean'][-1] )

nround_mean = int((nround_mean/LOOP) * 1.3)


imp = ex.getImp(model_all)
imp['split'] /= imp['split'].max()
imp['gain'] /= imp['gain'].max()
imp['total'] = imp['split'] + imp['gain']
imp.sort_values('total', ascending=False, inplace=True)
imp.reset_index(drop=True, inplace=True)


for i,y_pred in enumerate(y_preds):
    if i==0:
        oof = y_pred
    else:
        oof += y_pred
oof /= len(y_preds)



imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
pd.DataFrame(oof, columns=['oof']).to_csv(f'../data/oof_{__file__}.csv', index=False)

utils.savefig_imp(imp, f'LOG/imp_{__file__}.png', x='total')




#==============================================================================
utils.end(__file__)
#utils.stop_instance()

