is_debug = 0
is_down = 0
import os
import re
import gc
import sys
import glob
import pandas as pd
import numpy as np
import datetime
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold, StratifiedKFold
from scipy.sparse import vstack, csr_matrix, save_npz, load_npz
from sklearn.preprocessing import LabelEncoder, OneHotEncoder

#========================================================================
# original library 
HOME = os.path.expanduser('~')
sys.path.append(f"{HOME}/kaggle/data_analysis/library/")
sys.path.append(f"{HOME}/kaggle/data_analysis/model/")
import MS_utils
from params_MS import params_lgb
import utils, ml_utils
from utils import logger_func
try:
    if not logger:
        logger=logger_func()
except NameError:
    logger=logger_func()
#========================================================================

"""
argv[1]: comment
argv[2]: feature_key
"""
# Basic Args
seed = 1208
set_type = 'all'
fold_n = 5
key, target, ignore_list = MS_utils.get_basic_var()
ignore_list = [key, target, 'country_group', 'down_flg']
comment = sys.argv[1]

# Base
base_path = '../input/base_exclude*'
base_path = '../input/base_Av*'
base_path = '../input/base_group*'
base = utils.read_df_pkl(base_path)[[key, target]]
base_train = base[~base[target].isnull()]
train_ids = base_train.index

Y = base[target].values.astype('int8')

train = load_npz('train.npz')
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
skf.get_n_splits(train_ids, Y)

model_type = 'lgb'
metric = 'auc'

feim_list = []
score_list = []
oof_pred = np.zeros(len(train))
y_test = np.zeros(len(test))

start_time = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now())


for num_fold, (train_index, test_index) in enumerate(skf.split(train_ids, y_train)):

    x_train = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
    x_val = vstack([train[test_index[i*m:(i+1)*m]]  for i in range(test_index.shape[0] //  m + 1)])
    x_train, x_val = csr_matrix(x_train, dtype='float32'), csr_matrix(x_val, dtype='float32')
    y_train, y_val = Y[train_index], Y[test_index]

    del train
    gc.collect()

    x_test = load_npz('test.npz')
    x_test = csr_matrix(test, dtype='float32')

    params = params_lgb()
    params['num_threads'] = -1
    params['learning_rate'] = 0.05

    score, tmp_oof, tmp_pred, feim, _ = ml_utils.Classifier(
        model_type=model_type
        , x_train=x_train
        , y_train=y_train
        , x_val=x_val
        , y_val=y_val
        , x_test=x_test
        , params=params
        , seed=seed
        , get_score=metric
    )
    feim_list.append(feim.set_index('feature').rename(columns={'importance':f'imp_{num_fold}'}))

    logger.info(f"Fold{num_fold} CV: {score}")
    score_list.append(score)
    oof_pred[val_idx] = tmp_oof
    y_test += tmp_pred

feim = pd.concat(feim_list, axis=1)
feim_cols = [col for col in feim.columns if col.count('imp_')]
feim['importance'] = feim[feim_cols].mean(axis=1)
feim.drop(feim_cols, axis=1, inplace=True)
feim.sort_values(by='importance', ascending=False, inplace=True)
feim['rank'] = np.arange(len(feim))+1

cv_score = np.mean(score_list)
logger.info(f'''
#========================================================================
# Model: {model_type}
# CV   : {cv_score}
#========================================================================''')

y_test /= (num_fold+1)

pred_col = 'prediction'
train[pred_col] = oof_pred
test[pred_col] = y_test
stack_cols = [key, target, pred_col]

df_stack = pd.concat([train[stack_cols], test[stack_cols]], ignore_index=True, axis=0)

#========================================================================
# Saving
feim.to_csv(f'../valid/{start_time[4:12]}_valid_{model_type}_SET-{set_type}_feat{len(x_train.columns)}_{comment}_CV{str(cv_score)[:7]}_LB.csv', index=True)
utils.to_pkl_gzip(obj=df_stack, path=f'../stack/{start_time[4:12]}_stack_{model_type}_SET-{set_type}_feat{len(x_train.columns)}_{comment}_CV{str(cv_score)[:7]}_LB')

submit = pd.read_csv('../input/sample_submission.csv').set_index(key)
submit[target] = test[pred_col].values
submit.to_csv(f'../submit/{start_time[4:12]}_submit_{model_type}_SET-{set_type}_feat{len(x_train.columns)}_{comment}_CV{str(cv_score)[:7]}_LB.csv', index=True)
#========================================================================
