is_debug = 1
from tqdm import tqdm
import os
import gc
import re
import sys
import glob
import datetime
import pandas as pd
import numpy as np
HOME = os.path.expanduser('~')
sys.path.append(f"{HOME}/kaggle/data_analysis/library/")
sys.path.append(f"../py/")
import MS_utils
import utils, ml_utils, kaggle_utils
from utils import logger_func
try:
    if not logger:
        logger=logger_func()
except NameError:
    logger=logger_func()
import time
from sklearn.metrics import mean_squared_error, roc_auc_score
#========================================================================
# XdeepFM 
sys.path.append(f'{HOME}/kaggle/data_analysis/model')
sys.path.append('../../../tool/ctrNet-tool/')
import ctrNet
import tensorflow as tf
from src import misc_utils
import random
#========================================================================

start_time = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now())

# Columns
key, target, ignore_list = MS_utils.get_basic_var()
comment = sys.argv[1]

base = utils.read_df_pkl('../input/base_group*')[[key, target, 'country_group']]
base_train = base[~base[target].isnull()]

from scipy.sparse import vstack, csr_matrix, save_npz, load_npz, hstack
train = load_npz('../input/sp_train_1032.npz')
x_test = load_npz('../input/sp_test_1032.npz')
#  train = train.tocsr()
#  x_test = x_test.tocsr()
train = train.toarray()
x_test = x_test.toarray()


if is_debug:
    train = train[:10000, :]
    x_test = x_test[:10000, :]
    base_train = base_train.head(10000)

Y = base_train[target]
print(f"Train: {train.shape} | Test: {x_test.shape}")
# ========================================================================

#========================================================================
# CVの準備
seed = 1208
fold_n = 5
kfold = MS_utils.get_kfold(base=base_train)
kfold = zip(*kfold)
#========================================================================

#========================================================================
# NN Setting
hparam=tf.contrib.training.HParams(
            model='xdeepfm',
            norm=True,
            batch_norm_decay=0.9,
            hidden_size=[128,128],
            cross_layer_sizes=[128,128,128],
            k=8,
            hash_ids=int(2e5),
            batch_size=1024,
            optimizer="adam",
            learning_rate=0.001,
            num_display_steps=1000,
            num_eval_steps=1000,
            epoch=1,
            metric='auc',
            activation=['relu','relu','relu'],
            cross_activation='identity',
            init_method='uniform',
            init_value=0.1,
            feature_nums=train.shape[1],
            kfold=5)
misc_utils.print_hparams(hparam)
#========================================================================

#========================================================================
# Result Box
model_list = []
result_list = []
score_list = []
val_pred_list = []

oof_pred = np.zeros(train.shape[0])
y_test = np.zeros(x_test.shape[0])
#========================================================================

model_type = 'NN'

#========================================================================
# Train & Prediction Start

oof_pred = np.zeros(train.shape[0])
for num_fold, (trn_idx, val_idx) in enumerate(kfold):

    #========================================================================
    # 複数スレッドでkfoldを取得する
    get_fold = sys.argv[2].split('_')
    get_fold_list = []
    for fold in get_fold:
        get_fold_list.append(np.int(fold))
    if num_fold not in get_fold_list:
        continue
    #========================================================================

    print("Fold",num_fold)

    x_train = train[trn_idx]
    y_train = Y.iloc[trn_idx].values
    x_val = train[val_idx]
    y_val = Y.iloc[val_idx].values

    print(x_train.shape, x_val.shape)

    model=ctrNet.build_model(hparam)
    model.train(train_data=(x_train, y_train), dev_data=(x_val, y_val))
    print("Training Done! Inference...")
    if i==0:
        y_pred = model.infer(dev_data=(x_val, y_val))/hparam.kfold
        oof_pred[val_idx] += y_pred
        y_test = model.infer(dev_data=(x_test, np.ones(x_test.shape[0])*np.nan ))/hparam.kfold
    else:
        y_pred = model.infer(dev_data=(x_val, y_val))/hparam.kfold
        oof_pred[val_idx] += y_pred
        y_test += model.infer(dev_data=(x_test, np.ones(x_test.shape[0])*np.nan ))/hparam.kfold

    #========================================================================
    # Scorring
    score = roc_auc_score(y_val, y_pred)
    print(f'AUC: {score}')
    score_list.append(score)
    #========================================================================

y_test /= len(get_fold_list)
pred_col = 'prediction'
base[pred_col] = np.hstack((oof_pred, y_test))
base = base[[key, pred_col]]
print(f"DF Stack Shape: {base.shape}")
#========================================================================
# Saving
utils.to_pkl_gzip(obj=base, path=f'../output/{start_time[4:12]}_{comment}_stack_{model_type}_FOLD-{sys.argv[2]}_feat{x_train.shape[1]}')
#========================================================================
