import pandas as pd
import numpy as np
import os
os.environ['OMP_NUM_THREADS'] = '4'
import gc

import keras
import tensorflow as tf

# set gpu usage
config = tf.ConfigProto(gpu_options=tf.GPUOptions(visible_device_list="0",
                                                  allow_growth=True))
session = tf.Session(config=config)
keras.backend.tensorflow_backend.set_session(session)

dtypes = {
        'ip'            : 'uint32',
        'app'           : 'uint16',
        'device'        : 'uint16',
        'os'            : 'uint16',
        'channel'       : 'uint16',
        'is_attributed' : 'uint8',
        'click_id'      : 'uint32'
        }
print('load train....')
train_df = pd.read_pickle("./data/training.pkl.gz")
val_df = pd.read_pickle("./data/validation.pkl.gz")

print('load test....')
test_cols = ['ip', 'app', 'device', 'os', 'click_time', 'channel', 'click_id']
test_df = pd.read_csv("./data/test.csv", dtype=dtypes, usecols=test_cols)


len_train = len(train_df)
len_test = len(test_df)
train_df = train_df.append(val_df)
train_df=train_df.append(test_df)
del test_df
del val_df
gc.collect()

train_df['click_time']= pd.to_datetime(train_df['click_time'])

print('click time....')
train_df['click_time'] = (train_df['click_time'].astype(np.int64) // 10 ** 9).astype(np.int32)
train_df['next_click'] = (train_df.groupby(['ip', 'app', 'device', 'os']).click_time.shift(-1) - train_df.click_time).astype(np.float32)
train_df['next_click'].fillna((train_df['next_click'].mean()), inplace=True)

print('hour, day, wday....')
train_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('uint8')
train_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('uint8')
train_df['wday']  = pd.to_datetime(train_df.click_time).dt.dayofweek.astype('uint8')

print('grouping by ip-day-hour combination....')
fname = 'qty'
gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': fname})
train_df = train_df.merge(gp, on=['ip','day','hour'], how='left')
train_df[fname] = train_df[fname].astype('uint16')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
del gp; gc.collect()

print('group by ip-app combination....')
fname = 'ip_app_count'
gp = train_df[['ip','app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': fname})
train_df = train_df.merge(gp, on=['ip','app'], how='left')
train_df[fname] = train_df[fname].astype('uint16')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
del gp; gc.collect()

print('group by ip-app-os combination....')
fname = 'ip_app_os_count'
gp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': fname})
train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
train_df[fname] = train_df[fname].astype('uint16')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
del gp; gc.collect()

print('group by : ip_day_test_hh')
fname = "nip_day_test_hh"
most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]
least_freq_hours_in_test_data = [6, 11, 15]
train_df['in_test_hh'] = (   3 
                     - 2*train_df['hour'].isin(  most_freq_hours_in_test_data ) 
                     - 1*train_df['hour'].isin( least_freq_hours_in_test_data ) ).astype('uint8')
gp = train_df[['ip', 'day', 'in_test_hh', 'channel']].groupby(by=['ip', 'day',
         'in_test_hh'])[['channel']].count().reset_index().rename(index=str, 
         columns={'channel': fname})
train_df = train_df.merge(gp, on=['ip','day','in_test_hh'], how='left')
del gp
train_df.drop(['in_test_hh'], axis=1, inplace=True)
train_df[fname] = train_df[fname].astype('uint32')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
gc.collect()

print('group by ip-app-os combination....')
fname = 'ip_ch_ncount'
gp = train_df[['ip', 'channel']].groupby(by=['ip'])[['channel']].nunique().reset_index().rename(index=str, columns={'channel': fname})
train_df = train_df.merge(gp, on=['ip'], how='left')
train_df[fname] = train_df[fname].astype('uint16')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
del gp; gc.collect()

print('group by ip-app-os combination....')
fname = 'ip_day_hour_ncount'
gp = train_df[['ip', 'day', 'hour']].groupby(by=['ip', 'day'])[['hour']].nunique().reset_index().rename(index=str, columns={'hour': fname})
train_df = train_df.merge(gp, on=['ip', 'day'], how='left')
train_df[fname] = train_df[fname].astype('uint16')
train_df[fname] = train_df[fname].values / np.max(train_df[fname].values)
train_df[fname] = train_df[fname].astype('float32')
del gp; gc.collect()

print("label encoding....")
from sklearn.preprocessing import LabelEncoder
train_df[['app','device','os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)

print ('final part of preparation....')
test_df = train_df.iloc[-len_test:]
val_df = train_df.iloc[len_train:-len_test]
train_df = train_df.iloc[:len_train]
y_train = train_df['is_attributed'].values
y_val = val_df['is_attributed'].values
train_df.drop(['click_id', 'click_time','ip','is_attributed'],1,inplace=True)
val_df.drop(['click_id', 'click_time','ip','is_attributed'],1,inplace=True)
gc.collect()


print ('neural network....')
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, SpatialDropout1D, Conv1D
from keras.callbacks import Callback
from keras.models import Model
from keras.optimizers import Adam

max_app = np.max([train_df['app'].max(), test_df['app'].max()])+1
max_ch = np.max([train_df['channel'].max(), test_df['channel'].max()])+1
max_dev = np.max([train_df['device'].max(), test_df['device'].max()])+1
max_os = np.max([train_df['os'].max(), test_df['os'].max()])+1
max_h = np.max([train_df['hour'].max(), test_df['hour'].max()])+1
max_d = np.max([train_df['day'].max(), test_df['day'].max()])+1
max_wd = np.max([train_df['wday'].max(), test_df['wday'].max()])+1
def get_keras_data(dataset):
    X = {
        'app': np.array(dataset.app),
        'ch': np.array(dataset.channel),
        'dev': np.array(dataset.device),
        'os': np.array(dataset.os),
        'h': np.array(dataset.hour),
        'd': np.array(dataset.day),
        'wd': np.array(dataset.wday),
        'qty': np.array(dataset.qty),
        'c1': np.array(dataset.ip_app_count),
        'c2': np.array(dataset.ip_app_os_count), 
        "c3": np.array(dataset.nip_day_test_hh), 
        "c4": np.array(dataset.ip_ch_ncount), 
        "c5": np.array(dataset.ip_day_hour_ncount), 
        'nc': np.array(dataset.next_click)
    }
    return X
train_df = get_keras_data(train_df)
val_df = get_keras_data(val_df)

emb_n = 50
dense_n = 1000
in_app = Input(shape=[1], name = 'app')
emb_app = Embedding(max_app, emb_n)(in_app)
in_ch = Input(shape=[1], name = 'ch')
emb_ch = Embedding(max_ch, emb_n)(in_ch)
in_dev = Input(shape=[1], name = 'dev')
emb_dev = Embedding(max_dev, emb_n)(in_dev)
in_os = Input(shape=[1], name = 'os')
emb_os = Embedding(max_os, emb_n)(in_os)
in_h = Input(shape=[1], name = 'h')
emb_h = Embedding(max_h, emb_n)(in_h) 
in_d = Input(shape=[1], name = 'd')
emb_d = Embedding(max_d, emb_n)(in_d) 
in_wd = Input(shape=[1], name = 'wd')
emb_wd = Embedding(max_wd, emb_n)(in_wd)

in_qty = Input(shape=[1], name = 'qty')
in_c1 = Input(shape=[1], name = 'c1')
in_c2 = Input(shape=[1], name = 'c2')
in_c3 = Input(shape=[1], name = 'c3')
in_c4 = Input(shape=[1], name = 'c4')
in_c5 = Input(shape=[1], name = 'c5')
in_nc = Input(shape=[1], name = 'nc')
fe = concatenate([(emb_app), (emb_ch), (emb_dev), (emb_os), (emb_h), 
                 (emb_d), (emb_wd)])
s_dout = SpatialDropout1D(0.2)(fe)
conv = Conv1D(100, kernel_size=4, strides=1, padding='same')(s_dout)
concat = concatenate([Flatten()(s_dout), Flatten()(conv), (in_nc), (in_qty), (in_c1), (in_c2), (in_c3),
                     (in_c4), (in_c5)])
x = Dropout(0.2)(Dense(dense_n,activation='relu')(concat))
x = Dropout(0.2)(Dense(dense_n,activation='relu')(x))
outp = Dense(1,activation='sigmoid')(x)
model = Model(inputs=[in_app,in_ch,in_dev,in_os,in_h,in_d,in_wd,in_qty,in_c1,in_c2,in_c3,in_c4,in_c5,in_nc], outputs=outp)

batch_size = 50000
epochs = 2
exp_decay = lambda init, fin, steps: (init/fin)**(1/(steps-1)) - 1
steps = int(len(list(train_df)[0]) / batch_size) * epochs
lr_init, lr_fin = 0.002, 0.0002
lr_decay = exp_decay(lr_init, lr_fin, steps)
optimizer_adam = Adam(lr=0.002, decay=lr_decay)
model.compile(loss='binary_crossentropy',optimizer=optimizer_adam,metrics=['accuracy'])

model.summary()

class_weight = {0:.01,1:.99} # magic
model.fit(train_df, y_train, batch_size=batch_size, epochs=2, class_weight=class_weight,
          shuffle=True, verbose=2)
del train_df, y_train, y_val; gc.collect()
model.save_weights('imbalanced_data.h5')

sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
test_df.drop(['click_id', 'click_time','ip','is_attributed'],1,inplace=True)
test_df = get_keras_data(test_df)

print("predicting....")
sub['is_attributed'] = model.predict(test_df, batch_size=batch_size, verbose=2).reshape(-1)
del test_df; gc.collect()
print("writing....")
sub.to_csv('nn2_sub.csv.gz', float_format='%.8f',index=False,compression='gzip')

del sub
gc.collect()

res_val = pd.DataFrame()
res_val['is_attributed'] = model.predict(val_df, batch_size=batch_size, verbose=2).reshape(-1)
del val_df
gc.collect()
res_val.to_csv('nn2_val.csv.gz', float_format='%.8f',index=False,compression='gzip')
print("done")
