import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib
matplotlib.use("agg")
import matplotlib.pyplot as plt
import seaborn as sns

import gc
import pickle
import cPickle
import datetime
import os
import copy
import sys
import time
from multiprocessing import Process, Pool, Manager 
from distutils.version import StrictVersion, LooseVersion

import preprocessing
import common

train_path = "data/trainset.csv"
test_path = "data/testset.csv"

params = {"num_leaves": 60,
          "min_data_in_leaf": 60, 
          "objective":'binary',
          "max_depth": -1,
          "max_bin": 63,
          "learning_rate": 0.1,
          "num_boost_round": 1000,
          "early_stopping_rounds": 10,
          "boosting": "gbdt",
          "feature_fraction": 0.8,
          "bagging_freq": 1,
          "bagging_fraction": 0.8 ,
          "bagging_seed": 11,
          "metric": 'auc',
          "lambda_l1": 0.1,
          "random_state": 133,
          "verbosity": -1}

def load_test_data():
    sys.stderr.write("Loading test data ......\n")
    test_data = pd.read_csv(test_path, dtype=common.dtypes)
    del test_data["MachineIdentifier"]
    test_label = pd.DataFrame(0, index=np.arange(test_data.shape[0]), columns=[0], dtype=np.int8)

    return test_data, test_label

def load_train_data():
    sys.stderr.write("Loading train data ......\n")
    train_data = pd.read_csv(train_path, dtype=common.dtypes)
    del train_data["MachineIdentifier"], train_data["HasDetections"]
    train_label = pd.DataFrame(1, index=np.arange(train_data.shape[0]), columns=[0], dtype=np.int8)
    
    return train_data, train_label

def main():
    tr_data, tr_label = load_train_data()
    te_data, te_label = load_test_data()
    offset = tr_data.shape[0]
    
    X = pd.concat([tr_data, te_data], sort=False, ignore_index=True)
    Y = pd.concat([tr_label, te_label], sort=False, ignore_index=True) 
    del tr_data, te_data, tr_label, te_label
    gc.collect()

    d = {}
    for col in common.category_col:
        X[col] = X[col].astype("category")
        X[col].cat.add_categories([""], inplace=True)
        d[col] = ""
    for col in common.non_category_col:
        d[col] = X[col].max() + 1
    X = X.fillna(d)

    ret = preprocessing.col_factorize(X)
    ret = preprocessing.reduce_mem_usage(ret)
    
    train_data = lgb.Dataset(ret, label=Y)
    del X, ret
    gc.collect()

    bst = lgb.train(params, train_data, valid_sets=[train_data])

main()
