import numpy as np
import pandas as pd
import lightgbm as lgb

import gc
import pickle
import cPickle
import datetime
import os
import copy
import sys
import time
from multiprocessing import Process, Pool, Manager 
from distutils.version import StrictVersion, LooseVersion

import preprocessing
import common

train_path = "data/trainset.csv"
valid_path = "data/validset.csv"
test_path = "data/testset.csv"

params = {"num_leaves": 60,
          "min_data_in_leaf": 60, 
          "objective":'binary',
          "max_depth": -1,
          "max_bin": 63,
          "learning_rate": 0.10,
          "num_boost_round": 1000,
          "early_stopping_rounds": 10,
          "boosting": "gbdt",
          "feature_fraction": 0.8,
          "bagging_freq": 1,
          "bagging_fraction": 0.8 ,
          "bagging_seed": 11,
          "metric": 'auc',
          "lambda_l1": 0.1,
          "random_state": 133,
          "verbosity": -1}
<<<<<<< HEAD

dtypes = {
    'MachineIdentifier':                                    'object',
    'ProductName':                                          'object',
    'EngineVersion':                                        'object',
    'AppVersion':                                           'object',
    'AvSigVersion':                                         'object',
    'IsBeta':                                               'int8',
    'RtpStateBitfield':                                     'float16',
    'IsSxsPassiveMode':                                     'int8',
    'DefaultBrowsersIdentifier':                            'float16',
    'AVProductStatesIdentifier':                            'float32',
    'AVProductsInstalled':                                  'float16',
    'AVProductsEnabled':                                    'float16',
    'HasTpm':                                               'int8',
    'CountryIdentifier':                                    'int16',
    'CityIdentifier':                                       'float32',
    'OrganizationIdentifier':                               'float16',
    'GeoNameIdentifier':                                    'float16',
    'LocaleEnglishNameIdentifier':                          'int8',
    'Platform':                                             'object',
    'Processor':                                            'object',
    'OsVer':                                                'object',
    'OsBuild':                                              'int16',
    'OsSuite':                                              'int16',
    'OsPlatformSubRelease':                                 'object',
    'OsBuildLab':                                           'object',
    'SkuEdition':                                           'object',
    'IsProtected':                                          'float16',
    'AutoSampleOptIn':                                      'int8',
    'PuaMode':                                              'object',
    'SMode':                                                'float16',
    'IeVerIdentifier':                                      'float16',
    'SmartScreen':                                          'object',
    'Firewall':                                             'float16',
    'UacLuaenable':                                         'float32',
    'Census_MDC2FormFactor':                                'object',
    'Census_DeviceFamily':                                  'object',
    'Census_OEMNameIdentifier':                             'float16',
    'Census_OEMModelIdentifier':                            'float32',
    'Census_ProcessorCoreCount':                            'float16',
    'Census_ProcessorManufacturerIdentifier':               'float16',
    'Census_ProcessorModelIdentifier':                      'float16',
    'Census_ProcessorClass':                                'object',
    'Census_PrimaryDiskTotalCapacity':                      'float32',
    'Census_PrimaryDiskTypeName':                           'object',
    'Census_SystemVolumeTotalCapacity':                     'float32',
    'Census_HasOpticalDiskDrive':                           'int8',
    'Census_TotalPhysicalRAM':                              'float32',
    'Census_ChassisTypeName':                               'object',
    'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
    'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
    'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
    'Census_PowerPlatformRoleName':                         'object',
    'Census_InternalBatteryType':                           'object',
    'Census_InternalBatteryNumberOfCharges':                'float32',
    'Census_OSVersion':                                     'object',
    'Census_OSArchitecture':                                'object',
    'Census_OSBranch':                                      'object',
    'Census_OSBuildNumber':                                 'int16',
    'Census_OSBuildRevision':                               'int32',
    'Census_OSEdition':                                     'object',
    'Census_OSSkuName':                                     'object',
    'Census_OSInstallTypeName':                             'object',
    'Census_OSInstallLanguageIdentifier':                   'float16',
    'Census_OSUILocaleIdentifier':                          'int16',
    'Census_OSWUAutoUpdateOptionsName':                     'object',
    'Census_IsPortableOperatingSystem':                     'int8',
    'Census_GenuineStateName':                              'object',
    'Census_ActivationChannel':                             'object',
    'Census_IsFlightingInternal':                           'float16',
    'Census_IsFlightsDisabled':                             'float16',
    'Census_FlightRing':                                    'object',
    'Census_ThresholdOptIn':                                'float16',
    'Census_FirmwareManufacturerIdentifier':                'float16',
    'Census_FirmwareVersionIdentifier':                     'float32',
    'Census_IsSecureBootEnabled':                           'int8',
    'Census_IsWIMBootEnabled':                              'float16',
    'Census_IsVirtualDevice':                               'float16',
    'Census_IsTouchEnabled':                                'int8',
    'Census_IsPenCapable':                                  'int8',
    'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
    'Wdft_IsGamer':                                         'float16',
    'Wdft_RegionIdentifier':                                'float16',
    'HasDetections':                                        'int8'
}
=======
>>>>>>> develop

def load_test_data():
    sys.stderr.write("Loading test data ......\n")
    test_data = pd.read_csv(test_path, dtype=common.dtypes)
    machine_id = copy.copy(test_data["MachineIdentifier"])
    del test_data["MachineIdentifier"]

    return test_data, machine_id

def load_valid_data():
    sys.stderr.write("Loading valid data ......\n")

    data = pd.read_csv(valid_path, dtype=common.dtypes)
    del data["MachineIdentifier"]

    valid_label = data["HasDetections"]
    valid_data = pd.DataFrame(data.drop("HasDetections", axis=1))
    
    return valid_data, valid_label

def load_train_data():
    sys.stderr.write("Loading train data ......\n")

    data = pd.read_csv(train_path, dtype=common.dtypes)
    del data["MachineIdentifier"]

    train_label = data["HasDetections"]
    train_data = pd.DataFrame(data.drop("HasDetections", axis=1))
    
    return train_data, train_label

def main():
    tr_data, tr_label = load_train_data()
    v_data, v_label = load_valid_data()
    te_data, machine_id = load_test_data()
    feature_name = tr_data.columns.tolist()
    offset1 = tr_data.shape[0]
    offset2 = offset1 + v_data.shape[0]

    X = pd.concat([tr_data, v_data, te_data], ignore_index=True, sort=False)
    del tr_data, v_data, te_data
    gc.collect()

    d = {}
    for col in common.category_col:
        X[col] = X[col].astype("category")
        X[col].cat.add_categories([""], inplace=True)
        d[col] = ""
    for col in common.non_category_col:
        d[col] = X[col].max() + 1 

    X = X.fillna(d)
    ret = preprocessing.col_factorize(X)
    ret = preprocessing.reduce_mem_usage(ret)

    ret_train_data = ret[:offset1]
    ret_valid_data = ret[offset1:offset2]
    ret_test_data = ret[offset2:]
    del ret, X
    gc.collect()

    train_data = lgb.Dataset(ret_train_data, label=tr_label, free_raw_data=True)
    ret_train_data = None
    tr_label = None
    gc.collect()

    valid_data = lgb.Dataset(ret_valid_data, label=v_label)
    ret_valid_data = None
    v_label = None
    gc.collect()

    sys.stderr.write("Training ......\n")
    bst = lgb.train(params, train_data, valid_sets=[valid_data], feature_name=feature_name, categorical_feature=common.category_col)

    sys.stderr.write("Testing ......\n")
    predict = bst.predict(ret_test_data)
    predict = (predict > 0.5).astype(np.int)

    out = np.stack((machine_id, predict), axis=1)
    print(out.shape)
    np.savetxt("submission.csv", fmt="%s,%s", X=out, header="MachineIdentifier,HasDetections")

main()
