#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar  7 15:12:56 2019

@author: kazuki.onodera
"""

import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.externals import joblib
from sklearn.metrics import roc_auc_score

from category_encoders import OrdinalEncoder, TargetEncoder
from tqdm import tqdm

import gc, os

# =============================================================================
# setting
# =============================================================================

SEED = 71
np.random.seed(SEED)

# =============================================================================

dtypes = {
        'MachineIdentifier':                                    'str',
        'ProductName':                                          'str',
        'EngineVersion':                                        'str',
        'AppVersion':                                           'str',
        'AvSigVersion':                                         'str',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'str',
        'Processor':                                            'str',
        'OsVer':                                                'str',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'str',
        'OsBuildLab':                                           'str',
        'SkuEdition':                                           'str',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'str',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'str',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'str',
        'Census_DeviceFamily':                                  'str',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'str',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'str',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'str',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'str',
        'Census_InternalBatteryType':                           'str',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'str',
        'Census_OSArchitecture':                                'str',
        'Census_OSBranch':                                      'str',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'str',
        'Census_OSSkuName':                                     'str',
        'Census_OSInstallTypeName':                             'str',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'str',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'str',
        'Census_ActivationChannel':                             'str',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'str',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df

df_train = pd.read_csv('../input/train.csv.zip', dtype=dtypes)
tr_smartscreen = df_train['SmartScreen']

df_test = pd.read_csv('../input/test.csv.zip', dtype=dtypes)

replace_dict = {
'requireadmin': 'RequireAdmin', 
'OFF': 'Off',
'of': 'Off', 
'off': 'Off',  
'on': 'On', 
'ON': 'On', 
'BLOCK': 'Block', 
'warn': 'Warn', 
'Promprt': 'Prompt', 
'Promt': 'Prompt', 
'RequiredAdmin': 'RequireAdmin', 
'requireadmin': 'RequireAdmin', 
'requireAdmin': 'RequireAdmin'
}

df_train.SmartScreen.replace(replace_dict, inplace=True)
df_test.SmartScreen.replace(replace_dict, inplace=True)

y_train = df_train.HasDetections

X_train = df_train.drop(['HasDetections', 'MachineIdentifier', 'AvSigVersion', 'Census_OSVersion'], axis=1)
X_test = df_test.drop(['MachineIdentifier', 'AvSigVersion', 'Census_OSVersion'], axis=1)

del df_train, df_test


X_train['OsBuildLab'] = X_train['OsBuildLab'].fillna('0.0.0.0.0-0')
X_test['OsBuildLab'] = X_test['OsBuildLab'].fillna('0.0.0.0.0-0')

# =============================================================================
# # この辺はPublic Kernelから
# =============================================================================
def fe(df):

    df['OsBuildLab_0'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0]).astype('str')
    df['OsBuildLab_1'] = df['OsBuildLab'].apply(lambda x: x.split('.')[1]).astype('str')
    df['OsBuildLab_2'] = df['OsBuildLab'].apply(lambda x: x.split('.')[2]).astype('str')
    df['OsBuildLab_3'] = df['OsBuildLab'].apply(lambda x: x.split('.')[3]).astype('str')

    df['primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']

    df['aspect_ratio'] = df['Census_InternalPrimaryDisplayResolutionHorizontal']/ df['Census_InternalPrimaryDisplayResolutionVertical']

    df['monitor_dims'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str) + '*' + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')
    df['monitor_dims'] = df['monitor_dims'].astype('category')

    df['dpi'] = ((df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype('float32')**2 + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('float32')**2)**.5)/(df['Census_InternalPrimaryDiagonalDisplaySizeInInches'])

    df['dpi_square'] = df['dpi'] ** 2

    df['MegaPixels'] = (df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype('float32') * df['Census_InternalPrimaryDisplayResolutionVertical'].astype('float32'))/1e6

    df['Screen_Area'] = (df['aspect_ratio']* (df['Census_InternalPrimaryDiagonalDisplaySizeInInches']**2))/(df['aspect_ratio']**2 + 1)

    df['ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']

    df['new_num_0'] = df['Census_InternalPrimaryDiagonalDisplaySizeInInches'] / df['Census_ProcessorCoreCount']

    df['new_num_1'] = df['Census_ProcessorCoreCount'] * df['Census_InternalPrimaryDiagonalDisplaySizeInInches']
    
    df['Census_IsFlightingInternal'] = df['Census_IsFlightingInternal'].fillna(1)
    df['Census_ThresholdOptIn'] = df['Census_ThresholdOptIn'].fillna(1)
    df['Census_IsWIMBootEnabled'] = df['Census_IsWIMBootEnabled'].fillna(1)
    df['Wdft_IsGamer'] = df['Wdft_IsGamer'].fillna(0)
    
    return df

X_train = fe(X_train)
X_test = fe(X_test)

X_train = reduce_mem_usage(X_train)
X_test = reduce_mem_usage(X_test)


# =============================================================================
# 
# =============================================================================
categorical_wo_version = ['Census_OSUILocaleIdentifier',
                                         'AVProductsInstalled',
                                         'Census_FirmwareVersionIdentifier',
                                         'Wdft_IsGamer',
                                         'Census_ThresholdOptIn',
                                         'RtpStateBitfield',
                                         'Census_IsSecureBootEnabled',
                                         'AVProductsEnabled',
                                         'HasTpm',
                                         'IsProtected',
                                         'Census_PrimaryDiskTypeName',
                                         'PuaMode',
                                         'DefaultBrowsersIdentifier',
                                         'IsSxsPassiveMode',
                                         'OrganizationIdentifier',
                                         'Census_IsAlwaysOnAlwaysConnectedCapable',
                                         'ProductName',
                                         'GeoNameIdentifier',
                                         'Census_IsVirtualDevice',
                                         'Census_PowerPlatformRoleName',
                                         'Census_IsTouchEnabled',
                                         'Census_OSSkuName',
                                         'OsPlatformSubRelease',
                                         'Census_FlightRing',
                                         'Census_OSEdition',
                                         'Census_IsPortableOperatingSystem',
                                         'Firewall',
                                         'OsBuildLab',
                                         'Census_DeviceFamily',
                                         'Census_IsPenCapable',
                                         'SMode',
                                         'Platform',
                                         'Census_IsFlightingInternal',
                                         'Census_OEMNameIdentifier',
                                         'Census_InternalBatteryType',
                                         'OsBuild',
                                         'Census_HasOpticalDiskDrive',
                                         'Census_IsWIMBootEnabled',
                                         'Census_OSBuildRevision',
                                         'CityIdentifier',
                                         'IeVerIdentifier',
                                         'Census_ProcessorClass',
                                         'OsSuite',
                                         'Census_IsFlightsDisabled',
                                         'Census_ChassisTypeName',
                                         'LocaleEnglishNameIdentifier',
                                         'Census_OSArchitecture',
                                         'CountryIdentifier',
                                         'Census_OSInstallLanguageIdentifier',
                                         'Census_OSInstallTypeName',
                                         'Census_OSBuildNumber',
                                         'AutoSampleOptIn',
                                         'OsVer',
                                         'SkuEdition',
                                         'UacLuaenable',
                                         'Census_OEMModelIdentifier',
                                         'Census_OSBranch',
                                         'Processor',
                                         'Census_ProcessorModelIdentifier',
                                         'Census_ActivationChannel',
                                         'IsBeta',
                                         'Census_MDC2FormFactor',
                                         'Census_OSWUAutoUpdateOptionsName',
                                         'AVProductStatesIdentifier',
                                         'Census_GenuineStateName',
                                         'Census_FirmwareManufacturerIdentifier',
                                         'Wdft_RegionIdentifier',
                                         'Census_ProcessorManufacturerIdentifier', 
                                         'OsBuildLab_0', 
                                         'OsBuildLab_1', 
                                          'OsBuildLab_2',
                                          'OsBuildLab_3'
                         ]

#X_train = reduce_mem_usage(X_train, categorical_wo_version + ['SmartScreen'])
#X_test = reduce_mem_usage(X_test, categorical_wo_version + ['SmartScreen'])

#y_train = pd.read_feather('../data/target.f')['HasDetections']
# =============================================================================
# add noise to ExistsNotSet
# ExistsNotSetの時に確率的にtargetを裏返す（publicのtest AUC: 0.537を参考）
# =============================================================================
y_train_ = []
for i,j,p in zip(y_train.tolist(), 
                 ((tr_smartscreen=='ExistsNotSet')*1).tolist(), 
                 np.random.uniform(size=len(X_train))):
    if j==1:
        if p<0.260:
            i = 1-i
    y_train_.append(i)

y_train = pd.Series(y_train_)


# =============================================================================
# add noise to RequireAdmin
# RequireAdminの時に確率的にtargetを裏返す（publicのtest AUC: 0.465を参考）
# =============================================================================
y_train_ = []
for i,j,p in zip(y_train.tolist(), 
                 ((tr_smartscreen=='RequireAdmin')*1).tolist(), 
                 np.random.uniform(size=len(X_train))):
    if j==1:
        if p<0.12:
            i = 1-i
    y_train_.append(i)

y_train = pd.Series(y_train_)

print('Noised ExistsNotSet AUC(0.537):', roc_auc_score(y_train, ((tr_smartscreen=='ExistsNotSet')*1)) )
print('Noised RequireAdmin AUC(0.465):', roc_auc_score(y_train, ((tr_smartscreen=='RequireAdmin')*1)) )

# =============================================================================
# SmartScreenTE
# =============================================================================

te = TargetEncoder(cols=['SmartScreen'], drop_invariant=False, handle_unknown='impute',
                               impute_missing=True, min_samples_leaf=100, return_df=True,
                               smoothing=1.0, verbose=1)

X_train = te.fit_transform(X_train, y_train)
X_test = te.transform(X_test)

# =============================================================================
# TargetEncodingされたSmartScreenで他のカテゴリカル変数をTargetEncodingしている。
# =============================================================================
pseudoTarget = 'SmartScreen'
alpha = 0.5
min_samples_leaf = 10
smooth_coeff = 1.0
impute = True

global_mean = (1-alpha)*X_train[pseudoTarget].astype(float).mean() + alpha*X_test[pseudoTarget].astype(float).mean()

for col in tqdm(categorical_wo_version):
    
    summary = (1-alpha)*X_train[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count']) + alpha*X_test[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count'])
    
    smoove = 1 / (1 + np.exp(-(summary['count'] - min_samples_leaf) / smooth_coeff))
    smoothing = global_mean * (1 - smoove) + summary['mean'] * smoove
    smoothing[summary['count'] == 1] = global_mean 
    
    X_train[col] = X_train[col].map(smoothing)
    X_test[col] = X_test[col].map(smoothing)
    
    if impute == True:
        X_train[col].fillna(global_mean, inplace=True)
        X_test[col].fillna(global_mean, inplace=True)

gc.collect()

X_train = reduce_mem_usage(X_train[categorical_wo_version + ['SmartScreen']])
X_test = reduce_mem_usage(X_test[categorical_wo_version + ['SmartScreen']])

# =============================================================================
# save pkl
# =============================================================================
def to_pkl_gzip(df, path):
    df.to_pickle(path)
    os.system('rm ' + path + '.gz')
    os.system('gzip ' + path)
    return


to_pkl_gzip(X_train, 'X_train_SSTE_{0}.pkl'.format(SEED))
to_pkl_gzip(X_test,  'X_test_SSTE_{0}.pkl'.format(SEED))



