#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 11 12:26:07 2019

@author: Kazuki

nejumi_solution_PublicLB0.701

"""

import numpy as np
import pandas as pd
import os, gc
from tqdm import tqdm
from category_encoders import OrdinalEncoder, TargetEncoder

import utils
utils.start(__file__)

PREF = 'f701'

def FrequencyEncoding(X_train, X_test, cols, inplace=False):
    
    X_concat = pd.concat([X_train, X_test])
    
    for col in tqdm(cols):
    
        freq = X_concat.groupby(col).size()
        freq = freq/len(X_concat)

        # カテゴリ変数を出現割合で置換
        X_train[col + '_freq'] = X_train[col].map(freq)
        X_test[col + '_freq'] = X_test[col].map(freq)
        
        if inplace == True:
            X_train.drop(col, axis=1, inplace=True)
            X_test.drop(col, axis=1, inplace=True)
            
            X_train.rename(columns={col+'_freq' : col}, inplace=True)
            X_test.rename(columns={col+'_freq' : col}, inplace=True)
    
    del X_concat
    gc.collect()
    
    return

#https://www.kaggle.com/theoviel/load-the-totality-of-the-data
dtypes = {
        'MachineIdentifier':                                    'str',
        'ProductName':                                          'str',
        'EngineVersion':                                        'str',
        'AppVersion':                                           'str',
        'AvSigVersion':                                         'str',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'str',
        'Processor':                                            'str',
        'OsVer':                                                'str',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'str',
        'OsBuildLab':                                           'str',
        'SkuEdition':                                           'str',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'str',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'str',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'str',
        'Census_DeviceFamily':                                  'str',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'str',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'str',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'str',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'str',
        'Census_InternalBatteryType':                           'str',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'str',
        'Census_OSArchitecture':                                'str',
        'Census_OSBranch':                                      'str',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'str',
        'Census_OSSkuName':                                     'str',
        'Census_OSInstallTypeName':                             'str',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'str',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'str',
        'Census_ActivationChannel':                             'str',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'str',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df

df_train = pd.read_csv('../input/train.csv.zip', dtype=dtypes)
df_test = pd.read_csv('../input/test.csv.zip', dtype=dtypes)

# SmartScreenを多少掃除しておく
replace_dict = {
'requireadmin': 'RequireAdmin', 
'OFF': 'Off',
'of': 'Off', 
'off': 'Off',  
'on': 'On', 
'ON': 'On', 
'BLOCK': 'Block', 
'warn': 'Warn', 
'Promprt': 'Prompt', 
'Promt': 'Prompt', 
'RequiredAdmin': 'RequireAdmin', 
'requireadmin': 'RequireAdmin', 
'requireAdmin': 'RequireAdmin'
}

df_train.SmartScreen.replace(replace_dict, inplace=True)
df_test.SmartScreen.replace(replace_dict, inplace=True)

y_train = df_train.HasDetections

X_train = df_train.drop(['HasDetections', 'MachineIdentifier', 'AvSigVersion', 'Census_OSVersion'], axis=1)
X_test = df_test.drop(['MachineIdentifier', 'AvSigVersion', 'Census_OSVersion'], axis=1)

del df_train, df_test

gc.collect()

versions = ['EngineVersion', 'AppVersion']

for f in versions:
    for i in range(3):
        X_train[f + '_' +str(i)] = X_train[f].apply(lambda x: str(x.split('.')[i])).astype(int)
        X_test[f + '_' +str(i)] = X_test[f].apply(lambda x: str(x.split('.')[i])).astype(int)

# ある時間スナップショットにおける相対値が重要との仮説に基づいているために、trainとtestは個別に処理している。
X_train['EngineVersion_2'] = X_train['EngineVersion_2'].rank() / len(X_train)
X_test['EngineVersion_2'] = X_test['EngineVersion_2'].rank() / len(X_test)

# drop features without information
drop_col = [
'EngineVersion_1',
'AppVersion_0',
'EngineVersion_0'
]

X_train.drop(drop_col, axis=1, inplace=True)
X_test.drop(drop_col, axis=1, inplace=True)

X_train.drop(versions, axis=1, inplace=True)
X_test.drop(versions, axis=1, inplace=True)

X_train['OsBuildLab'] = X_train['OsBuildLab'].fillna('0.0.0.0.0-0')
X_test['OsBuildLab'] = X_test['OsBuildLab'].fillna('0.0.0.0.0-0')

# この辺はPublic Kernelから
def fe(df):

    df['OsBuildLab_0'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0]).astype('str')
    df['OsBuildLab_1'] = df['OsBuildLab'].apply(lambda x: x.split('.')[1]).astype('str')
    df['OsBuildLab_2'] = df['OsBuildLab'].apply(lambda x: x.split('.')[2]).astype('str')
    df['OsBuildLab_3'] = df['OsBuildLab'].apply(lambda x: x.split('.')[3]).astype('str')

    df['primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']

    df['aspect_ratio'] = df['Census_InternalPrimaryDisplayResolutionHorizontal']/ df['Census_InternalPrimaryDisplayResolutionVertical']

    df['monitor_dims'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str) + '*' + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')
#    df['monitor_dims'] = df['monitor_dims'].astype('category')

    df['dpi'] = ((df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype('float32')**2 + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('float32')**2)**.5)/(df['Census_InternalPrimaryDiagonalDisplaySizeInInches'])

    df['dpi_square'] = df['dpi'] ** 2

    df['MegaPixels'] = (df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype('float32') * df['Census_InternalPrimaryDisplayResolutionVertical'].astype('float32'))/1e6

    df['Screen_Area'] = (df['aspect_ratio']* (df['Census_InternalPrimaryDiagonalDisplaySizeInInches']**2))/(df['aspect_ratio']**2 + 1)

    df['ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']

    df['new_num_0'] = df['Census_InternalPrimaryDiagonalDisplaySizeInInches'] / df['Census_ProcessorCoreCount']

    df['new_num_1'] = df['Census_ProcessorCoreCount'] * df['Census_InternalPrimaryDiagonalDisplaySizeInInches']
    
    df['Census_IsFlightingInternal'] = df['Census_IsFlightingInternal'].fillna(1)
    df['Census_ThresholdOptIn'] = df['Census_ThresholdOptIn'].fillna(1)
    df['Census_IsWIMBootEnabled'] = df['Census_IsWIMBootEnabled'].fillna(1)
    df['Wdft_IsGamer'] = df['Wdft_IsGamer'].fillna(0)
    
    return

fe(X_train)
fe(X_test)

utils.reduce_mem_usage(X_train)
utils.reduce_mem_usage(X_test)

# Encodingの対象とするcategorical
categorical_wo_version = ['Census_OSUILocaleIdentifier',
                         'AVProductsInstalled',
                         'Census_FirmwareVersionIdentifier',
                         'Wdft_IsGamer',
                         'Census_ThresholdOptIn',
                         'RtpStateBitfield',
                         'Census_IsSecureBootEnabled',
                         'AVProductsEnabled',
                         'HasTpm',
                         'IsProtected',
                         'Census_PrimaryDiskTypeName',
                         'PuaMode',
                         'DefaultBrowsersIdentifier',
                         'IsSxsPassiveMode',
                         'OrganizationIdentifier',
                         'Census_IsAlwaysOnAlwaysConnectedCapable',
                         'ProductName',
                         'GeoNameIdentifier',
                         'Census_IsVirtualDevice',
                         'Census_PowerPlatformRoleName',
                         'Census_IsTouchEnabled',
                         'Census_OSSkuName',
                         'OsPlatformSubRelease',
                         'Census_FlightRing',
                         'Census_OSEdition',
                         'Census_IsPortableOperatingSystem',
                         'Firewall',
                         'OsBuildLab',
                         'Census_DeviceFamily',
                         'Census_IsPenCapable',
                         'SMode',
                         'Platform',
                         'Census_IsFlightingInternal',
                         'Census_OEMNameIdentifier',
                         'Census_InternalBatteryType',
                         'OsBuild',
                         'Census_HasOpticalDiskDrive',
                         'Census_IsWIMBootEnabled',
                         'Census_OSBuildRevision',
                         'CityIdentifier',
                         'IeVerIdentifier',
                         'Census_ProcessorClass',
                         'OsSuite',
                         'Census_IsFlightsDisabled',
                         'Census_ChassisTypeName',
                         'LocaleEnglishNameIdentifier',
                         'Census_OSArchitecture',
                         'CountryIdentifier',
                         'Census_OSInstallLanguageIdentifier',
                         'Census_OSInstallTypeName',
                         'Census_OSBuildNumber',
                         'AutoSampleOptIn',
                         'OsVer',
                         'SkuEdition',
                         'UacLuaenable',
                         'Census_OEMModelIdentifier',
                         'Census_OSBranch',
                         'Processor',
                         'Census_ProcessorModelIdentifier',
                         'Census_ActivationChannel',
                         'IsBeta',
                         'Census_MDC2FormFactor',
                         'Census_OSWUAutoUpdateOptionsName',
                         'AVProductStatesIdentifier',
                         'Census_GenuineStateName',
                         'Census_FirmwareManufacturerIdentifier',
                         'Wdft_RegionIdentifier',
                         'Census_ProcessorManufacturerIdentifier', 
                         'OsBuildLab_0', 
                         'OsBuildLab_1', 
                          'OsBuildLab_2',
                          'OsBuildLab_3'
                         ]

# Frequency Encodingしている
FrequencyEncoding(X_train, X_test, categorical_wo_version + ['SmartScreen'])

utils.reduce_mem_usage(X_train)
utils.reduce_mem_usage(X_test)

# SmartScreenのみHasDetectionsでTargetEncodingする。
te = TargetEncoder(cols=['SmartScreen'], 
                   drop_invariant=False, 
                   handle_unknown='impute',
                   impute_missing=True, 
                   min_samples_leaf=100, 
                   return_df=True,
                   smoothing=1.0, verbose=1)

X_train = te.fit_transform(X_train, y_train)
X_test = te.transform(X_test)

# AVProductStatesIdentifierのFrequencyに対してTarget Encodingしている。
pseudoTarget = 'AVProductStatesIdentifier_freq'
alpha = 0.5
min_samples_leaf = 100
smooth_coeff = 1.0
impute = True

for col in tqdm(categorical_wo_version):
    
    global_mean = (1-alpha)*X_train[pseudoTarget].astype(float).mean() + alpha*X_test[pseudoTarget].astype(float).mean()
    summary = (1-alpha)*X_train[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count']) + alpha*X_test[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count'])
    
    smoove = 1 / (1 + np.exp(-(summary['count'] - min_samples_leaf) / smooth_coeff))
    smoothing = global_mean * (1 - smoove) + summary['mean'] * smoove
    smoothing[summary['count'] == 1] = global_mean 
    
    X_train[col + '_AV'] = X_train[col].map(smoothing)
    X_test[col + '_AV'] = X_test[col].map(smoothing)
    
    if impute == True:
        X_train[col + '_AV'].fillna(global_mean, inplace=True)
        X_test[col + '_AV'].fillna(global_mean, inplace=True)
        
gc.collect()



# TargetEncodingされたSmartScreenで他のカテゴリカル変数をTargetEncodingしている。
pseudoTarget = 'SmartScreen'
alpha = 0.5
min_samples_leaf = 10
smooth_coeff = 1.0
impute = True

for col in tqdm(categorical_wo_version):
    
    global_mean = (1-alpha)*X_train[pseudoTarget].astype(float).mean() + alpha*X_test[pseudoTarget].astype(float).mean()
    summary = (1-alpha)*X_train[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count']) + alpha*X_test[[col, pseudoTarget]].groupby([col])[pseudoTarget].agg(['mean', 'count'])
    
    smoove = 1 / (1 + np.exp(-(summary['count'] - min_samples_leaf) / smooth_coeff))
    smoothing = global_mean * (1 - smoove) + summary['mean'] * smoove
    smoothing[summary['count'] == 1] = global_mean 
    
    X_train[col] = X_train[col].map(smoothing)
    X_test[col] = X_test[col].map(smoothing)
    
    if impute == True:
        X_train[col].fillna(global_mean, inplace=True)
        X_test[col].fillna(global_mean, inplace=True)

gc.collect()


utils.reduce_mem_usage(X_train)
utils.reduce_mem_usage(X_test)

#col_drop = X_train.dtypes[X_train.dtypes=='O'].index.tolist()

X_train['monitor_dims'] = X_train['monitor_dims'].astype('category')
X_test['monitor_dims'] = X_test['monitor_dims'].astype('category')

X_train.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
X_test.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')






#==============================================================================
utils.end(__file__)
#utils.stop_instance()



