#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 30 15:10:51 2019

@author: kazuki.onodera

external data were downloaded below links

https://www.kaggle.com/cdeotte/malware-timestamps
https://www.kaggle.com/cdeotte/malware-timestamps-2

"""

import numpy as np
import pandas as pd
from datetime import datetime
#from multiprocessing import cpu_count, Pool

import utils


PREF = 'f003'


col_cat = [
         'ProductName',
         'EngineVersion',
         'AppVersion',
         'AvSigVersion',
         'Platform',
         'Processor',
         'OsVer',
         'OsPlatformSubRelease',
         'OsBuildLab',
         'SkuEdition',
         'PuaMode',
         'SmartScreen',
         'Census_MDC2FormFactor',
         'Census_DeviceFamily',
         'Census_ProcessorClass',
         'Census_PrimaryDiskTypeName',
         'Census_ChassisTypeName',
         'Census_PowerPlatformRoleName',
         'Census_InternalBatteryType',
         'Census_OSVersion',
         'Census_OSArchitecture',
         'Census_OSBranch',
         'Census_OSEdition',
         'Census_OSSkuName',
         'Census_OSInstallTypeName',
         'Census_OSWUAutoUpdateOptionsName',
         'Census_GenuineStateName',
         'Census_ActivationChannel',
         'Census_FlightRing',
         ]

def fe(df, path):
    
    
    # time delta
    df['AvSigVersion-m-Census_OSVersion'] = (df['AvSigVersion']     - df['Census_OSVersion']).dt.seconds
    df['Census_OSVersion-m-OsBuildLab']   = (df['Census_OSVersion'] - df['OsBuildLab']).dt.seconds
    
    # min max
    col = ['AvSigVersion', 'Census_OSVersion', 'OsBuildLab']
    df['date_min'] = df[col].min(1)
    df['date_max'] = df[col].max(1)
    df['date_max-m-min'] = (df['date_max']     - df['date_min']).dt.seconds
    
    # from max
    d_max = df[col].max().max()
    df['max-m-AvSigVersion']     = (d_max - df['AvSigVersion']).dt.seconds
    df['max-m-Census_OSVersion'] = (d_max - df['Census_OSVersion']).dt.seconds
    df['max-m-OsBuildLab']       = (d_max - df['OsBuildLab']).dt.seconds
    
    
    # save dt
    df[['AvSigVersion', 'Census_OSVersion', 'OsBuildLab']].to_feather(path)
    
    # into int64 and rank
    for c in ['AvSigVersion', 'Census_OSVersion', 'OsBuildLab', 'date_min', 'date_max']:
        df[c]     = df[c].astype(np.int64) // 10**9
        df.loc[df[c]<0, c] = np.nan
        df[c] = df[c].rank(pct=True)
    
    utils.reduce_mem_usage(df)
    
    return

# =============================================================================
# main
# =============================================================================
if __name__ == "__main__":
    utils.start(__file__)
    
    tr = pd.read_feather('../data/train.f')[col_cat]
    te = pd.read_feather('../data/test.f')[col_cat]
    
    tr_f = pd.DataFrame(index=tr.index)
    te_f = pd.DataFrame(index=te.index)
    
    # AS timestamp
    datedictAS = np.load('../external/AvSigVersionTimestamps.npy')[()]
    tr_f['AvSigVersion'] = tr['AvSigVersion'].map(datedictAS)
    te_f['AvSigVersion'] = te['AvSigVersion'].map(datedictAS)
    
    # OS timestamp
    datedictOS = np.load('../external/OSVersionTimestamps.npy')[()]
    tr_f['Census_OSVersion'] = tr['Census_OSVersion'].map(datedictOS)
    te_f['Census_OSVersion'] = te['Census_OSVersion'].map(datedictOS)
    
    # BL timestamp
    def convert(x):
        try:
            d = datetime.strptime(x.split('.')[4],'%y%m%d-%H%M')
        except:
            d = np.nan
        return d
    
    tr_f['OsBuildLab'] = tr['OsBuildLab'].map(convert)
    te_f['OsBuildLab'] = te['OsBuildLab'].map(convert)
    
    
    fe(tr_f, '../data/train_datetime.f')
    fe(te_f, '../data/test_datetime.f')
    
    # output
    tr_f.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
    te_f.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')
    
    
    
    utils.end(__file__)
    
    
    