#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb  8 14:07:28 2019

@author: kazuki.onodera
"""

import numpy as np
import pandas as pd
import gc, os
from glob import glob
from multiprocessing import cpu_count, Pool
from itertools import combinations

import utils

PREF = 'f014'

funcs = ['min', 'mean', 'max', 'std', 'skew']

num_agg = {
         'RtpStateBitfield': funcs,
         'AVProductsInstalled': funcs,
         'AVProductsEnabled': funcs,
         'UacLuaenable': funcs,
         'Census_ProcessorCoreCount': funcs,
         'Census_PrimaryDiskTotalCapacity': funcs,
         'Census_SystemVolumeTotalCapacity': funcs,
         'Census_TotalPhysicalRAM': funcs,
         'Census_InternalPrimaryDiagonalDisplaySizeInInches': funcs,
         'Census_InternalPrimaryDisplayResolutionHorizontal': funcs,
         'Census_InternalPrimaryDisplayResolutionVertical': funcs,
         'Census_InternalBatteryNumberOfCharges': funcs,
         'Census_OSBuildNumber': funcs,
         'Census_OSBuildRevision': funcs,
         
         }

COL_CAT = [
        'ProductName',
         'EngineVersion',
         'AppVersion',
         'AvSigVersion',
         'DefaultBrowsersIdentifier',
         'AVProductStatesIdentifier',
         'CountryIdentifier',
         'CityIdentifier',
         'OrganizationIdentifier',
         'GeoNameIdentifier',
         'LocaleEnglishNameIdentifier',
         'Platform',
         'Processor',
         'OsVer',
         'OsPlatformSubRelease',
         'OsBuildLab',
         'SkuEdition',
         'IeVerIdentifier',
         'SmartScreen',
         'Census_MDC2FormFactor',
         'Census_DeviceFamily',
         'Census_OEMNameIdentifier',
         'Census_OEMModelIdentifier',
         'Census_ProcessorManufacturerIdentifier',
         'Census_ProcessorModelIdentifier',
         'Census_ProcessorClass',
         'Census_PrimaryDiskTypeName',
         'Census_ChassisTypeName',
         'Census_PowerPlatformRoleName',
         'Census_InternalBatteryType',
         'Census_OSVersion',
         'Census_OSArchitecture',
         'Census_OSBranch',
         'Census_OSEdition',
         'Census_OSSkuName',
         'Census_OSInstallTypeName',
         'Census_OSInstallLanguageIdentifier',
         'Census_OSUILocaleIdentifier',
         'Census_OSWUAutoUpdateOptionsName',
         'Census_GenuineStateName',
         'Census_ActivationChannel',
         'Census_FlightRing',
         'Census_FirmwareManufacturerIdentifier',
         'Census_FirmwareVersionIdentifier',
         'Wdft_RegionIdentifier',
         
         'OsBuild',
         'OsSuite',
         
         'OsBuildLab_major',
         'OsBuildLab_minor',
         'OsBuildLab_build',
         'OsBuildLab_architecture',
         ]

use_cat = ['SmartScreen', 'AVProductStatesIdentifier', 'Census_OSInstallTypeName',
           'Census_ProcessorModelIdentifier', 'Census_OEMModelIdentifier', 
           'Census_ActivationChannel']

li = []
comb = list(combinations(COL_CAT, 2))
for i,j in comb:
    if i in use_cat or j in use_cat:
        li.append([i,j])
comb = li

def multi_tr(args):
    
    cat, outpath = args
    
    tbl = tr.groupby(cat).agg(num_agg)
    tbl.columns = [f'{"-".join(cat)}_{i}_{j}' for i,j in tbl.columns]
    tbl.reset_index(inplace=True)
    
    
    tr_f = pd.merge(tr[[cat]], tbl, on=cat, how='left')
    del tr_f[cat]
    
    utils.reduce_mem_usage(tr_f)
    tr_f.add_prefix(PREF+'_').to_feather(outpath)
    
    return



def multi_te(args):
    
    cat, outpath = args
    
    tbl = te.groupby(cat).agg(num_agg)
    tbl.columns = [f'{"-".join(cat)}_{i}_{j}' for i,j in tbl.columns]
    tbl.reset_index(inplace=True)
    
    
    te_f = pd.merge(te[[cat]], tbl, on=cat, how='left')
    del te_f[cat]
    
    utils.reduce_mem_usage(te_f)
    te_f.add_prefix(PREF+'_').to_feather(outpath)
    
    return


# =============================================================================
# main
# =============================================================================
if __name__ == "__main__":
    utils.start(__file__)
    
    os.system(f'rm -rf ../data/{PREF}')
    os.system(f'mkdir  ../data/{PREF}')
    
    # train
    tr = pd.read_feather('../data/train.f')
    
    argss = []
    for i,c in enumerate(comb):
        argss.append([c, f'../data/{PREF}/train_{PREF}_{c}.f'])
    
    pool = Pool( 10 )
    pool.map(multi_tr, argss)
    pool.close()
    del tr; gc.collect()
    
#    df = pd.concat([pd.read_feather(f) for f in sorted(glob(f'../data/train_{PREF}*'))], axis=1)
#    df.to_feather(f'../data/train_{PREF}.f')
#    del df; gc.collect()
    
    
    # test
    te = pd.read_feather('../data/test.f')
    
    argss = []
    for i,c in enumerate(COL_CAT):
        argss.append([c, f'../data/{PREF}/test_{PREF}_{c}.f'])
    
    pool = Pool( 10 )
    pool.map(multi_te, argss)
    pool.close()
    del te; gc.collect()
    
#    df = pd.concat([pd.read_feather(f) for f in sorted(glob(f'../data/test_{PREF}*'))], axis=1)
#    df.to_feather(f'../data/test_{PREF}.f')
#    del df; gc.collect()
    
    
    utils.end(__file__)
