#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar  5 13:02:06 2019

@author: kazuki.onodera
"""


import numpy as np
import pandas as pd
from tqdm import tqdm
import os
import utils
utils.start(__file__)
#==============================================================================


PREF = 'f022'

categorical_w_version = [
                         
                         'EngineVersion',
                         'AppVersion',
                         'AvSigVersion',
                         'Census_OSVersion',
                         
                         ]

cat1 = 'AvSigVersion'
cat2 = 'Census_OSVersion'

def fe(input_path, output_path):
    """
    input_path  = '../data/train.f'
    output_path = '../data/train_{PREF}.f'
    """
    
    print('loading...', input_path)
    base = pd.read_feather(input_path)[categorical_w_version]
    
    feature = pd.DataFrame(index=base.index)
    cat = [cat1, cat2]
    cat_size = base.groupby(cat).size()
    
    for c in tqdm(categorical_w_version):
        if c in [cat1, cat2]:
            continue
        
        col = cat + [c]
        df = base.groupby(col).size() / cat_size
        df = df.reset_index()
        
        feature[f'{PREF}_{c}-in-{"-".join(cat)}'] = pd.merge(base[col], df, on=col, how='left')[0]
    
    print('writing...', output_path)
    feature.to_feather(output_path)
    
    return


#tr = pd.read_feather('../data/train.f')[categorical_wo_version+['Census_OSVersion']]
#te = pd.read_feather('../data/test.f')[categorical_wo_version+['Census_OSVersion']]

fe('../data/train.f', f'../data/train_{PREF}.f')
fe('../data/test.f',  f'../data/test_{PREF}.f')


#==============================================================================
utils.end(__file__)
#utils.stop_instance()

