#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 16 09:59:20 2019

@author: Kazuki
"""

import numpy as np
import pandas as pd
import os
import utils
utils.start(__file__)


PREF = 'f007'


report = pd.read_csv('../external/Google Safe Browsing Transparency Report Data.csv')
report['WeekOf'] = report['WeekOf'].map(pd.to_datetime)

date_min = report['WeekOf'].min()
report['WeekOf'] = (report['WeekOf'] - date_min).dt.days
report['WeekOf'] = report['WeekOf'] // 7

report.rename(columns={'WeekOf': 'key'}, inplace=True)
report.columns = [c.replace(' ', '-') for c in report.columns]


tr_dt = pd.read_feather('../data/train_datetime.f')
te_dt = pd.read_feather('../data/test_datetime.f')

def fe(df):
    df['AvSigVersion'] = (df['AvSigVersion'] - date_min).dt.days
    df['AvSigVersion'] = df['AvSigVersion'] // 7
    df.rename(columns={'AvSigVersion': 'key'}, inplace=True)
    
    df = pd.merge(df[['key']], report, on='key', how='left')
    del df['key']
    df = df.rank(pct=True)
    utils.reduce_mem_usage(df)
    
    return df

tr = fe(tr_dt)
te = fe(te_dt)

# output
tr.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
te.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')



utils.end(__file__)

