# -*- coding: utf-8 -*-
"""
Created on Mon Feb 18 12:58:58 2019

@author: Yufeng Zhang
"""

import pandas as pd
import numpy as np

def summary(x):
    
    df = pd.read_hdf('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\value_group'+str(x)+'.h5',key='g'+str(x))
    result = df.describe()
    result.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\summary_g'+str(x)+'.xlsx')

#summary(1)
#summary(2)
#summary(3)
#summary(4)


#NaPercentageFunction
def na_percent(df,columns):
    r=df.shape[0]
    na=df[columns].isnull().sum()
    return na/r
'''
#Group1: Binary values
'''

df = pd.read_hdf('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\value_group1.h5',key='g1')

#WeirdValues
df['PuaMode']=df['PuaMode'].replace('on',1)#'on' to 1
df['PuaMode']=df['PuaMode'].replace('audit',0)#'audit' to 0
df.loc[df['UacLuaenable']>1,'UacLuaenable']=np.nan#>1 equals to na

#na
na_result = dict()
for i in df.columns:
    temp = na_percent(df,i)
    na_result[i] = temp
    
#distribution
def get_binary_dis(df,columns):
    d = {}
    for c in columns:
        dis = pd.value_counts(df[c])
        d[c] = dict(dis)
    return d
binary_dis = get_binary_dis(df,df.columns)

na_result_g1 = pd.DataFrame(na_result,index=['na_percent']).T
na_result_g1.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\na_result_g1.xlsx')
binary_dis_g1 = pd.DataFrame(binary_dis).T
binary_dis_g1.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\binary_dis_g1.xlsx')


'''
#Group2: Numeric values
'''

df = pd.read_hdf('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\value_group2.h5',key='g2')

#na
na_result = dict()
for i in df.columns:
    temp = na_percent(df,i)
    na_result[i] = temp
na_result_g2 = pd.DataFrame(na_result,index=['na_percent']).T
na_result_g2.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\na_result_g2.xlsx')

#distrubution
import matplotlib.pyplot as plt
def get_numeric_dis(df,columns):
    for c in columns:
        try:
            plt.figure()
            plt.subplot(121)
            plt.title(c)
            plt.hist(np.array(df[c].dropna()),bins=50)
            plt.subplot(122)
            plt.boxplot(np.array(df[c].dropna()))
        except Exception as e:
            print('column '+c+' error, message: '+str(e))
get_numeric_dis(df,df.columns)

dis_g2 = df[['Census_FirmwareManufacturerIdentifier',
       'Census_FirmwareVersionIdentifier',
       'Census_InternalBatteryNumberOfCharges',
       'Census_InternalPrimaryDiagonalDisplaySizeInInches',
       'Census_InternalPrimaryDisplayResolutionHorizontal',
       'Census_InternalPrimaryDisplayResolutionVertical', 
       'Census_PrimaryDiskTotalCapacity', 'Census_ProcessorModelIdentifier',
       'Census_SystemVolumeTotalCapacity']].astype(float).describe()
dis_g2.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\dis_g2.xlsx')
'''  
#WeirdValues
    newlst=[]
    for i in map(lambda x: x.replace('.',''),sample_group2['Census_OSVersion'].tolist()):
        newlst.append(i)
    sample_group2['Census_OSVersion']=newlst
    sample_group2['Census_OSVersion']=sample_group2['Census_OSVersion'].astype(float)
'''

'''
Group3 Categorical Features
'''
df = pd.read_hdf('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\value_group3.h5',key='g3')

#na
na_result = dict()
for i in df.columns:
    temp = na_percent(df,i)
    na_result[i] = temp
na_result_g3 = pd.DataFrame(na_result,index=['na_percent']).T
na_result_g3.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\na_result_g3.xlsx')




'''
Group4 Categorical Features
'''
df = pd.read_hdf('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\value_group4.h5',key='g4')

#na
na_result = dict()
for i in df.columns:
    temp = na_percent(df,i)
    na_result[i] = temp
na_result_g4 = pd.DataFrame(na_result,index=['na_percent']).T
na_result_g4.to_excel('D:\\pycodes\\kaggle\\microsoft-malware-prediction\\na_result_g4.xlsx')

