# -*- coding: utf-8 -*-
"""
Created on Sun Feb 17 12:10:01 2019

@author: luxiaotong
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random
import seaborn as sns

###########Part1: Load and Descriptive Analysis

#LoadData
train_df = pd.read_csv('train.csv', iterator = True)
test_df = pd.read_csv('test.csv',iterator = True)
#Approach1: Chunks, All Data
data=pd.DataFrame()
i=0
while i<8921483:#RowCount
    now_df=train_df.get_chunk(400000)
    data=pd.concat([data,now_df])
    i+=400000
#Approach2(Current): Load samples
train_sample=train_df.get_chunk(200000)

#GetColumnsFunction
def get_columns(group):
    column_group={
   'ProductName':4,
   'EngineVersion':4,
   'AppVersion':4,
   'AvSigVersion':4,
   'IsBeta':1,
   'RtpStateBitfield':4,
   'IsSxsPassiveMode':1,
   'DefaultBrowsersIdentifier':4,
   'AVProductStatesIdentifier':4,
   'AVProductsInstalled':4,
   'AVProductsEnabled':4,
   'HasTpm':1,
   'CountryIdentifier':4,
   'CityIdentifier':4,
   'OrganizationIdentifier':4,
   'GeoNameIdentifier':4,
   'LocaleEnglishNameIdentifier':4,
   'Platform':4,
   'Processor':4,
   'OsVer':4,
   'OsBuild':4,
   'OsSuite':4,
   'OsPlatformSubRelease':4,
   'OsBuildLab':4,
   'SkuEdition':4,
   'IsProtected':1,
   'AutoSampleOptIn':1,
   'PuaMode':1,
   'SMode':1,
   'IeVerIdentifier':4,
   'SmartScreen':4,
   'Firewall':1,
   'UacLuaenable':1,
   'Census_MDC2FormFactor':3,
   'Census_DeviceFamily':3,
   'Census_OEMNameIdentifier':2,#Was 3
   'Census_OEMModelIdentifier':2,#Was 3
   'Census_ProcessorCoreCount':3,
   'Census_ProcessorManufacturerIdentifier':3,
   'Census_ProcessorModelIdentifier':2,
   'Census_ProcessorClass':3,
   'Census_PrimaryDiskTotalCapacity':2,
   'Census_PrimaryDiskTypeName':3,
   'Census_SystemVolumeTotalCapacity':2,
   'Census_HasOpticalDiskDrive':1,
   'Census_TotalPhysicalRAM':2,#Was 3
   'Census_ChassisTypeName':3,
   'Census_InternalPrimaryDiagonalDisplaySizeInInches':2,
   'Census_InternalPrimaryDisplayResolutionHorizontal':2,
   'Census_InternalPrimaryDisplayResolutionVertical':2,
   'Census_PowerPlatformRoleName':3,
   'Census_InternalBatteryType':3,
   'Census_InternalBatteryNumberOfCharges':3,#Was 2
   'Census_OSVersion':3,#Was 2
   'Census_OSArchitecture':3,
   'Census_OSBranch':3,
   'Census_OSBuildNumber':3,
   'Census_OSBuildRevision':3,
   'Census_OSEdition':3,
   'Census_OSSkuName':3,
   'Census_OSInstallTypeName':3,
   'Census_OSInstallLanguageIdentifier':3,
   'Census_OSUILocaleIdentifier':3,
   'Census_OSWUAutoUpdateOptionsName':3,
   'Census_IsPortableOperatingSystem':1,
   'Census_GenuineStateName':3,
   'Census_ActivationChannel':3,
   'Census_IsFlightingInternal':1,
   'Census_IsFlightsDisabled':1,
   'Census_FlightRing':3,
   'Census_ThresholdOptIn':1,
   'Census_FirmwareManufacturerIdentifier':2,
   'Census_FirmwareVersionIdentifier':2,
   'Census_IsSecureBootEnabled':1,
   'Census_IsWIMBootEnabled':1,
   'Census_IsVirtualDevice':1,
   'Census_IsTouchEnabled':1,
   'Census_IsPenCapable':1,
   'Census_IsAlwaysOnAlwaysConnectedCapable':1,
   'Wdft_IsGamer':1,
   'Wdft_RegionIdentifier':4
    }
    lst=[]
    for i,v in column_group.items():
        if v==group:
            lst.append(i)
    return lst
#NaPercentageFunction
def na_percent(df,columns):
    r=df.shape[0]
    na=df[columns].isnull().sum()
    return na/r

#Group1: Binary values
sample_group1=train_sample[get_columns(1)]
#na
na_percent(sample_group1,get_columns(1))
#distribution
def get_binary_dis(df,columns):
    d={}
    for c in columns:
        dis=pd.value_counts(df[c].tolist())
        temp={c:dict(dis)}
        d.update(temp)
    return d
get_binary_dis(sample_group1,get_columns(1))
#WeirdValues
sample_group1['PuaMode']=sample_group1['PuaMode'].replace('on',1)#'on' to 1
sample_group1.loc[sample_group1['UacLuaenable']>1,'UacLuaenable']=np.nan#>1 equals to na

#Group2: Numeric values
sample_group2=train_sample[get_columns(2)]
#na
na_percent(sample_group2,get_columns(2))
#distrubution
def get_numeric_dis(df,columns):
    for c in columns:
        try:
            plt.figure()
            plt.subplot(121)
            plt.title(c)
            plt.hist(np.array(df[c].dropna()),bins=50)
            plt.subplot(122)
            plt.boxplot(np.array(df[c].dropna()))
        except Exception as e:
            print('column '+c+' error, message: '+str(e))
get_numeric_dis(sample_group2,get_columns(2))
#WeirdValues
#newlst=[]
#for i in map(lambda x: x.replace('.',''),sample_group2['Census_OSVersion'].tolist()):
#    newlst.append(i)
#sample_group2['Census_OSVersion']=newlst
#sample_group2['Census_OSVersion']=sample_group2['Census_OSVersion'].astype(float)
    
###########Part2: Processing: Delete Features, Outliers etc.
#Group1: Binary values
#Drop
drop_features=['IsBeta', 
               'AutoSampleOptIn',
               'PuaMode',
               'Census_IsFlightingInternal', 
               'Census_IsFlightsDisabled', 
               'Census_ThresholdOptIn', 
               'Census_IsWIMBootEnabled'
               ]
sample_group1=sample_group1.drop(drop_features,axis=1)
#EDA-HasDetections
##DataPreparation
y=train_sample['HasDetections']
sample_group1y=sample_group1.copy()
sample_group1y['HasDetections']=y
sample_group1y=sample_group1y.dropna()#for ploting purpuose
sample_group1y=sample_group1y.reset_index()#for sampling purpose
sample_group1y=sample_group1y.drop('index',axis=1)
##EDAFunction
###DistributionAgainstY
def plot_binary_y(df,column):
    plt.figure()
    sns.countplot(x=column,hue='HasDetections',data=df)
    plt.title(column)
new_binary_columns=list(set(get_columns(1))-set(drop_features))
for i in new_binary_columns:
    plot_binary_y(sample_group1y,i)
###CorrPlot
plt.figure(figsize=(15,15))
sns.heatmap(sample_group1y.corr(),annot=True)
plt.savefig('corr_g1.jpg')

#Group2: Numeric values
#Change Data Type
##'Census_OSVersion': Use it as categorical feature
##'Census_InternalBatteryNumberOfCharges': Use it as categorical feature
#battery_feature=sample_group2['Census_InternalBatteryNumberOfCharges']
#cate1bool=battery_feature==max(battery_feature)
#cate2bool=(battery_feature<max(battery_feature)) & (battery_feature>0)
#battery_feature[cate1bool]=2
#battery_feature[cate2bool]=1
#sample_group2['Census_InternalBatteryNumberOfCharges']=battery_feature
#Outliers
##ReplaceOrObservePercentageOurlierFunction
def quantile_outlier(df,column,percent,replace_bool=0):
    upper=df[column].quantile(percent)
    lower=df[column].quantile(1-percent)
    out_u=df[column]>upper
    out_l=df[column]<lower
    out=out_u | out_l
    if replace_bool==0:
        return df[column][out]
    else:
        df[column][out_u]=upper
        df[column][out_l]=lower
##ObserveReplacingDifferentPercentageOurliersDistributionFunction
def observe_outliers_g2(column,start_percent,step):
    p=start_percent
    for i in range(10):
        df=sample_group2.copy()#initialize
        quantile_outlier(df,column,p,1)
        columns=[column]
        print('try '+str(i)+' cut percentile '+str(p))
        get_numeric_dis(df,columns)
        p-=step
observe_outliers_g2('Census_PrimaryDiskTotalCapacity',1,0.01)#Suggesting using percentile 0.99
observe_outliers_g2('Census_SystemVolumeTotalCapacity',1,0.01)#Suggesting using percentile 0.99
observe_outliers_g2('Census_InternalPrimaryDiagonalDisplaySizeInInches',1,0.01)#Suggesting using percentile 0.93
observe_outliers_g2('Census_InternalPrimaryDisplayResolutionHorizontal',1,0.01)#Suggesting using percentile 0.98
observe_outliers_g2('Census_InternalPrimaryDisplayResolutionVertical',1,0.005)#Suggesting using percentile 0.98
observe_outliers_g2('Census_TotalPhysicalRAM',1,0.01)#Suggesting using percentile 0.99
quantile_outlier(sample_group2,'Census_PrimaryDiskTotalCapacity',0.99,1)
quantile_outlier(sample_group2,'Census_SystemVolumeTotalCapacity',0.99,1)
quantile_outlier(sample_group2,'Census_InternalPrimaryDiagonalDisplaySizeInInches',0.93,1)
quantile_outlier(sample_group2,'Census_InternalPrimaryDisplayResolutionHorizontal',0.98,1)
quantile_outlier(sample_group2,'Census_InternalPrimaryDisplayResolutionVertical',0.98,1)      
quantile_outlier(sample_group2,'Census_TotalPhysicalRAM',0.99,1)  
#EDA-HasDetections
##DataPreparation
y=train_sample['HasDetections']
sample_group2y=sample_group2.copy()
sample_group2y['HasDetections']=y
sample_group2y=sample_group2y.dropna()#for ploting purpuose
sample_group2y=sample_group2y.reset_index()#for sampling purpose
sample_group2y=sample_group2y.drop('index',axis=1)
##EDAFunction
###DistributionAgainstY
def plot_numeric_y_hist(column):
    ##sample 10000 from both positive and negative samples (due to large amount)
    pos_ind=sample_group2y[sample_group2y['HasDetections']==1].index.tolist()
    neg_ind=sample_group2y[sample_group2y['HasDetections']==0].index.tolist()
    pos_rows=random.sample(pos_ind,10000)
    pos_rows=sample_group2y[column].iloc[pos_rows].tolist()
    neg_rows=random.sample(neg_ind,10000)
    neg_rows=sample_group2y[column].iloc[neg_rows].tolist()
    sample_rows=np.reshape(np.array(pos_rows+neg_rows),(10000,2),order='F')
    plt.figure()
    plt.hist(sample_rows,label=['Detection_1','Detection_0'],bins=50)
    plt.legend()
    plt.title(column)
    plt.xlabel('value')
    plt.ylabel('freq')
def plot_numeric_y_kde(df,column):
    plt.figure()
    sns.distplot(df[column][df['HasDetections']==1],color='r',label='Detection_1')
    sns.distplot(df[column][df['HasDetections']==0],color='b',label='Detection_0')
    plt.title(column)
new_number_columns=list(set(get_columns(2))-set(['Census_InternalBatteryNumberOfCharges','Census_OSVersion']))
for i in new_number_columns:
    plot_numeric_y_hist(i)
for i in new_number_columns:
    plot_numeric_y_kde(sample_group2y,i)
###CorrPlot
plt.figure(figsize=(15,15))
sns.heatmap(sample_group2y.corr(),annot=True)
plt.savefig('corr_g2.jpg')
#Binning & Normalization
##Binning-QuantileBased
bin_features=['Census_FirmwareManufacturerIdentifier','Census_InternalPrimaryDiagonalDisplaySizeInInches',
              'Census_InternalPrimaryDisplayResolutionHorizontal','Census_InternalPrimaryDisplayResolutionVertical',
              'Census_PrimaryDiskTotalCapacity','Census_TotalPhysicalRAM']
def observe_binning_number(df,column,bins,replace_bool=0):
    bined=pd.qcut(df[column],bins,duplicates='drop',retbins=True)
    df_b=bined[0]
    df_b=pd.DataFrame(df_b,columns=[column])
    df_b.loc[:,'HasDetections']=sample_group2y['HasDetections']
    plot_binary_y(df_b,column)
    if replace_bool==0:
        return print(bined[1])
    else:
        df[column]=df_b[column]
for i in [2,3,4,5,6,7,8,9]:
    print(i)
    observe_binning_number(sample_group2y,column=bin_features[5],bins=i)
observe_binning_number(sample_group2y,column=bin_features[0],bins=9,replace_bool=1)
observe_binning_number(sample_group2y,column=bin_features[1],bins=7,replace_bool=1)
observe_binning_number(sample_group2y,column=bin_features[2],bins=2,replace_bool=1)
observe_binning_number(sample_group2y,column=bin_features[3],bins=5,replace_bool=1)
observe_binning_number(sample_group2y,column=bin_features[4],bins=4,replace_bool=1)
observe_binning_number(sample_group2y,column=bin_features[5],bins=3,replace_bool=1)
#Normalization
non_bin_features=list(set(get_columns(2))-set(bin_features))
sample_group2y[non_bin_features]=(sample_group2y[non_bin_features]-sample_group2y[non_bin_features].mean())/sample_group2y[non_bin_features].std()
