# -*- coding: utf-8 -*-
"""
Created on Fri Mar  8 11:25:51 2019

@author: luxiaotong
"""

from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import Imputer
from sklearn.model_selection import cross_val_score
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#LoadData
train_df = pd.read_csv('train.csv', iterator = True)
train_sample=train_df.get_chunk(20000)
def get_columns(group):
    column_group={
   'ProductName':4,
   'EngineVersion':4,
   'AppVersion':4,
   'AvSigVersion':4,
   'IsBeta':1,
   'RtpStateBitfield':4,
   'IsSxsPassiveMode':1,
   'DefaultBrowsersIdentifier':4,
   'AVProductStatesIdentifier':4,
   'AVProductsInstalled':4,
   'AVProductsEnabled':4,
   'HasTpm':1,
   'CountryIdentifier':4,
   'CityIdentifier':4,
   'OrganizationIdentifier':4,
   'GeoNameIdentifier':4,
   'LocaleEnglishNameIdentifier':4,
   'Platform':4,
   'Processor':4,
   'OsVer':4,
   'OsBuild':4,
   'OsSuite':4,
   'OsPlatformSubRelease':4,
   'OsBuildLab':4,
   'SkuEdition':4,
   'IsProtected':1,
   'AutoSampleOptIn':1,
   'PuaMode':1,
   'SMode':1,
   'IeVerIdentifier':4,
   'SmartScreen':4,
   'Firewall':1,
   'UacLuaenable':1,
   'Census_MDC2FormFactor':3,
   'Census_DeviceFamily':3,
   'Census_OEMNameIdentifier':2,#Was 3
   'Census_OEMModelIdentifier':2,#Was 3
   'Census_ProcessorCoreCount':3,
   'Census_ProcessorManufacturerIdentifier':3,
   'Census_ProcessorModelIdentifier':2,
   'Census_ProcessorClass':3,
   'Census_PrimaryDiskTotalCapacity':2,
   'Census_PrimaryDiskTypeName':3,
   'Census_SystemVolumeTotalCapacity':2,
   'Census_HasOpticalDiskDrive':1,
   'Census_TotalPhysicalRAM':2,#Was 3
   'Census_ChassisTypeName':3,
   'Census_InternalPrimaryDiagonalDisplaySizeInInches':2,
   'Census_InternalPrimaryDisplayResolutionHorizontal':2,
   'Census_InternalPrimaryDisplayResolutionVertical':2,
   'Census_PowerPlatformRoleName':3,
   'Census_InternalBatteryType':3,
   'Census_InternalBatteryNumberOfCharges':3,#Was 2
   'Census_OSVersion':3,#Was 2
   'Census_OSArchitecture':3,
   'Census_OSBranch':3,
   'Census_OSBuildNumber':3,
   'Census_OSBuildRevision':3,
   'Census_OSEdition':3,
   'Census_OSSkuName':3,
   'Census_OSInstallTypeName':3,
   'Census_OSInstallLanguageIdentifier':3,
   'Census_OSUILocaleIdentifier':3,
   'Census_OSWUAutoUpdateOptionsName':3,
   'Census_IsPortableOperatingSystem':1,
   'Census_GenuineStateName':3,
   'Census_ActivationChannel':3,
   'Census_IsFlightingInternal':1,
   'Census_IsFlightsDisabled':1,
   'Census_FlightRing':3,
   'Census_ThresholdOptIn':1,
   'Census_FirmwareManufacturerIdentifier':2,
   'Census_FirmwareVersionIdentifier':2,
   'Census_IsSecureBootEnabled':1,
   'Census_IsWIMBootEnabled':1,
   'Census_IsVirtualDevice':1,
   'Census_IsTouchEnabled':1,
   'Census_IsPenCapable':1,
   'Census_IsAlwaysOnAlwaysConnectedCapable':1,
   'Wdft_IsGamer':1,
   'Wdft_RegionIdentifier':4
    }
    lst=[]
    for i,v in column_group.items():
        if v==group:
            lst.append(i)
    return lst
y=train_sample['HasDetections']
x=train_sample[get_columns(1)]
x=x.drop('PuaMode',axis=1)
labels=list(x.columns)
x = Imputer().fit_transform(x)


#Model Train Initialization
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1, train_size=0.7)
clf0 = RandomForestClassifier()
clf0.fit(x_train,y_train)
y_predict=clf0.predict(x_test)

#Model Evaluation and Adjusting
clf0.set_params(min_samples_split=3)
def eval_model(model):
    model.fit(x_train,y_train)
    y_predict=model.predict(x_test)
    auc=metrics.roc_auc_score(y_test,y_predict)

    importance=model.feature_importances_
    d=dict(zip(labels,importance))
    #sort dict and make list
    sort_d=sorted(d.items(),key=lambda x:x[1],reverse=True)
    #labels_sort=[sort_d[sort_d.index(i)][0] for i in sort_d]
    labels_sort=[i[0] for i in sort_d]
    #importance_sort=[sort_d[sort_d.index(i)][1] for i in sort_d]
    importance_sort=[i[1] for i in sort_d]
    #plot
    y_pos=np.arange(len(labels),0,-1)
    plt.barh(y_pos,importance_sort,alpha=0.4)
    plt.yticks(y_pos,labels_sort)
    plt.title(str(model))

    return 'auc: '+str(auc)
eval_model(clf0)