import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split,cross_val_score
from sklearn.metrics import accuracy_score,confusion_matrix,roc_auc_score,mean_squared_error,roc_curve
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

start = time.time()
test_df = pd.read_csv('test.csv', dtype=dtypes)
print('Test data loaded in '+str(round(time.time()-start,2))+' seconds.\n')

column_names=list(test_df.columns.values)
print('Columns originally in Test: '+str(len(column_names))+'\n')

KeepColumns = ['AppVersion','EngineVersion','AVProductStatesIdentifier','Census_OSInstallTypeName',' Census_ActivationChannel', 'AVProductsInstalled', 'Census_PrimaryDiskTotalCapacity', 'LocaleEnglishNameIdentifier', 'Census_TotalPhysicalRAM', 'Census_InternalPrimaryDiagonalDisplaySizeInInches', 'Wdft_RegionIdentifier', 'Wdft_IsGamer', 'Census_IsVirtualDevice', 'RtpStateBitfield', 'Census_SystemVolumeTotalCapacity', 'Census_GenuineStateName', 'Census_OSUILocaleIdentifier', 'Census_OSBuildRevision','Census_InternalPrimaryDisplayResolutionVertical', 'IeVerIdentifier']
print(len(KeepColumns))

for col in column_names:
    if col not in KeepColumns:
        test_df.drop(col, axis=1, inplace=True)

test_column_names=list(test_df.columns.values)
print('Columns after removing in Test: '+str(len(train_column_names))+'\n')
print('Columns left: ',test_column_names,'\n')

def fill_with_max(df):
    x = df.value_counts().argmax()
    df.fillna(x,inplace = True)

column_names=list(test_df.columns.values)
for column in column_names:
    fill_with_max(train_df[column])

test_df_category = test_df.select_dtypes(include='category')

start = time.time()
for i in range(len(test_df_category.columns)):
    labelencoder_X_i = LabelEncoder()
    test_df[test_df_category.columns[i]] = labelencoder_X_i.fit_transform(test_df[test_df_category.columns[i]])
    print(test_df.columns[i])
print(time.time()-start)

for i in range(len(test_df_category.columns)):
    test_df[test_df_category.columns[i]] = test_df[test_df_category.columns[i]].astype(np.int8)

print(test_df.info())
print(test_df.columns)
print(len(test_df.columns))

# start = time.time()
# onehotencoder = OneHotEncoder(categorical_features=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],sparse = False)
# train_df_final_coded = onehotencoder.fit_transform(train_df_final)
# print(time.time()-start)

#train_df.to_csv('ModelTrain.csv', index=False)

start = time.time()
train_df = pd.read_csv('ModelTrain.csv')
#,dtype=dtypes
print('Train data loaded in '+str(round(time.time()-start,2))+' seconds.\n')

#train['MachineIdentifier'] = train.index.astype('uint32')

#gc.collect()

print(list(train_df.columns.values))

X = train_df.drop('HasDetections',axis=1)
#X = X.iloc[:].values
Y = train_df['HasDetections']#.values

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.25, random_state = 0)

from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier, plot_importance
from sklearn.svm import SVC
from sklearn.feature_selection import SelectFromModel

def plotRoc(testLabels, y_pred_probabilities, label):
    oneHotEncoding = np.zeros([testLabels.shape[0], 2])
    y_pred_probabilities = y_pred_probabilities.T
    for i in range(testLabels.shape[0]):
        oneHotEncoding[i][testLabels[i]] = 1
    plt.figure()
    oneHotEncoding = oneHotEncoding.T
    for i in range (2):
        auc_score = roc_auc_score(oneHotEncoding[i], y_pred_probabilities[i])
        fpr, tpr, thresholds = roc_curve(oneHotEncoding[i], y_pred_probabilities[i])
        plt.plot(fpr, tpr, label="Class: " + str(i) + ": " + str(auc_score)[0:5], linewidth=0.5)
    
    plt.legend(loc="lower right")
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.plot([0, 1], [0, 1], 'k--', linewidth=2)
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC Curves for ' + label)
    plt.savefig(label+'.png')

#m = RandomForestClassifier(n_estimators=60, min_samples_leaf=100000, max_features='sqrt', n_jobs=-1)
#m = XGBClassifier(learning_rate = 0.1, n_estimators = 100, gamma = 0.4, subsample = 0.55, colsample_bytree = 0.85, max_depth = 3, min_child_weight = 5, objective= 'binary:logistic', nthread=4, seed=27, n_jobs=-1)
#m = SVC(kernel = 'rbf', C = 1)
m = XGBClassifier(learning_rate = 0.1, n_estimators = 100, gamma = 0.4, subsample = 0.55, colsample_bytree = 0.85, max_depth = 3, min_child_weight = 5, objective= 'binary:logistic', nthread=-1, seed=27)

m.fit(X_train, Y_train)

predictions = m.predict(X_test)

accuracy = accuracy_score(Y_test, predictions)
print("Accuracy: %.2f%%" % (accuracy * 100.0))

print(confusion_matrix(Y_test, predictions))

thresholds = np.sort(m.feature_importances_)
thresholds = thresholds[::-1]
I=20
for thresh in thresholds:
    I-=1
    if (I<0):
        break
    elif (I==0):
        selection = SelectFromModel(m, threshold=thresh, prefit=True)
        select_X_train = selection.transform(X_train)
        selection_model = XGBClassifier(learning_rate = 0.1, n_estimators = 100, gamma = 0.4, subsample = 0.55, colsample_bytree = 0.85, max_depth = 3, min_child_weight = 5, objective= 'binary:logistic', nthread=-1, seed=27)
        selection_model.fit(select_X_train, Y_train)
        select_X_test = selection.transform(X_test)
        y_pred = selection_model.predict(select_X_test)
        predictions = [round(value) for value in y_pred]
        accuracy = accuracy_score(Y_test, predictions)
        print(confusion_matrix(Y_test, predictions))
        print("Thresh=%.3f, n=%d, Accuracy: %.2f%%" % (thresh, select_X_train.shape[1], accuracy*100.0))

plot_importance(m,max_num_features=20)
plt.show()

# m.fit(X_train, Y_train)
# YProb = m.predict_proba(X_test)
# YPred = m.predict(X_test)
# print(confusion_matrix(Y_test, YPred))
# print(np.mean(Y_test==YPred)*100)
# plotRoc(Y_test,YProb,'SVM')

#==================

# X_submission_test = pd.read_csv('test.csv', dtype=dtypes)
# # Next 12 lines are the same process that we did on train set.
# test_df_copy = X_submission_test.copy()
# test_df_Noncategory = test_df_copy.select_dtypes(exclude='category')
# test_df_category = test_df_copy.select_dtypes(include='category')
# fill_with_max(test_df_Noncategory['AVProductStatesIdentifier'])
# fill_with_max(test_df_Noncategory['AVProductsInstalled'])
# fill_with_max(test_df_Noncategory['AVProductsEnabled'])
# fill_with_max(test_df_Noncategory['Firewall'])
# fill_with_max(test_df_Noncategory['Census_FirmwareManufacturerIdentifier'])
# fill_with_max(test_df_Noncategory['Census_FirmwareVersionIdentifier'])

# test_df_Noncategory_NoNull = test_df_Noncategory[['IsBeta','IsSxsPassiveMode','AVProductStatesIdentifier','AVProductsInstalled','AVProductsEnabled','HasTpm','CountryIdentifier','LocaleEnglishNameIdentifier','OsBuild','OsSuite','AutoSampleOptIn','Firewall','Census_HasOpticalDiskDrive','Census_OSBuildNumber','Census_OSBuildRevision','Census_OSUILocaleIdentifier','Census_IsPortableOperatingSystem','Census_FirmwareManufacturerIdentifier','Census_FirmwareVersionIdentifier','Census_IsSecureBootEnabled','Census_IsTouchEnabled','Census_IsPenCapable']].copy()
# fill_with_max(test_df_category['Census_PrimaryDiskTypeName'])
# fill_with_max(test_df_category['Census_PowerPlatformRoleName'])
# test_df_category.drop(['MachineIdentifier','EngineVersion','AppVersion','OsVer','OsBuildLab','AvSigVersion','PuaMode','SmartScreen','Census_ProcessorClass','Census_ChassisTypeName','Census_InternalBatteryType','Census_OSVersion', 'Census_OSBranch','Census_OSEdition','Census_OSSkuName'],axis =1,inplace=True)
# # Test data have some differencies with Train data
# # 2 categorical columns have 1 more category than train data. As we have more categories when we apply 
# # OneHotEncoding we have more columns than train data. This situation causes an eror 
# # when we try to apply our model to test data.
# # So we need to decrease category number. I look the data as extra actegory has small number of repetation, 
# # I change the category to the nearst value in the column. 
# test_df_category['Census_MDC2FormFactor'].replace('Other','IoTOther',inplace=True)
# test_df_category['Census_FlightRing'].replace('CBCanary','Canary',inplace=True)
# test_df_category['Census_GenuineStateName'].fillna('IS_GENUINE',inplace=True)
# # Same as train set create the final dataframe.
# test_df_final = test_df_category.join(test_df_Noncategory_NoNull)

# start = time.time()
# for i in range(0,len(test_df_category.columns)):
#     labelencoder_X_test_i = LabelEncoder()
#     test_df_final[test_df_category.columns[i]] = labelencoder_X_test_i.fit_transform(test_df_final[test_df_category.columns[i]])
#     print(test_df_category.columns[i])
# end = time.time()
# print(end-start)

# for i in range(0,len(test_df_category.columns)):
#     test_df_final[test_df_category.columns[i]] = test_df_final[test_df_category.columns[i]].astype(np.int8)

# start = time.time()
# onehotencoder = OneHotEncoder(categorical_features=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],sparse = False)
# test_df_final_coded = onehotencoder.fit_transform(test_df_final)
# end = time.time()
# print(end-start)
# # Do a prediction with our model.
# # We use predict_proba because competition asks for the probablity of the machine infected by Malware.
# pred_prob = m.predict_proba(test_df_final_coded)
# # Second column of the pred_prob is the probablity of the infection (probablity of 1)
# # We assign this column to HasDetections as having the submission format.
# test_df_copy['HasDetections'] = pred_prob[:,-1]
# # Also get only MachineIdentifier and HasDetections column to have the format.  
# test_df_copy = test_df_copy.loc[:,['MachineIdentifier','HasDetections']]
# # Create submission File and submit it to competition.
# test_df_copy.to_csv('submission.csv',index=False)

#================================================================

# start = time.time()
# test  = pd.read_csv('test.csv')
# print('Test data loaded in '+str(round(time.time()-start,2))+' seconds.\n')

# column_names=list(test.columns.values)
# print('Columns originally in Test: '+str(len(column_names))+'\n')

# RemoveColumns = ['PuaMode', 'Census_ProcessorClass', 'DefaultBrowsersIdentifier', 'Census_IsFlightingInternal', 'Census_InternalBatteryType', 'SMode', 'Census_ThresholdOptIn', 'Census_IsWIMBootEnabled', 'SmartScreen', 'OrganizationIdentifier']

# test.drop(RemoveColumns, axis=1, inplace=True)

# test_column_names=list(test.columns.values)
# print('Columns after removing in Test: '+str(len(test_column_names))+'\n')
# print('Columns removed: ',RemoveColumns,'\n')

#start = time.time()
#KeepColumns = list(set(train_column_names) & set(test_column_names))
#print('Intersection found in  '+str(round(time.time()-start,2))+' seconds.\n')
#print('Columns to keep: '+str(len(KeepColumns))+'\n')
#print('Columns Kept: ', KeepColumns, '\n')
#print('Columns Removed: ', list(set(train_column_names) - set(KeepColumns)), '\n')
