#generic
import math
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import eli5
import pickle
from sklearn.externals import joblib
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
#===========================================
#boost
import catboost as catb
import lightgbm as lgb
import xgboost
#===========================================
#validation
from sklearn.model_selection import train_test_split,cross_val_score
from sklearn.metrics import accuracy_score,confusion_matrix,roc_auc_score,mean_squared_error
from sklearn.model_selection import KFold
#===========================================
#model
from sklearn.linear_model import LinearRegression,LogisticRegression,Ridge,Lasso
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor,ExtraTreesRegressor,ExtraTreesClassifier,AdaBoostRegressor,AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
# from sklearn.tree import DecisionTreeClassifer,DecisionTreeRegressor
# from neighbors import KNeighborsClassifier,KNeighborsRegressor
#===========================================
#clustering
from sklearn.mixture import GaussianMixture
from sklearn.cluster import KMeans,AgglomerativeClustering
from nltk.cluster.kmeans import KMeansClusterer
import scipy.cluster.hierarchy as sc
#===========================================
#deeplearning
import tensorflow as tf 
import keras

train=pd.read_csv('train.csv')
# =========================
# EDA
column_names=list(train.columns.values)

# for i in range(len(column_names)):
# 	print(train[column_names[i]].value_counts())


# ===========================================================

# Distribution of Categories
# true_numerical_columns = ['Census_ProcessorCoreCount','Census_PrimaryDiskTotalCapacity','Census_SystemVolumeTotalCapacity','Census_TotalPhysicalRAM','Census_InternalPrimaryDiagonalDisplaySizeInInches','Census_InternalPrimaryDisplayResolutionHorizontal','Census_InternalPrimaryDisplayResolutionVertical','Census_InternalBatteryNumberOfCharges']
# binary_variables = [c for c in train.columns if train[c].nunique() == 2]
# categorical_columns = [c for c in train.columns if (c not in true_numerical_columns) & (c not in binary_variables)]
# raw_data = {'Type': ['Numerical', 'Binary', 'Categorical'],'Number': [len(true_numerical_columns), len(binary_variables),len(categorical_columns)]}
# df = pd.DataFrame(raw_data, columns = ['Type', 'Number'])
# colors = ["#E13F29", "#D69A80", "#D63B59", "#AE5552", "#CB5C3B", "#EB8076", "#96624E"]
# plt.pie(df['Number'],labels=df['Type'],shadow=False,colors=colors,startangle=90,autopct='%1.1f%%')
# plt.axis('equal')
# plt.tight_layout()
# plt.show()
# print(true_numerical_columns)
# print(binary_variables)
# print(categorical_columns)

# ===========================================================

# Distribution of Missing Values in Train
nans = []
pcts = []
for cols in train:
	nans.append(train[cols].isnull().sum())
	pcts.append(train[cols].isnull().sum()/train.shape[0]*100)

missing_data = pd.DataFrame({"Col":train.columns,"Missing values": nans, "Pct missing [%]": pcts}).sort_values("Missing values", ascending=False)
missing_data[missing_data["Missing values"]!=0].reset_index(drop=True)
missing_data.to_csv('missing_datatrain.csv',index=False)
print(missing_data.head(50))

# complete = (missing_data["Pct missing [%]"]<=10).sum()
# a = ((missing_data["Pct missing [%]"]!=0) & (missing_data["Pct missing [%]"]<=10)).sum()
# b = ((missing_data["Pct missing [%]"]>10) & (missing_data["Pct missing [%]"]<=50)).sum()
# c = (missing_data["Pct missing [%]"]>50).sum()
# print("There are:\n{} columns without missing values\n{} columns with less than 10% of missing values\n {} columns withmissing values between 10% and 50%\n {} columns with more than 50% of missing values".format(complete,a,b,c))

# labels =["No missing data", "Missing 0-10%", "Missing 10-50%", "Missing over 50% of data"]
# fig1, ax1 = plt.subplots(figsize=(8,8))
# ax1.pie([complete,a,b,c],autopct='%1.1f%%',labels=labels, textprops={'fontsize': 15})
# ax1.axis('equal')
# plt.show()
# ===========================================================

# Distribution of Malware
# detect_no = (train["HasDetections"]==0).sum()
# detect_yes = (train["HasDetections"]==1).sum()
# labels = 'No detection', 'Detection',
# sizes = [detect_no, detect_yes]
# fig1, ax1 = plt.subplots(figsize=(8,8))
# ax1.pie(sizes, labels=labels, autopct='%1.1f%%',shadow=True, startangle=90, textprops={'fontsize': 15})
# ax1.axis('equal')
# plt.show()

# ===========================================================

# Bar Graph for Percentage Missing Across Features
na_train = train.isna().sum()/train.shape[0]
plt.figure(figsize=(20,50))
na_train[na_train!=0].sort_values().plot(kind='barh')
plt.show()

# ===========================================================
na_some = na_train[na_train!=0]
print('Less than 30per= ',na_some[na_some<0.3].shape)
print('More than 30per= ',na_some[na_some>0.3].shape)
print(na_some[na_some>0.3].shape)

# ===========================================================
test  = pd.read_csv('test.csv')
total = test.isnull().sum().sort_values(ascending = False)
percent = (test.isnull().sum()/test.isnull().count()*100).sort_values(ascending = False)
missing_test_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_test_data.to_csv('missing_test_data.csv',index=False)

print(missing_test_data.head(50))