
# coding: utf-8

# In[1]:


import pandas as pd
from IPython.display import Markdown, display
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
import numpy as np
from sklearn import metrics



# def printmd(string):
#     display(Markdown(string))
    
    
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.model_selection import cross_val_score

from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_curve, auc
from sklearn.preprocessing import label_binarize


# get_ipython().run_line_magic('matplotlib', 'inline')


# In[2]:


#adult = pd.read_csv('adult.csv')

column_names = ['age', 'workclass', 'fnlwgt', 'education', 'educational-num','marital-status', 'occupation', 'relationship', 'race', 'gender','capital-gain', 'capital-loss', 'hours-per-week', 'native-country','income']

train = pd.read_csv('adult.data', sep=",\s", header=None, names = column_names, engine = 'python')
test = pd.read_csv('adult.test', sep=",\s", header=None, names = column_names, engine = 'python')
test['income'].replace(regex=True,inplace=True,to_replace=r'\.',value=r'')


adult = pd.concat([test,train])
adult.reset_index(inplace = True, drop = True)


# ---
# # 1. Preliminary Data Analysis

# In[7]:


# Setting all the categorical columns to type category
for col in set(adult.columns) - set(adult.describe().columns):
    adult[col] = adult[col].astype('category')
    
# printmd('## 1.1. Columns and their types')
print(adult.info())


# In[8]:


# Top 5 records
# printmd('## 1.2. Data')
adult.head()


# In[9]:


# printmd('## 1.3. Summary Statistics')

adult.describe()


# In[10]:


# printmd('## 1.4. Missing values')
# for i,j in zip(adult.columns,(adult.values.astype(str) == '?').sum(axis = 0)):
#     if j > 0:
#         printmd(str(i) + ': ' + str(j) + ' records')



# ### Treating Missing Values by predicting them
# 
# I fill the missing values in each of the three columns by predicting their values. For each of the three columns, I use all the attributes (including 'income') as independent variables and treat that column as the dependent variable, making it a multi-class classification task. I use three classification algorithms, namely, logistic regression, decision trees and random forest to predict the class when the value is missing (in this case a '?'). I then take a majority vote amongst the three classifiers to be the class of the missing value. In case of a tie, I pick the majority class of that column using the entire dataset.

# In[11]:


# Create one hot encoding of the categorical columns in the data frame.
def oneHotCatVars(df, df_cols):
    
    df_1 = adult_data = df.drop(columns = df_cols, axis = 1)
    df_2 = pd.get_dummies(df[df_cols])
    
    return (pd.concat([df_1, df_2], axis=1, join='inner'))


# In[12]:


# printmd('### 1.4.1. Filling in missing values for Attribute workclass')

test_data = adult[(adult.workclass.values == '?')].copy()
test_label = test_data.workclass

train_data = adult[(adult.workclass.values != '?')].copy()
train_label = train_data.workclass

test_data.drop(columns = ['workclass'], inplace = True)
train_data.drop(columns = ['workclass'], inplace = True)

train_data = oneHotCatVars(train_data, train_data.select_dtypes('category').columns)
test_data = oneHotCatVars(test_data, test_data.select_dtypes('category').columns)

log_reg = LogisticRegression()
log_reg.fit(train_data, train_label)
log_reg_pred = log_reg.predict(test_data)


clf = tree.DecisionTreeClassifier()
clf = clf.fit(train_data, train_label)
clf_pred = clf.predict(test_data)

r_forest = RandomForestClassifier(n_estimators=10)
r_forest.fit(train_data, train_label)
r_forest_pred = r_forest.predict(test_data)

majority_class = adult.workclass.value_counts().index[0]

pred_df =  pd.DataFrame({'RFor': r_forest_pred, 'DTree' : clf_pred, 'LogReg' : log_reg_pred})
overall_pred = pred_df.apply(lambda x: x.value_counts().index[0] if x.value_counts()[0] > 1 else majority_class, axis = 1)

adult.loc[(adult.workclass.values == '?'),'workclass'] = overall_pred.values
print(adult.workclass.value_counts())
print(adult.workclass.unique())


# In[13]:


# printmd('### 1.4.2. Filling in missing values for Occupation occupation')

test_data = adult[(adult.occupation.values == '?')].copy()
test_label = test_data.occupation

train_data = adult[(adult.occupation.values != '?')].copy()
train_label = train_data.occupation

test_data.drop(columns = ['occupation'], inplace = True)
train_data.drop(columns = ['occupation'], inplace = True)

train_data = oneHotCatVars(train_data, train_data.select_dtypes('category').columns)
test_data = oneHotCatVars(test_data, test_data.select_dtypes('category').columns)

log_reg = LogisticRegression()
log_reg.fit(train_data, train_label)
log_reg_pred = log_reg.predict(test_data)


clf = tree.DecisionTreeClassifier()
clf = clf.fit(train_data, train_label)
clf_pred = clf.predict(test_data)

r_forest = RandomForestClassifier(n_estimators=10)
r_forest.fit(train_data, train_label)
r_forest_pred = r_forest.predict(test_data)


majority_class = adult.occupation.value_counts().index[0]

pred_df =  pd.DataFrame({'RFor': r_forest_pred, 'DTree' : clf_pred, 'LogReg' : log_reg_pred})
overall_pred = pred_df.apply(lambda x: x.value_counts().index[0] if x.value_counts()[0] > 1 else majority_class, axis = 1)

adult.loc[(adult.occupation.values == '?'),'occupation'] = overall_pred.values
print(adult.occupation.value_counts())
print(adult.occupation.unique())


# In[14]:


# printmd('### 1.4.3. Filling in missing values for Native Country')

test_data = adult[(adult['native-country'].values == '?')].copy()
test_label = test_data['native-country']

train_data = adult[(adult['native-country'].values != '?')].copy()
train_label = train_data['native-country']

test_data.drop(columns = ['native-country'], inplace = True)
train_data.drop(columns = ['native-country'], inplace = True)

train_data = oneHotCatVars(train_data, train_data.select_dtypes('category').columns)
test_data = oneHotCatVars(test_data, test_data.select_dtypes('category').columns)

log_reg = LogisticRegression()
log_reg.fit(train_data, train_label)
log_reg_pred = log_reg.predict(test_data)


clf = tree.DecisionTreeClassifier()
clf = clf.fit(train_data, train_label)
clf_pred = clf.predict(test_data)

r_forest = RandomForestClassifier(n_estimators=10)
r_forest.fit(train_data, train_label)
r_forest_pred = r_forest.predict(test_data)


majority_class = adult['native-country'].value_counts().index[0]

pred_df =  pd.DataFrame({'RFor': r_forest_pred, 'DTree' : clf_pred, 'LogReg' : log_reg_pred})
overall_pred = pred_df.apply(lambda x: x.value_counts().index[0] if x.value_counts()[0] > 1 else majority_class, axis = 1)

adult.loc[(adult['native-country'].values == '?'),'native-country'] = overall_pred.values
print(adult['native-country'].value_counts())
print(adult['native-country'].unique())


# In[15]:


# Resetting the categories

adult['workclass'] = adult['workclass'].cat.remove_categories('?')
adult['occupation'] = adult['occupation'].cat.remove_categories('?')
adult['native-country'] = adult['native-country'].cat.remove_categories('?')



# # In[22]:


# # printmd('## 1.5. Correlation Matrix')

# display(adult.corr())

# # printmd('We see that none of the columns are highly correlated.')


# # ---
# # # 2. Exploratory Analysis

# # In[23]:


# # Creating a dictionary that contain the education and it's corresponding education level
# edu_level = {}
# for x,y in adult[['educational-num','education']].drop_duplicates().itertuples(index=False):
#     edu_level[y] = x


# # In[46]:


# # printmd('## 2.1. Education vs Income')

# education = round(pd.crosstab(adult.education, adult.income).div(pd.crosstab(adult.education, adult.income).apply(sum,1),0),2)
# education = education.reindex(sorted(edu_level, key=edu_level.get, reverse=False))

# ax = education.plot(kind ='bar', title = 'Proportion distribution across education levels', figsize = (10,8))
# ax.set_xlabel('Education level')
# ax.set_ylabel('Proportion of population')

# # printmd('I plot a bar graph showing the proportion of income classes across education levels         in the figure below. As one would expect, we see from the bar graph below that as the         education level increase, the proportion of people who earn more than 50k a year also         increase. It is interesting to note that only after a master\'s degree, the proportion of         people earning more than 50k a year, is a majority.')
        


# # In[37]:


# # printmd('## 2.2 Gender vs Income')

# gender = round(pd.crosstab(adult.gender, adult.income).div(pd.crosstab(adult.gender, adult.income).apply(sum,1),0),2)
# gender.sort_values(by = '>50K', inplace = True)
# ax = gender.plot(kind ='bar', title = 'Proportion distribution across gender levels')
# ax.set_xlabel('Gender level')
# ax.set_ylabel('Proportion of population')

# # printmd('We plot a bar graph showing the proportion of income classes across the genders in figure         below. From the graph, at an overall view, there exists a wage gap between females and males.         Since we do not have the exactly value of the income, we are limited to only observing that the         proportion of males earning more than 50k a year is more than double of their female counterparts.         ')


# # In[33]:


# gender_workclass = round(pd.crosstab(adult.workclass, [adult.income, adult.gender]).div(pd.crosstab(adult.workclass, [adult.income, adult.gender]).apply(sum,1),0),2)
# gender_workclass[[('>50K','Male'), ('>50K','Female')]].plot(kind = 'bar', title = 'Proportion distribution across gender for each workclass', figsize = (10,8), rot = 30)
# ax.set_xlabel('Gender level')
# ax.set_ylabel('Proportion of population')

# # printmd('Taking a closer look at the disparity in income between men and women, plot the         proportion of men and women who earn more than 50k a year, across all the working         classes as seen in Fig. 3. We see that men always have a higher proportion earning         more than 50k a year than women, except for the \'without.pay\' working class, where         they have the same proportion.')


# # In[35]:


# # printmd('### Gender across working classes')

# # for i in adult.workclass.unique():
# #     df = adult[adult.workclass == i]

# #     hours_per_week = round(pd.crosstab(df.gender, df.income).div(pd.crosstab(df.gender, df.income).apply(sum,1),0),2)
# #     # hours_per_week.sort_values(by = '>50K', inplace = True)
# #     ax = hours_per_week.plot(kind ='bar', title = 'Proportion distribution across Gender for '+ i)
# #     ax.set_xlabel('Gender')
# #     ax.set_ylabel('Proportion of population')

# #     print()


# # In[38]:


# printmd('## 2.3. Occupation vs Income')

# occupation = round(pd.crosstab(adult.occupation, adult.income).div(pd.crosstab(adult.occupation, adult.income).apply(sum,1),0),2)
# occupation.sort_values(by = '>50K', inplace = True)
# ax = occupation.plot(kind ='bar', title = 'Proportion distribution across Occupation levels', figsize = (10,8))
# ax.set_xlabel('Occupation level')
# ax.set_ylabel('Proportion of population')

# print()


# # In[39]:


# # printmd('## 2.4. Workclass vs Income')

# workclass = round(pd.crosstab(adult.workclass, adult.income).div(pd.crosstab(adult.workclass, adult.income).apply(sum,1),0),2)
# workclass.sort_values(by = '>50K', inplace = True)
# ax = workclass.plot(kind ='bar', title = 'Proportion distribution across workclass levels', figsize = (10,8))
# ax.set_xlabel('Workclass level')
# ax.set_ylabel('Proportion of population')

# print()


# # In[40]:


# printmd('## 2.5. Race vs Income')

# race = round(pd.crosstab(adult.race, adult.income).div(pd.crosstab(adult.race, adult.income).apply(sum,1),0),2)
# race.sort_values(by = '>50K', inplace = True)
# ax = race.plot(kind ='bar', title = 'Proportion distribution across race levels', figsize = (10,8))
# ax.set_xlabel('Race level')
# ax.set_ylabel('Proportion of population')

# print()


# # In[47]:


# # printmd('## 2.6. Native Country')

# native_country = round(pd.crosstab(adult['native-country'], adult.income).div(pd.crosstab(adult['native-country'], adult.income).apply(sum,1),0),2)
# native_country.sort_values(by = '>50K', inplace = True)
# ax = native_country.plot(kind ='bar', title = 'Proportion distribution across Native Country levels', figsize = (20,12))
# ax.set_xlabel('Native country')
# ax.set_ylabel('Proportion of population')

# # printmd('I plot a bar graph showing the proportion of income classes across the native country in figure         below. From the graph, we notice a trend in positioning of the country. South American country are         at the left end of the plot, with low proportion of population that make more than 50k a year. The         United States is located somewhat centrally, and at the right are countries from Europe and Asia,         with higher proportion of population that make more than 50k a year.')


# # In[48]:


# # printmd('## 2.7. Hours per week vs Income')

# hours_per_week = round(pd.crosstab(adult['hours-per-week'], adult.income).div(pd.crosstab(adult['hours-per-week'], adult.income).apply(sum,1),0),2)
# # hours_per_week.sort_values(by = '>50K', inplace = True)
# ax = hours_per_week.plot(kind ='bar', title = 'Proportion distribution across Hours per week', figsize = (20,12))
# ax.set_xlabel('Hours per week')
# ax.set_ylabel('Proportion of population')

# # printmd('I plot a bar graph showing the proportion of income classes across the hours worked.         We would expected to notice a trend that higher the hours worked per week, the higher the proportion         of population making more than 50k a year. However, this was not necessarily true from the graph.         For several hours instance (for example, where hours worked was 77, 79, 81, 82, 87, 88 and so on)         no one earned more than 50k a year. ')


# # In[51]:


# # printmd('### 2.7.1 Hours per week with categories')

# adult['hour_worked_bins'] = ['<40' if i < 40 else '40-60' if i <= 60 else '>60'  for i in adult['hours-per-week']]
# adult['hour_worked_bins'] = adult['hour_worked_bins'].astype('category')
# hours_per_week = round(pd.crosstab(adult.hour_worked_bins, adult.income).div(pd.crosstab(adult.hour_worked_bins, adult.income).apply(sum,1),0),2)

# hours_per_week.sort_values(by = '>50K', inplace = True)
# ax = hours_per_week.plot(kind ='bar', title = 'Proportion distribution across Hours per week', figsize = (10,6))
# ax.set_xlabel('Hours per week')
# ax.set_ylabel('Proportion of population')

# # printmd('Therefore, I decided to transform this column into 3 categories, less than 40 hours,         40 to 60 hours and greater than 60 hours. Plotting a bar graph with these 3 categories,         we can see from the figure below that there is an increasing trend in the proportion of         population making more than 50k a year.')


# # In[52]:


# # printmd('### 2.7.2 Hours worked across working classes')

# for i in adult.workclass.unique():
#     df = adult[adult.workclass == i]

#     hours_per_week = round(pd.crosstab(df['hours-per-week'], df.income).div(pd.crosstab(df['hours-per-week'], df.income).apply(sum,1),0),2)
#     # hours_per_week.sort_values(by = '>50K', inplace = True)
#     ax = hours_per_week.plot(kind ='bar', title = 'Proportion distribution across Hours per week for '+ i, figsize = (20,12))
#     ax.set_xlabel('Hours per week')
#     ax.set_ylabel('Proportion of population')

#     print()


# # In[ ]:


# # plt.rcParams['figure.figsize'] = (5, 4)
# # #plt.xlim([20,110])
# # #plt.ylim([20,110])
            
# # pos = adult[adult.income == '>50K']
# # neg = adult[adult.income != '>50K']


# # plt.scatter(pos['educational-num'], pos['hours-per-week'], marker='o', c='b')
# # plt.scatter(neg['educational-num'], neg['hours-per-week'], marker='x', c='r')
# # plt.xlabel('Exam 1 score')
# # plt.ylabel('Exam 2 score')
# # plt.legend(['Not Admitted', 'Admitted'])

# # plt.show()


# # ---
# # # 3. Data Transformations

# # ## 3.1. Feature Selection

# # In[59]:


# # Remove education and fnlwgt
# #adult.drop(columns = ['education','fnlwgt','hours-per-week'], inplace = True)

# # printmd('* For education level, we have 2 features that convey the same meaning, \'education\'         and \'educational-num\'. To avoid the effect of this attribute on the models to be         overstated, I am not going to use the categorical education attribute.')
# # printmd('* I use the categorical Hours work column and drop the \'hour-per-week\' column')
# # printmd('* Also, I chose not to use the \'Fnlwgt\' attribute that is used by the census,         as the inverse of sampling fraction adjusted for non-response and over or under sampling         of particular groups. This attribute does not convey individual related meaning.')


# # ## 3.2 Normalization

# # In[60]:


# # printmd('## Box plot')
# adult.select_dtypes(exclude = 'category').plot(kind = 'box', figsize = (10,8))


# # In[61]:


# # printmd ('Normalization happens on the training dataset, by removing the mean and         scaling to unit variance. These values are stored and then later applied          to the test data before the test data is passed to the model for prediction. ')


# # ---
# # # 4. Model Development & Classification 

# # ## 4.1. Data Preparation'
# # 
# # One-hot encoding is the process of representing multi-class categorical features as binary features, one for each class. Although this process increases the dimensionality of the dataset, classification algorithms tend to work better on this format of data.
# # 
# # I use one-hot encoding to represent all the categorical features in the dataset. 
# # 

# In[64]:


# Data Prep
adult_data = adult.drop(columns = ['income'])
adult_label = adult.income


adult_cat_1hot = pd.get_dummies(adult_data.select_dtypes('category'))
adult_non_cat = adult_data.select_dtypes(exclude = 'category')

adult_data_1hot = pd.concat([adult_non_cat, adult_cat_1hot], axis=1, join='inner')


# In[65]:


# Train - Test split
train_data, test_data, train_label, test_label = train_test_split(adult_data_1hot, adult_label, test_size  = 0.25)




# # In[66]:


# # Normalization
# from sklearn.preprocessing import StandardScaler  
# scaler = StandardScaler()  

# # Fitting only on training data
# scaler.fit(train_data)  
# train_data = scaler.transform(train_data)  

# # Applying same transformation to test data
# test_data = scaler.transform(test_data) 


# # In[67]:


# def model_eval(actual, pred):
    
#     confusion = pd.crosstab(actual, pred, rownames=['Actual'], colnames=['Predicted'])
#     TP = confusion.loc['>50K','>50K']
#     TN = confusion.loc['<=50K','<=50K']
#     FP = confusion.loc['<=50K','>50K']
#     FN = confusion.loc['>50K','<=50K']

#     accuracy = ((TP+TN))/(TP+FN+FP+TN)
#     precision = (TP)/(TP+FP)
#     recall = (TP)/(TP+FN)
#     f_measure = (2*recall*precision)/(recall+precision)
#     sensitivity = TP / (TP + FN)
#     specificity = TN / (TN + FP)
#     error_rate = 1 - accuracy
    
#     out = {}
#     out['accuracy'] =  accuracy
#     out['precision'] = precision
#     out['recall'] = recall
#     out['f_measure'] = f_measure
#     out['sensitivity'] = sensitivity
#     out['specificity'] = specificity
#     out['error_rate'] = error_rate
    
#     return out


# # ## 4.2. Model Development
# # ### 4.2.1. Decision Tree
# # 
# # For the decision tree classifier, I experimented with the splitting criteria, minimum samples required to split, max depth of the tree, minimum samples required at the leaf level and the maximum features to consider when looking for the best split. The following values of the parameters attained the best accuracy during classification. Results in the table below.
# # 
# # *	**Splitting criteria:** Gini Index (Using Gini Index marginally outperformed Entropy with a higher accuracy.)
# # *	**Min samples required to split:** 5% (Best amongst 1%, 10% and 5%.)
# # *	**Max Depth:** None
# # *	**Min samples required at leaf:**  0.1 % (Best amongst 1%, 5% and 0.1%.)
# # *	**Max features:** number of features (Performs better than 'auto', 'log2' and 'sqrt'.)
# # 

# # In[74]:


# #printmd('### 3.1.1. Model Development ')

# # Gini 
# clf_gini = tree.DecisionTreeClassifier(criterion = 'gini', min_samples_split = 0.05, min_samples_leaf = 0.001, max_features = None)
# clf_gini = clf_gini.fit(train_data, train_label)
# clf_gini_pred = clf_gini.predict(test_data)
# DTree_Gini = model_eval(test_label, clf_gini_pred)
# print('Desicion Tree using Gini Index : %.2f percent.' % (round(DTree_Gini['accuracy']*100,2)))


# # Entropy
# clf_entropy = tree.DecisionTreeClassifier(criterion = 'entropy', min_samples_split = 0.05, min_samples_leaf = 0.001)
# clf_entropy = clf_entropy.fit(train_data, train_label)
# clf_entropy_pred = clf_entropy.predict(test_data)
# DTree_Entropy = model_eval(test_label, clf_entropy_pred)
# print('Desicion Tree using Entropy : %.2f percent.' % (round(DTree_Entropy['accuracy']*100,2)))


# #printmd('### 3.1.2. Model Evaulation ')
# ovl_dtree = round(pd.DataFrame([DTree_Entropy, DTree_Gini], index = ['DTree_Entropy','DTree_Gini']),4)
# display(ovl_dtree)


# # In[ ]:


# # Plot tree

# # from sklearn.tree import export_graphviz
# # from IPython.display import SVG
# # from graphviz import Source



# # graph = Source( tree.export_graphviz(clf, out_file=None, feature_names=train_data.columns))

# # # Display Tree
# # SVG(graph.pipe(format='svg'))

# # # Save Tree as PNG
# # png_bytes = graph.pipe(format='png')
# # with open('dtree_pipe.png','wb') as f:
# #     f.write(png_bytes)


# # ### 4.2.2. Artificial Neural Network
# # For the ANN classifier, I experimented with the activation function, the solver for weight optimization, regularization term and learning schedule for weight updates. The following values of the parameters attained the best accuracy during classification. Other parameters were neither applicable to the 'adam' solver nor did it improve the performance of the model. Results in the table below.
# # 
# # *	**Activation:** Logistic (Marginally outperformed 'relu', 'tanh' and 'identity' functions.)
# # *   **Solver:** Adam (Works well on relatively large datasets with thousands of training samples or more)
# # *   **Alpha:** 1e-4 (Best amongst 1, 1e-1, 1e-2, 1e-3, 1e-4 and 1e-5)
# # *   **Learning Rate:**  'invscaling' (Gradually decreases the learning rate at each time step 't' using an inverse scaling exponent of 'power_t'.)
# # 

# # In[75]:


# # Tan H
# ann_tanh = MLPClassifier(activation = 'tanh', solver='lbfgs', alpha=1e-1, hidden_layer_sizes=(10, 2), random_state=1, warm_start=True)
# ann_tanh.fit(train_data, train_label)                         
# ann_tanh_pred = ann_tanh.predict(test_data)
# ANN_TanH = model_eval(test_label, ann_tanh_pred)
# print('ANN using TanH and lbfgs solver : %.2f percent.' % (round(ANN_TanH['accuracy']*100,2)))


# # Relu
# ann_relu = MLPClassifier(activation = 'relu', solver='adam', alpha=1e-1, 
#                     hidden_layer_sizes=(5, 2), random_state=1,
#                     learning_rate  = 'invscaling',
#                     warm_start = True)
# ann_relu.fit(train_data, train_label)                         
# ann_relu_pred = ann_relu.predict(test_data)
# ANN_relu = model_eval(test_label, ann_relu_pred)
# print('ANN using relu and adam solver : %.2f percent.' % (round(ANN_relu['accuracy']*100,2)))

# # Log
# ann_log = MLPClassifier(activation = 'logistic', solver='adam', 
#                     alpha=1e-4, hidden_layer_sizes=(5, 2),
#                     learning_rate  = 'invscaling', 
#                     random_state=1, warm_start = True)
# ann_log.fit(train_data, train_label)                         
# ann_log_pred = ann_log.predict(test_data)
# ANN_log = model_eval(test_label, ann_log_pred)
# print('ANN using logistic and adam solver : %.2f percent.' % (round(ANN_log['accuracy']*100,2)))

# # Identity
# ann_identity = MLPClassifier(activation = 'identity', solver='adam', alpha=1e-1, hidden_layer_sizes=(5, 2), random_state=1, warm_start = True)
# ann_identity.fit(train_data, train_label)                         
# ann_identity_pred = ann_identity.predict(test_data)
# ANN_identity = model_eval(test_label, ann_identity_pred)
# print('ANN using identity and adam solver : %.2f percent.' % (round(ANN_identity['accuracy']*100,2)))

# #printmd('### 3.2.2. Model Evaulation ')
# ovl_ann = round(pd.DataFrame([ANN_TanH, ANN_relu, ANN_log, ANN_identity], index = ['ANN_TanH','ANN_relu', 'ANN_log', 'ANN_identity']),4)
# display(ovl_ann)


# # ### 4.2.3. Support Vector Machine
# # For the SVM classifier, I experimented with the various available kernels, the penalty of the error term and the tolerance for stopping criteria. The following values of the parameters attained the best accuracy during classification. Results in the table below.
# # 
# # *	**Kernel:** rbf (Marginally outperformed 'linear, 'poly' and 'sigmoid' kernels.)
# # *	**C, penalty of the error term:** 1 (Best amongst 0.1, 0.5, 1 and 10)
# # *	**Tolerance for stopping criteria:** 1e-3 (Best amongst 1e-1, 1e-2, 1e-3, 1e-4 and 1e-5)
# # 

# # In[77]:


# # rbf kernal
# svm_clf_rbf = svm.SVC(kernel = 'rbf', C = 1, tol = 1e-3)
# svm_clf_rbf.fit(train_data, train_label)
# svm_clf_rbf_pred = svm_clf_rbf.predict(test_data)
# SVM_rbf = model_eval(test_label, svm_clf_rbf_pred)
# print('SVM using rbf kernel : %.2f percent.' % (round(SVM_rbf['accuracy']*100,2)))

# # Linear kernel
# svm_clf_linear = svm.SVC(kernel = 'linear')
# svm_clf_linear.fit(train_data, train_label)
# svm_clf_linear_pred = svm_clf_linear.predict(test_data)
# SVM_linear = model_eval(test_label, svm_clf_linear_pred)
# print('SVM using linear kernel : %.2f percent.' % (round(SVM_linear['accuracy']*100,2)))


# # Poly kernal
# svm_clf_poly = svm.SVC(kernel = 'poly')
# svm_clf_poly.fit(train_data, train_label)
# svm_clf_poly_pred = svm_clf_poly.predict(test_data)
# SVM_poly = model_eval(test_label, svm_clf_poly_pred)
# print('SVM using poly kernel : %.2f percent.' % (round(SVM_poly['accuracy']*100,2)))


# svm_clf_sigmoid = svm.SVC(kernel = 'sigmoid')
# svm_clf_sigmoid.fit(train_data, train_label)
# svm_clf_sigmoid_pred = svm_clf_sigmoid.predict(test_data)
# SVM_sigmoid = model_eval(test_label, svm_clf_sigmoid_pred)
# print('SVM using sigmoid kernel : %.2f percent.' % (round(SVM_sigmoid['accuracy']*100,2)))



# #printmd('### 3.3.2. Model Evaulation ')
# ovl_svm = round(pd.DataFrame([SVM_rbf, SVM_linear, SVM_poly, SVM_sigmoid], index = ['SVM_rbf','SVM_linear', 'SVM_poly', 'SVM_sigmoid']),4)
# display(ovl_svm)


# # ### 4.2.4. Ensemble Models
# # 
# # ### 4.2.4.1. Random Forest
# # For the random forests classifier, I experimented with the number of trees, splitting criteria, minimum samples required to split, max depth of the tree, minimum samples required at the leaf level and the maximum features to consider when looking for the best split. The following values of the parameters attained the best accuracy during classification. Results in the table below.
# # 
# # *	**Num estimators:** 100 (Best amongst 10, 50 and 100)
# # *	**Splitting criteria:** Gini Index (Using Gini Index marginally outperformed Entropy with a higher accuracy.)
# # *	**Min samples required to split:** 5% (Best amongst 1%, 10% and 5%.)
# # *	**Max Depth:** None
# # *	**Min samples required at leaf:**  0.1 % (Best amongst 1%, 5% and 0.1%.)
# # *	**Max features:** number of features (Performs better than 'auto', 'log2' and 'sqrt'.)
# # 

# # In[78]:


# # Gini
# r_forest_gini = RandomForestClassifier(n_estimators=100, criterion = 'gini', max_features = None,  min_samples_split = 0.05, min_samples_leaf = 0.001)
# r_forest_gini.fit(train_data, train_label)
# r_forest_gini_pred = r_forest_gini.predict(test_data)
# rforest_gini = model_eval(test_label, r_forest_gini_pred)
# print('Random Forest using Gini Index : %.2f percent.' % (round(rforest_gini['accuracy']*100,2)))

# # Entropy
# r_forest_entropy = RandomForestClassifier(n_estimators=100, criterion = 'entropy', max_features = None,  min_samples_split = 0.05, min_samples_leaf = 0.001)
# r_forest_entropy.fit(train_data, train_label)
# r_forest_entropy_pred = r_forest_entropy.predict(test_data)
# rforest_entropy = model_eval(test_label, r_forest_entropy_pred)
# print('Random Forest using Entropy : %.2f percent.' % (round(rforest_entropy['accuracy']*100,2)))

# #printmd('### 3.4.1.2. Model Evaulation ')
# ovl_rf = round(pd.DataFrame([rforest_gini, rforest_entropy], index = ['rforest_gini','rforest_entropy']),4)
# display(ovl_rf)


# # ### 4.2.4.2. Adaboost
# # For the adaboost classifier, I experimented with base estimator from which the boosted ensemble is built and number of estimators. The following values of the parameters attained the best accuracy during classification. Results in the table below.
# # 
# # *	**Base Estimator:** DecisionTreeClassifier
# # *	**Num estimators:** 100 (Best amongst 10, 50 and 100.)
# # 

# # In[79]:


# ada = AdaBoostClassifier(n_estimators=100)                     
# ada.fit(train_data, train_label)
# ada_pred = ada.predict(test_data)
# adaboost = model_eval(test_label, ada_pred)
# print('Adaboost : %.2f percent.' % (round(adaboost['accuracy']*100,2)))

# #printmd('### 3.4.2.2. Model Evaulation ')
# ovl_ada = round(pd.DataFrame([adaboost], index = ['adaboost']),4)
# display(ovl_ada)


# # ### 4.2.5. Logistic Regression

# # In[80]:


# log_reg = LogisticRegression(penalty = 'l2', dual = False, tol = 1e-4, fit_intercept = True, 
#                             solver = 'liblinear')
# log_reg.fit(train_data, train_label)
# log_reg_pred = log_reg.predict(test_data)
# logistic_reg = model_eval(test_label, log_reg_pred)
# print('Logistic Regression : %.2f percent.' % (round(logistic_reg['accuracy']*100,3)))

# #printmd('### 3.5.2. Model Evaulation ')
# ovl_logreg = round(pd.DataFrame([logistic_reg], index = ['logistic_reg']),4)
# display(ovl_logreg)


# # ### 4.2.6. k Nearest Neighbours
# # For the K nearest neighbours classifier, I experimented with the num of neighbours values, every odd number ranging from 1 to 50.

# # In[90]:


# from sklearn.neighbors import KNeighborsClassifier
# knn_outs = []
# for i in range(1,50,2):
#     knn = KNeighborsClassifier(n_neighbors=i)
#     knn.fit(train_data, train_label) 
#     knn_pred = knn.predict(test_data)
#     knn_perf = model_eval(test_label, knn_pred)
#     knn_perf['k'] = i
#     knn_outs.append(knn_perf)

# ovl_knn = round(pd.DataFrame(knn_outs),4)
# display(ovl_knn)


# # ---
# # # 5. Model Evaluation
# # ## 5.1. Overall Performance Statistics

# # In[83]:


# overall_eval = pd.concat([ovl_dtree, ovl_ann, ovl_svm, ovl_rf, ovl_ada, ovl_logreg], axis = 0)
# overall_eval.sort_values(by = ['f_measure', 'accuracy'], ascending = False, inplace = True)

# # printmd('Combing the performance statistics of all the model developed, as seen in table below,         we see that the ensemble model Adaboost hast the highest F-measure (0.6833), precision (0.7812)         and accuracy (0.8647). The Artificial neural network models are only marginally being in terms of         accuracy and F-measure. Almost all the model have an accuracy greater than 0.84, expect for two SVM         models. The table below lists the accuracy, error rate, F-measure, precision, recall, sensitivity and         specificity of all the models developed.')

# display(overall_eval)


# # ## 5.2. ROC Curve

# # In[84]:


# def generateRoc(test_data, test_label, classifiers, pred_labels, plot_labels, limiter):
    
#     color = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan']
    
#     y_test = label_binarize(test_label, classes=['<=50K', '>50K'])
#     plt.figure()
    
#     for i in range(len(classifiers)):
        
#         if plot_labels[i] not in limiter:
#             continue
        
#         y_score = classifiers[i].predict_proba(test_data)
#         pos_class_index = list(np.unique(pred_labels[i])).index('>50K')
        
#         fpr, tpr, thres = metrics.roc_curve(y_test.ravel(),y_score[:,pos_class_index], pos_label=1)
                               
#         lw = 2
#         plt.plot(fpr, tpr, color=color[i % len(color)],lw=lw, label=plot_labels[i])
        
#     plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
#     plt.xlim([0.0, 1.0])
#     plt.ylim([0.0, 1.05])
#     plt.xlabel('False Positive Rate')
#     plt.ylabel('True Positive Rate')
#     plt.title('Receiver operating characteristic (ROC)')
#     plt.legend(loc="lower right")
#     plt.show()


# # In[86]:


# get_ipython().run_line_magic('matplotlib', 'notebook')
# classifier_list = [clf_gini
#                 ,clf_entropy
#                 ,ann_tanh
#                 ,ann_relu
#                 ,ann_log
#                 ,ann_identity
# #                 ,svm_clf_rbf
# #                 ,svm_clf_linear
# #                 ,svm_clf_poly
# #                 ,svm_clf_sigmoid
#                 ,r_forest_gini
#                 ,r_forest_entropy
#                 ,ada
#                 ,log_reg
#                 ] 
# pred_list = [clf_gini_pred
#             ,clf_entropy_pred
#             ,ann_tanh_pred
#             ,ann_relu_pred
#             ,ann_log_pred
#             ,ann_identity_pred
# #             ,svm_clf_rbf_pred
# #             ,svm_clf_linear_pred
# #             ,svm_clf_poly_pred
# #             ,svm_clf_sigmoid_pred
#             ,r_forest_gini_pred
#             ,r_forest_entropy_pred
#             ,ada_pred
#             ,log_reg_pred
#             ]

# clf_labels = ['DTree Gini'
#             ,'DTree Entropy'
#             ,'ANN TanH'
#             ,'ANN relu'
#             ,'ANN Logistic'
#             ,'ANN Identity'
# #             ,svm_clf_rbf_pred
# #             ,svm_clf_linear_pred
# #             ,svm_clf_poly_pred
# #             ,svm_clf_sigmoid_pred
#             ,'RForest Gini'
#             ,'RForest Entropy'
#             ,'Adaboost'
#             ,'Logistic Regression'
#             ]

# limiter = ['Adaboost', 'ANN TanH', 'ANN relu', 'ANN Logistic', 'Logistic Regression']
# generateRoc(test_data, test_label, classifier_list, pred_list, clf_labels, limiter)


# # The plot above of the receiver operating characteristic curve for the top 5 models; Adaboost, ANN with logistics activation function, ANN with relu activation function, ANN with tanH activation function and logistic regression model. I chose to plot only the top 5 models as the ROC curves of most of the models overlap and the it is not easy to interpret the curve. 
# # 
# # From figure, we can see that the ROC curve of the Adaboost model has the highest lift and is closest to the top left corner (TPR of 1 and FPR of 0) of the plot. The Adaboost model's curve clearly separates itself from the ROC curves of the other 4 models, which overlap with each other. 

# # ---
# # # 5. Conclusion

# # I choose **Adaboost** model as my preferred my approach. The Adaboost model not only has the **highest accuracy**, but also has the **highest precision and F-measure** of all the models developed as a part of this analysis. The advantages of using Adaboost over other models is that they are very simple to implement. Since they are made up of weak individual learners, they are less susceptible to overfitting. However, Adaboost is sensitive to noisy data and outliers. 
# # 