from sklearn import datasets
import numpy as np

# Read Data
iris = datasets.load_iris()

X = iris.data[:, [2, 3]]
y = iris.target

# Split dataset into SEPARATE Training and Test Datasets
from sklearn.cross_validation import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

# Standardize the Features for Gradient Descent
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()

sc.fit(X_train)  # using the 'fit' method to estimate the parameters, sample mean and standard deviation, for EACH feature dimension

X_train_std = sc.transform(X_train)
X_test_std = sc.transform(X_test)

# Train a Perceptron Model
from sklearn.linear_model import Perceptron

ppn = Perceptron(n_iter=40, eta0=0.1, random_state=0)
ppn.fit(X_train_std, y_train)

y_pred = ppn.predict(X_test_std)
print('Misclassified samples: %d' % (y_test != y_pred).sum())

# Calculate the Classification Accuracy
from sklearn.metrics import accuracy_score

print('Accuracy: %.2f' % accuracy_score(y_test, y_pred))

# Plot
from matplotlib.colors import ListedColormap
import matplotlib.pyplot as plt


def plot_decision_regions(X, y, classifier, test_idx=None, resolution=0.02):
    # setup marker generator and color map
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface
    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution), np.arange(x2_min, x2_max, resolution))

    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
    Z = Z.reshape(xx1.shape)

    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    # plot all samples
    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],
                    alpha=0.8, c=cmap(idx),
                    marker=markers[idx], label=cl)

    # highlist TEST samples
    if test_idx:
        X_test, y_test = X[test_idx, :], y[test_idx]
        plt.scatter(X_test[:, 0], X_test[:, 1], c='y',
                    alpha=1.0, marker='o',
                    s=20, label='test set')


X_combined_std = np.vstack((X_train_std, X_test_std))
y_combined = np.hstack((y_train, y_test))

# plot_decision_regions(X=X_combined_std, y=y_combined, classifier=ppn, test_idx=range(105, 150))
#
# plt.xlabel('petal length [standardized]')
# plt.ylabel('petal width [standardized]')
# plt.legend(loc='upper left')
# plt.title('Perceptron')
# plt.show()
#
#
# # Logistic Regression
# from sklearn.linear_model import LogisticRegression
# lr = LogisticRegression(C=1000.0, random_state=0)
#
# lr.fit(X_train_std, y_train)
#
# plot_decision_regions(X_combined_std, y_combined, classifier=lr, test_idx=range(105, 150))
#
# plt.xlabel('petal length [standardized]')
# plt.ylabel('petal width [standardized]')
# plt.legend(loc='upper left')
# plt.title('Logistic Regression')
# plt.show()
#
# # print(lr)
# # lr.predict_proba(X_test_std[0, :])
#
# # Regularization Parameter Visualization
# weights, params = [], []
#
# for c in np.arange(-5, 5):
#     lr = LogisticRegression(C=10.0**c, random_state=0)
#     lr.fit(X_train_std, y_train)
#     weights.append(lr.coef_[1])
#     params.append(10.0**c)
#
# weights = np.array(weights)
#
# plt.plot(params, weights[:, 0], label='petal length')
# plt.plot(params, weights[:, 1], linestyle='--' ,label='petal width')
# plt.ylabel('weights coefficient')
# plt.xlabel('C')
#
# plt.legend(loc = 'upper left')
# plt.xscale('log')
# plt.show()

# # SVM
# from sklearn.svm import SVC
#
# svm = SVC(kernel='linear', C=1.0, random_state=0)
# svm.fit(X_train_std, y_train)
#
# plot_decision_regions(X_combined_std, y_combined, classifier=svm, test_idx=range(105, 150))
# plt.xlabel('petal length [standardized]')
# plt.ylabel('petal width [standardized]')
# plt.legend(loc='upper left')
# plt.title('SVM')
#
# plt.show()

# SGDClassifier (Stochastic GD Classifier)
'''
Also supports online learning via the partial_fit method.
We could initialize the stochastic gradient descent version of the Percetron, LR and SVM with default parameters as follow
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier
'''

# from sklearn.linear_model import SGDClassifier
# ppn = SGDClassifier(loss='perceptron')
# lr = SGDClassifier(loss='log')
# svm = SGDClassifier(loss='hinge')

# Solving Non-Linear Problems using a Kernel SVM
## Create a simple dataset that has the form of an XOR gate
np.random.seed(0)

X_xor = np.random.randn(200, 2)
y_xor = np.logical_xor(X_xor[:, 0]>0, X_xor[:, 1]>0)
y_xor = np.where(y_xor, 1, -1)
#
# plt.scatter(X_xor[y_xor==1, 0], X_xor[y_xor==1, 1], c='b', marker='x', label='1')
# plt.scatter(X_xor[y_xor==-1, 0], X_xor[y_xor==-1, 1], c='r', marker='s', label='-1')
# plt.ylim(-3.0)
# plt.legend()
# plt.show()

# svm = SVC(kernel='rbf', random_state=0, gamma=0.1, C=10)
# svm.fit(X_xor, y_xor)
# plot_decision_regions(X_xor, y_xor, classifier=svm)
# plt.legend(loc='upper left')
# plt.show()
#
# svm = SVC(kernel='rbf', random_state=0, gamma=0.2, C=1.0)
# svm.fit(X_train_std, y_train)
#
# plot_decision_regions(X_combined_std, y_combined, classifier=svm, test_idx=range(105, 150))
# plt.xlabel('petal length [standardized]')
# plt.ylabel('petal width [standardized]')
# plt.legend(loc='upper left')
# plt.title('RBF Kernel SVM, Small Gamma, Soft Boundary')
# plt.show()
#
# svm = SVC(kernel='rbf', random_state=0, gamma=100.0, C=1.0)
# svm.fit(X_train_std, y_train)
#
# plot_decision_regions(X_combined_std, y_combined, classifier=svm, test_idx=range(105, 150))
# plt.xlabel('petal length [standardized]')
# plt.ylabel('petal width [standardized]')
# plt.legend(loc='upper left')
# plt.title('RBF Kernel SVM, Large Gamma, Much Tighter')
# plt.show()

# # Different Impurity Criteria Visualization
# def gini(p):
#     return p*(1-p) + (1-p)*(1-(1-p))
#
# def entropy(p):
#     return -p*np.log2(p) - (1-p)*np.log2(1-p)
#
# def error(p):
#     return 1-np.max([p, 1-p])
#
# x = np.arange(0.0, 1.0, 0.01)
# ent = [entropy(p) if p!= 0 else None for p in x]
# sc_ent = [e*0.5 if e else None for e in ent]
# err = [error(i) for i in x]
# fig = plt.figure()
# ax = plt.subplot(111)
# for i, lab, ls, c in zip([ent, sc_ent, gini(x), err],
#                          ['Entropy', 'Entropy(scaled)', 'Gini Impurity', 'Misclassification'],
#                          ['-', '-', '--', '-.'],
#                          ['black', 'lightgray', 'red', 'green', 'cyan']):
#     line = ax.plot(x, i, label=lab, linestyle=ls, lw=2, color=c)
#
# ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.15), ncol=3, fancybox=True, shadow=False)
# ax.axhline(y=0.5, linewidth=1, color='k', linestyle='--')
# ax.axhline(y=1.0, linewidth=1, color='k', linestyle='--')
#
# plt.ylim([0, 1.1])
# plt.xlabel('p(i=1')
# plt.ylabel('Impurity Index')
# plt.show()

# Building a Decision Tree
'''
Train a decision tree with a Maximum-Depth of 3 using Entropy as a criterion for impurity
'''
from sklearn.tree import DecisionTreeClassifier
tree = DecisionTreeClassifier(criterion='entropy', max_depth=3, random_state=0)

tree.fit(X_train, y_train)
X_combined = np.vstack((X_train, X_test))
y_combined = np.hstack((y_train, y_test))

plot_decision_regions(X_combined, y_combined, classifier=tree, test_idx=range(105, 150))
plt.xlabel('petal length [cm]')
plt.ylabel('petal width [cm]')
plt.legend(loc='upper left')
plt.show()

# Decision tree can be exported as a .dot file after training, which we can visualize using the GraphViz program
# from sklearn.tree import export_graphviz
#
# export_graphviz(tree, out_file='tree.dot', feature_names=['petal length', 'petal width'])

# Combining weak to strong learners via Random Forests
from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier(criterion='entropy', n_estimators=10, random_state=1, n_jobs=2)
forest.fit(X_train, y_train)

plot_decision_regions(X_combined, y_combined, classifier=forest, test_idx=range(105, 150))
plt.xlabel('petal length')
plt.ylabel('petal width')
plt.legend(loc='upper left')
plt.show()

# KNN - Lazy Algorithm
from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=5, p=2, metric='minkowski')
knn.fit(X_train_std, y_train)

plot_decision_regions(X_combined_std, y_combined, classifier=knn, test_idx=range(105, 150))
plt.xlabel('petal length [standardized]')
plt.ylabel('petal width [standardized]')
plt.show()