from __future__ import division
import numpy as np
from numpy import *
import matplotlib.pyplot as plt
import time
import scipy as sc
from scipy.optimize import *


from plotData import plotData
from mapFeature import mapFeature
from costFunctionReg import costFunctionReg
from costFunctionReg import costFunctionReg_0
from costFunctionReg import gradFunctionReg
from plotDecisionBoundary import plotDecisionBoundary
from predict import predict

#Initialization

plt.clf() ; plt.close('all')

#Load data

# The first two columns contains the X values and the third column
# contains the label (y)

data = loadtxt("C:\\Python37\\Data\\ex2data2.txt", delimiter = ",")

X = data[:,0:2] ; y = data[:, 2]

plotData(X, y)

# Labels and legend

plt.xlabel("Microchip Test 1")
plt.ylabel("Microchip Test 2")

# Specified in plot order

plt.legend(["y = 1", "y = 0"])
plt.show()

#### Part 1: Regularized Logistic Regression

#  In this part, you are given a dataset with data points that are not
#  linearly separable. However, you would still like to use logistic
#  regression to classify the data points.
#
#  To do so, you introduce more features to use -- in particular, you add
#  polynomial features to our data matrix (similar to polynomial
#  regression).


# Add Polynomial Features

# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled

X = mapFeature(X[:,0], X[:,1])

# Initializae fitting parameters
initial_theta = np.zeros((X.shape[1],))

# Set regularization parameter to 1
lambdaa = 1

# Compute and display initial cost and gradient for regularized logistic
# regression

cost,grad = costFunctionReg(initial_theta, X, y, lambdaa)

print('Cost at initial theta (zeros): %f\n' %cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:\n')
print(grad[0:5])
print('Expected gradients (approx) - first five values only:\n')
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')

print('\nProgram paused. Press enter to continue.\n')
time.sleep(2)

# Compute and display cost and gradient
# with all-ones theta and lambda = 10

test_theta = np.ones((X.shape[1],))

cost,grad = costFunctionReg(test_theta, X, y, 10)

print('Cost at test theta (with lambdaa = 10): %f\n' %cost)
print('Expected cost (approx): 3.16\n')
print('Gradient at test theta - first five values only:\n')
print(grad[0:5])
print('Expected gradients (approx) - first five values only:\n')
print(' 0.3460\n 0.1614\n 0.1948\n 0.2269\n 0.0922\n')

print('\nProgram paused. Press enter to continue.\n')
time.sleep(2)

#### Part 2: Regularization and Accuracies
# Optional exercise :

#  In this part, you will get to try different values of lambda and
#  see how regularization affects the decision coundart
#
#  Try the following values of lambda (0, 1, 10, 100).
#
#  How does the decision boundary change when you vary lambda? How does
#  the training set accuracy vary?
#

# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1],))

# Set regularization parameter lambdaa to 1 (you should vary this)
lambdaa = 1

# Optional
#lambdaa = 0 # overfitting the data e.g it predicts x =[-0.25, 1.5] is accepted (y=1) which seems to be an incorrect decision given the training set.
#lambdaa = 100 # underfitting the data i.e not a good fit and the decision boundary is not following the data so well.

#### Part 3 : Optimizing using fmin_ncg

opts = {'disp': disp}
theta, cost, u, v, w ,x = fmin_ncg(f = costFunctionReg_0,
                                   x0= initial_theta,
                                   fprime= gradFunctionReg,
                                   args = (X, y, lambdaa),
                                   maxiter = 400,
                                   full_output = 1
                                    )
# from the guide

#    u = fcalls : int
#        Number of function calls made.
#    v = gcalls : int
#        Number of gradient calls made.
#    w = hcalls : int
#        Number of hessian calls made.
#    x = warnflag : int
#        Warnings generated by the algorithm.
#        1 : Maximum number of iterations exceeded.

#Plot Boundary

plotDecisionBoundary(theta, X, y)
plt.title("lambda = %g" %lambdaa )

#Labels and legend
plt.xlabel("Microship Test 1")
plt.ylabel("Microship Test 2")

plt.legend(["y = 1", "y = 0", "Decision Boundary"], loc = 'best')
plt.show()

# Compute accuracy on our training set
p = predict(theta, X)

print("\nTrain Accuracy : %f\n"%(np.mean( p == y ) * 100))
print("Expected accuracy (with lambda = 1): 83.1 (approx)\n")

