import numpy as np
from LossFunction import L

# random search bad idea - option 1
# assume X_train is the data where each column is an example (e.g. 3073 x 50,000)
# assume Y_train are the labels (e.g. 1D array of 50,000)
# assume the function L evaluates the loss function

X_train = [['data'] * 3073] * 50000
Y_train = ['label'] * 50000

bestloss = float("inf")  # Python assigns the highest possible float value
for num in range(1000):
    W = np.random.randn(10, 3073) * 0.0001  # generate random parameters
    loss = L(X_train, Y_train, W)  # get the loss over the entire training set
    if loss < bestloss:  # keep track of the best solution
        bestloss = loss
        bestW = W
    print('in attempt %d the loss was %f, best %f' % (num, loss, bestloss))

# prints:
# in attempt 0 the loss was 9.401632, best 9.401632
# in attempt 1 the loss was 8.959668, best 8.959668
# in attempt 2 the loss was 9.044034, best 8.959668
# in attempt 3 the loss was 9.278948, best 8.959668
# in attempt 4 the loss was 8.857370, best 8.857370
# in attempt 5 the loss was 8.943151, best 8.857370
# in attempt 6 the loss was 8.605604, best 8.605604
# ... (trunctated: continues for 1000 lines)

Wbest = bestloss

X_test = 'matrix with test values in it'
Xte_cols = X_test['columns']  # mmm test columns
# this isn't a real bit of code, it's just notes
# this entire goddamn file is notes
# ffs
# I'm just cleaning it up because otherwise pycharm gives me stupid red lines
Yte = 'Y testing labels'

# Assume X_test is [3073 x 10000], Y_test [10000 x 1]
scores = Wbest.dot(Xte_cols)  # 10 x 10000, the class scores for all test examples
# find the index with max score in each column (the predicted class)
Yte_predict = np.argmax(scores, axis=0)
# and calculate accuracy (fraction of predictions that are correct)
np.mean(Yte_predict == Yte)
# returns 0.1555

# option 2 - random local search
Xtr_cols = X_train['columns']  # pick some random values
Ytr = 'Y training labels'

W = np.random.randn(10, 3073) * 0.001  # generate random starting W
bestloss = float("inf")
for i in range(1000):
    step_size = 0.0001
    Wtry = W + np.random.randn(10, 3073) * step_size
    loss = L(Xtr_cols, Ytr, Wtry)
    if loss < bestloss:
        W = Wtry
        bestloss = loss
    print('iter %d loss is %f' % (i, bestloss))


# option 3 - compute gradient
def eval_numerical_gradient(f, x):
    """
    a naive implementation of numerical gradient of f at x
    - f should be a function that takes a single argument
    - x is the point (numpy array) to evaluate the gradient at
    """

    fx = f(x)  # evaluate function value at original point
    grad = np.zeros(x.shape)
    h = 0.00001

    # iterate over all indexes in x
    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        # evaluate function at x+h
        ix = it.multi_index
        old_value = x[ix]
        x[ix] = old_value + h  # increment by h
        fxh = f(x)  # evalute f(x + h)
        x[ix] = old_value  # restore to previous value (very important!)

        # compute the partial derivative
        grad[ix] = (fxh - fx) / h  # the slope
        it.iternext()  # step to next dimension

    return grad


# to use the generic code above we want a function that takes a single argument
# (the weights in our case) so we close over X_train and Y_train
def CIFAR10_loss_fun(W):
    return L(X_train, Y_train, W)


W = np.random.rand(10, 3073) * 0.001  # random weight vector
df = eval_numerical_gradient(CIFAR10_loss_fun, W)  # get the gradient

loss_original = CIFAR10_loss_fun(W)  # the original loss
print('original loss: %f' % (loss_original,))

# lets see the effect of multiple step sizes
for step_size_log in [-10, -9, -8, -7, -6, -5, -4, -3, -2, -1]:
    step_size = 10 ** step_size_log
    W_new = W - step_size * df  # new position in the weight space
    loss_new = CIFAR10_loss_fun(W_new)
    print('for step size %f new loss: %f' % (step_size, loss_new))

    # prints:
    # original loss: 2.200718
    # for step size 1.000000e-10 new loss: 2.200652
    # for step size 1.000000e-09 new loss: 2.200057
    # for step size 1.000000e-08 new loss: 2.194116
    # for step size 1.000000e-07 new loss: 2.135493
    # for step size 1.000000e-06 new loss: 1.647802
    # for step size 1.000000e-05 new loss: 2.844355
    # for step size 1.000000e-04 new loss: 25.558142
    # for step size 1.000000e-03 new loss: 254.086573
    # for step size 1.000000e-02 new loss: 2539.370888
    # for step size 1.000000e-01 new loss: 25392.214036

# Vanilla Gradient Descent

while True:
    weights_grad = evaluate_gradient(loss_fun, data, weights)
    weights += - step_size * weights_grad  # perform parameter update

# Vanilla Minibatch Gradient Descent

while True:
    data_batch = sample_training_data(data, 256)  # sample 256 examples
    weights_grad = evaluate_gradient(loss_fun, data_batch, weights)
    weights += - step_size * weights_grad  # perform parameter update
