import numpy as np
from random import randrange

def rel_error(x, y):
    """Relative error between x and y."""
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))


def eval_numerical_gradient(f, x):
    """A naive implementation of numerical gradient of f at x
        - f should be a function that takes a single argument
        - x is the point (numpy array) to evaluate the gradient at
    """
    fx = f(x)  # evaluate function value at original point
    grad = np.zeros(x.shape)
    h = 0.00001

    # iterate over all indexes in x
    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        # evaluate function at x+h
        ix = it.multi_index
        x[ix] += h  # increment by h
        fxh = f(x)  # evalute f(x + h)
        x[ix] -= h  # restore to previous value (very important!)
        # compute the partial derivative
        grad[ix] = (fxh - fx) / h  # the slope
        it.iternext()  # step to next dimension
    return grad


def grad_check_sparse(f, x, analytic_grad, num_checks):
    """Run some checks of the computed analytic gradient vs. numeric gradient.
    """
    h = 1e-5

    for i in xrange(num_checks):
        ix = tuple([randrange(m) for m in x.shape])

        x[ix] += h  # increment by h
        fxph = f(x)  # evaluate f(x + h)
        x[ix] -= 2 * h  # increment by h
        fxmh = f(x)  # evaluate f(x - h)
        x[ix] += h  # reset

        grad_numerical = (fxph - fxmh) / (2 * h)
        grad_analytic = analytic_grad[ix]
        rel_error = abs(grad_numerical - grad_analytic) / (
            abs(grad_numerical) + abs(grad_analytic))
        print 'numerical: %f analytic: %f, relative error: %e' % (
            grad_numerical, grad_analytic, rel_error)

import numpy as np
from random import randrange

def rel_error(x, y):
    """Relative error between x and y."""
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))


def eval_numerical_gradient(f, x):
    """A naive implementation of numerical gradient of f at x
        - f should be a function that takes a single argument
        - x is the point (numpy array) to evaluate the gradient at

        NOTE: don't use this -- the centered formula is much better
        as described in http://cs231n.github.io/neural-networks-3/#gradcheck
    """
    fx = f(x)  # evaluate function value at original point
    grad = np.zeros(x.shape)
    h = 0.00001

    # iterate over all indexes in x
    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        # evaluate function at x+h
        ix = it.multi_index
        x[ix] += h  # increment by h
        fxh = f(x)  # evalute f(x + h)
        x[ix] -= h  # restore to previous value (very important!)
        # compute the partial derivative
        grad[ix] = (fxh - fx) / h  # the slope
        it.iternext()  # step to next dimension
    return grad


def grad_check_sparse(f, x, analytic_grad, num_checks):
    """Run some checks of the computed analytic gradient vs. numeric gradient.
    """
    h = 1e-5

    for i in xrange(num_checks):
        ix = tuple([randrange(m) for m in x.shape])

        x[ix] += h  # increment by h
        fxph = f(x)  # evaluate f(x + h)
        x[ix] -= 2 * h  # increment by h
        fxmh = f(x)  # evaluate f(x - h)
        x[ix] += h  # reset

        grad_numerical = (fxph - fxmh) / (2 * h)
        grad_analytic = analytic_grad[ix]
        rel_error = abs(grad_numerical - grad_analytic) / (
            abs(grad_numerical) + abs(grad_analytic))
        print 'numerical: %f analytic: %f, relative error: %e' % (
            grad_numerical, grad_analytic, rel_error)

Update more comments