{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from random import randint\n",
    "import h5py\n",
    "\n",
    "\n",
    "class NNetwork:\n",
    "    def __init__(self, dimension, hidden_units=100, classes=10):\n",
    "       \n",
    "        self.d = dimension\n",
    "        self.d_h = hidden_units\n",
    "        self.k = classes\n",
    "\n",
    "        np.random.seed(72)\n",
    "        self.w = np.random.randn(self.d_h, self.d) * math.sqrt(2.0 / self.d)\n",
    "        self.b1 = np.zeros(self.d_h)\n",
    "        self.c = np.random.randn(self.k, self.d_h) * math.sqrt(2.0 / self.d_h)\n",
    "        self.b2 = np.zeros(self.k)\n",
    "\n",
    "    def train(self, training_data, learning_rate=0.1, epochs=100):\n",
    " \n",
    "        X = training_data[0]\n",
    "        Y = training_data[1]\n",
    "\n",
    "      \n",
    "\n",
    "        for epoch in range(1, epochs + 1):\n",
    "            if epoch > epochs // 20 :\n",
    "                learning_rate = 0.02\n",
    "            elif epoch > 2 * (epochs // 20 ):\n",
    "                learning_rate = 0.002\n",
    "            else:\n",
    "                learning_rate = 0.0002\n",
    "\n",
    "            correct_sum = 0\n",
    "            for _ in range(X.shape[0]):\n",
    "                index = randint(0, X.shape[0] - 1)\n",
    "                x = X[index]\n",
    "                y = Y[index]\n",
    "\n",
    "                \n",
    "                z, h, u, f = self.forward(x)\n",
    "                g_c, g_b2, g_b1, g_w = self.backward(x, y, z, h, u, f)\n",
    "                self.weight_update(learning_rate, g_c, g_b2, g_b1, g_w)\n",
    "\n",
    "                if self.predict(f) == y:\n",
    "                    correct_sum += 1\n",
    "\n",
    "            acc = correct_sum / np.float(X.shape[0])\n",
    "            print(\"epoch {}, training accuracy = {}\".format(epoch, acc))\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"Calculate output f and intermediary network values.\"\"\"\n",
    "        z = np.matmul(self.w, x) + self.b1\n",
    "        h = relu(z)\n",
    "        # h = sigmoid(z)\n",
    "        u = np.matmul(self.c, h) + self.b2\n",
    "        f = softmax(u)\n",
    "        return z, h, u, f\n",
    "\n",
    "    def backward(self, x, y, z, h, u, f):\n",
    "        \"\"\"Calculate the gradient w.r.t parameters.\"\"\"\n",
    "        e_y = np.zeros(self.k)\n",
    "        e_y[y] = 1\n",
    "        de_u = - (e_y - f)\n",
    "\n",
    "        de_b2 = de_u\n",
    "        de_c = np.matmul(de_u[:, np.newaxis], h[np.newaxis, :])  # col * row\n",
    "\n",
    "        delta = np.matmul(self.c.T, de_u)\n",
    "        relu_prime = gradient_for_relu(z)\n",
    "        de_b1 = np.multiply(delta, relu_prime)\n",
    "        de_w = np.matmul(de_b1[:, np.newaxis], x[np.newaxis, :])  # col * row\n",
    "\n",
    "        return de_c, de_b2, de_b1, de_w\n",
    "\n",
    "    def weight_update(self, learning_rate, g_c, g_b2, g_b1, g_w):\n",
    "        self.c -= learning_rate * g_c\n",
    "        self.b2 -= learning_rate * g_b2\n",
    "        self.b1 -= learning_rate * g_b1\n",
    "        self.w -= learning_rate * g_w\n",
    "\n",
    "    def predict(self, f):\n",
    "        return np.argmax(f)\n",
    "\n",
    "    def test(self, test_data):\n",
    "        \"\"\"Test the trained model.\n",
    "        Args:\n",
    "            test_data(tuple): x_test(10000, 784) and y_test(10000,)\n",
    "        \"\"\"\n",
    "        X = test_data[0]\n",
    "        Y = test_data[1]\n",
    "        correct_sum = 0\n",
    "\n",
    "        for i in range(X.shape[0]):\n",
    "            x = X[i]\n",
    "            y = Y[i]\n",
    "            _, _, _, f = self.forward(x)\n",
    "            if self.predict(f) == y:\n",
    "                correct_sum += 1\n",
    "\n",
    "        acc = correct_sum / np.float(X.shape[0])\n",
    "        print(\"testing accuracy = {}\".format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def load_data(path):\n",
    "    \"\"\"\n",
    "    Read data from HDF5 files.\n",
    "    Returns:\n",
    "        x_train (60000, 784)\n",
    "        y_train (60000,)\n",
    "        x_test  (10000, 784)\n",
    "        y_test  (10000,)\n",
    "    \"\"\"\n",
    "    data_hf = h5py.File(path, 'r')\n",
    "    x_train = np.array(data_hf['x_train'])\n",
    "    y_train = np.array(data_hf['y_train'][:, 0])\n",
    "    x_test = np.array(data_hf['x_test'])\n",
    "    y_test = np.array(data_hf['y_test'][:, 0])\n",
    "\n",
    "    data_hf.close()\n",
    "    return x_train, y_train, x_test, y_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(z, 0)\n",
    "\n",
    "def gradient_for_relu(z):\n",
    "    z[z >= 0] = 1\n",
    "    z[z < 0] = 0\n",
    "    return z\n",
    "\n",
    "def softmax(z):\n",
    "    return np.exp(z) / np.sum(np.exp(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training model...\n",
      "epoch 1, training accuracy = 0.8179666666666666\n",
      "epoch 2, training accuracy = 0.9436833333333333\n",
      "epoch 3, training accuracy = 0.9700333333333333\n",
      "epoch 4, training accuracy = 0.9783666666666667\n",
      "epoch 5, training accuracy = 0.9818333333333333\n",
      "epoch 6, training accuracy = 0.9844\n",
      "epoch 7, training accuracy = 0.9862666666666666\n",
      "epoch 8, training accuracy = 0.9888166666666667\n",
      "epoch 9, training accuracy = 0.9901166666666666\n",
      "epoch 10, training accuracy = 0.99175\n",
      "epoch 11, training accuracy = 0.9923833333333333\n",
      "epoch 12, training accuracy = 0.9923666666666666\n",
      "epoch 13, training accuracy = 0.99405\n",
      "epoch 14, training accuracy = 0.9942166666666666\n",
      "epoch 15, training accuracy = 0.99445\n",
      "epoch 16, training accuracy = 0.9949666666666667\n",
      "epoch 17, training accuracy = 0.9964333333333333\n",
      "epoch 18, training accuracy = 0.9962833333333333\n",
      "epoch 19, training accuracy = 0.99765\n",
      "epoch 20, training accuracy = 0.99765\n",
      "testing model...\n",
      "testing accuracy = 0.9768\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def main():\n",
    "    \"\"\"High level pipeline.\"\"\"\n",
    "\n",
    "    data = load_data(\"MNISTdata.hdf5\")\n",
    "    training_data, test_data = data[:2], data[2:]\n",
    "\n",
    "    nn = NNetwork(training_data[0].shape[1], hidden_units=150)\n",
    "\n",
    "    print(\"training model...\")\n",
    "    nn.train(training_data, learning_rate=0.1, epochs=20)\n",
    "\n",
    "    print(\"testing model...\")\n",
    "    nn.test(test_data)\n",
    "\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
