Function_Regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Function Regression.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "97c9ZW1T4xyW",
        "lBl5zths4lkf",
        "gBWqCuKTUFSx",
        "d4QVJ_HbS7Ff",
        "NhaDwRn8TCQK"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/avilash/neural-networks-numpy/blob/master/Function_Regression.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l2RiubY0VHSx",
        "colab_type": "text"
      },
      "source": [
        "This notebook trains a feed-forward neural network only in **numpy** to overfit to a standard mathematical function (in this case the **sine**) in a given range"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s60Hhc1WUNXa",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-T4u_C4nSbGu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "97c9ZW1T4xyW",
        "colab_type": "text"
      },
      "source": [
        "## Model Forward and Backward"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wud99NehVBEX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def act(x):\n",
        "  return np.maximum(x, 0.)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tLLN7hE_33zU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def act_back(incoming_gradient, activation):\n",
        "  incoming_gradient[activation == 0.] = 0.\n",
        "  return incoming_gradient"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XgONq8c1S4iF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_model(layer_sizes, input_size, output_size):\n",
        "  model = []\n",
        "  current_input_size = input_size\n",
        "  for i, layer_size in enumerate(layer_sizes):\n",
        "    layer = {}\n",
        "    layer[\"W\"] = np.random.randn(current_input_size, layer_size) / np.sqrt(current_input_size)\n",
        "    layer[\"b\"] = np.zeros(layer_size, dtype=np.float32)\n",
        "    model.append(layer)\n",
        "    current_input_size = layer_size\n",
        "    \n",
        "  layer = {}\n",
        "  layer[\"W\"] = np.random.randn(current_input_size, output_size) / np.sqrt(current_input_size)\n",
        "  layer[\"b\"] = np.zeros(output_size, dtype=np.float32)\n",
        "  model.append(layer)\n",
        "\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zu7hqIF0Tc8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def forward(input, model):\n",
        "  model_len = len(model)\n",
        "    \n",
        "  activations = []\n",
        "  current_act = input\n",
        "  activations.append(current_act)\n",
        " \n",
        "  \n",
        "  for i in range(model_len-1):\n",
        "    layer = model[i]\n",
        "    # print (\"prev_A = {}, W = {}, B = {}\".format(current_act.shape, layer['W'].shape, layer['b'].shape))\n",
        "    current_act = act(np.matmul(current_act, layer['W']) + layer['b'])\n",
        "    # print (\"A = {}\".format(current_act.shape))\n",
        "    activations.append(current_act)\n",
        "\n",
        "  layer = model[model_len-1]\n",
        "  # print (\"prev_A = {}, W = {}, B = {}\".format(current_act.shape, layer['W'].shape, layer['b'].shape))\n",
        "  pred = np.matmul(current_act, layer['W']) + layer['b']\n",
        "  # print (\"A = {}\".format(pred.shape))\n",
        "  activations.append(pred)\n",
        "  \n",
        "  \n",
        "  return activations, pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J3WHjZYvaCLI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def backward(gradient, activations, model):\n",
        "  gradients = []\n",
        "  Z_grad = gradient.copy()\n",
        "    \n",
        "  activation_len = len(activations)\n",
        "  \n",
        "  for i in range(activation_len-2, -1, -1):\n",
        "    grad = {}\n",
        "    prev_A = activations[i]\n",
        "    W = model[i]['W']\n",
        "    grad['W'] = prev_A.T.dot(Z_grad)\n",
        "    grad['b'] = np.sum(Z_grad, axis=0)\n",
        "    grad['A'] = Z_grad.dot(W.T)\n",
        "    gradients.insert(0, grad)\n",
        "    Z_grad = grad['A'].copy()\n",
        "    Z_grad = act_back(Z_grad, prev_A)\n",
        "    \n",
        "  return gradients\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZJ6neixzh0vb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ms_loss(y_pred, y):\n",
        "  loss = np.square(y_pred - y)\n",
        "  return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDASsqyPnzht",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ms_loss_back(y_pred, y):\n",
        "  loss_back = 2*(y_pred-y)\n",
        "  return loss_back"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lBl5zths4lkf",
        "colab_type": "text"
      },
      "source": [
        "## Data Generation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ElDyYxryWl47",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_data():\n",
        "  x = np.linspace(-1.0*np.pi, 1.0*np.pi, num=20000)\n",
        "  e = np.random.normal(0, 0.005, x.size)\n",
        "  y_f = np.sin(x)\n",
        "  y = np.sin(x) + e\n",
        "  return x, y_f, y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TLKngxZLXVWR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_data(x, y_f, y):\n",
        "  fig, ax = plt.subplots()\n",
        "  if y_f is not None:\n",
        "    ax.plot(x, y_f)\n",
        "  ax.scatter(x, y, s=1)\n",
        "  plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kDSUTbcjaf2A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_train_test(x, y, test_percentage):\n",
        "  x_size = x.shape[0]\n",
        "  test_size = int(round(test_percentage/100.*x_size))\n",
        "  indices = np.random.permutation(x.shape[0])\n",
        "  test_idx, train_idx = indices[:test_size], indices[test_size:]\n",
        "  x_test, x_train = x[test_idx], x[train_idx]\n",
        "  y_test, y_train = y[test_idx], y[train_idx]\n",
        "  \n",
        "  return x_train, y_train, x_test, y_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIJnUlN0-leB",
        "colab_type": "code",
        "outputId": "3caa89f7-499b-4ce9-dc7e-63f7ef203a8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "x, y_f, y = generate_data()\n",
        "x_train, y_train, x_test, y_test = split_train_test(x, y, 0.)\n",
        "plot_data(x_train, None, y_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt4VPW1N/DvEoxg8GgGBi+IISda\nEKjGOIA3kEJQjFXEok04VWJFXvFCK2rBKkeOYsEq3ngVpKhBa4MU4QWOWCXIxZZLCWlUQNBEiJiq\nRAJawiUC6/1jduzsy+Q2k9mzZ76f55kns9fek1lRMiu//buJqoKIiKjecW4nQERE8YWFgYiITFgY\niIjIhIWBiIhMWBiIiMiEhYGIiExYGIiIyISFgYiITFgYiIjIpK3bCbREp06dtFu3bm6nQUTkKZs2\nbfpGVf2NXefJwtCtWzeUlJS4nQYRkaeISGVTruOtJCIiMmFhICIiExYGIiIyYWEgIiITFgYiIjJh\nYSAiIpOoFAYReVlEdovI5jDnRUSeE5FyEflQRLJDzo0SkU+Nx6ho5ENERC0XrRZDIYChDZy/CsA5\nxmMMgJkAICI+AA8D6AegL4CHRSQtSjkRURyrqa3Di6srUFNb53YqZBGVCW6qukZEujVwyTAAr2pw\ng+n1InKKiJwOYCCA5apaAwAishzBAlMUjbyIKD5UVO/H3a9vwie7a3HkmHmf+alvb4MAqI+2ayu4\n9vwumJh7LnypKTHPlWI387kLgF0hx18YsXBxIvK40sq9uPnlDdh/+Gij14aWikNHFPM3fYH5m74A\nAAzq7seTN2axSMSQZ5bEEJExCN6GwllnneVyNkTkZNW23bjzT5tQW3csat/zve3VyH50OdoKMLJf\nOn495EcsEq0sVqOSqgB0DTk+04iFi9uo6mxVDahqwO9vdA0oIoqhmto6/OyFv6KgcGNUi0KoIwq8\nur4S2Y8ux+PLtrbKe1BQrArDEgA3G6OTLgLwrap+CeAdAFeISJrR6XyFESMij3h82VZkP7ocmz7/\nttmvbekH0Mw1O9Dv0XfZcd1KonIrSUSKEOxI7iQiXyA40uh4AFDVWQCWAcgFUA7gAIBbjHM1IvIo\ngI3Gt3qkviOaiOLbnDUVmLJsW5OuFQC/G94b+f3SG7yuprYOs1ZVYMW2r1FRXdvgtV/Xfo/sR5cj\no2N7vHnHZby9FEUSHCjkLYFAQLnsNpE7amrr8Ks/leL9ij0NXnfCccCoy/4Tt1+eGdGH9tKyKtw9\nr6zR62bkZeGaLI5daYiIbFLVQKPXsTAQUVPV1NbhoinLUdfAx4avfVv8+Y5LkenvENX3nbbs4x9G\nKoUz/PzT8HT+hVF730TDwkBEUfX4sq2YuWZH2PP9uqVh5k2BVr2lU1Nbh98u/BB/2fJ12Gtye3XG\nCzf1abUcvIyFgYii5rni7XiquDzs+YVjL0F2euwWLWhKgZjahD6NZNPUwsBF9IioQTf/YV3YovAj\n/4konTQkpkUBAHypKZh1UwALx14S9poHFm1GaeXeGGaVOFgYiMhRTW0dBj/xHtZUOA8UnJGXhXfv\n/Ymro4Gy09NQOmkIMju1dzx//cy1mLig8Y5rMmNhICJHV05fiYo9B23x1LaC0klD4mYEkC81BSvu\nG4Spw3s7np9XUoWiDZUxzsrbWBiIyKSmtg79pxWj+sAR2zl/alu8/0BOXM4ZyO+Xjhl5WY7nHli0\nGRXV+2OckXexMBCRycDfF2PXvsO2+MXdTsHGSVfGZVGod01Wl7D9DoOnr8ZzxdtjnJE3sTAQ0Q+G\nPbca3x22j1TM6d4JRbdf6kJGzZednoYV917ueO6p4nIsLXNcjo1CsDAQEYDg6KMP/mm/3VJw8VmY\nc0s/FzJquUx/h7Ath6bMok52LAxEhHuKNjmOPsoLdMHkYT92IaPINdRy6DbxLazatjvGGXkHCwNR\nklu1bTcWffCVLT52QAamjXDuzPWKTH8HlE4agrR29o+6gsKNXJ01DBYGoiRWUb0fBYUbbfGHcntg\nQm5PFzKKPl9qCv4x+SrHc4OeWBHjbLyBhYEoSdXU1mHw9NW2eK/TUjF6QKYLGbWuwgL7+kn7Dh3D\nzX9Y50I28Y2FgShJXf74clvslPZt8Npt4ZeZ8LKBPTo7znNYU1HDYawWLAxESWjYc6vxL4fb6+/d\nNyiu5ylE6pqsLo4th6eKyzkBLkRUCoOIDBWR7SJSLiITHc4/LSJlxuMTEdkXcu5oyLkl0ciHiMKb\ns6bCcVjqwrGXJHRRqDewR2cUXHyWLT54+mp2RhsiLgwi0gbA8wCuAtATQL6ImHqtVPUeVc1S1SwA\nMwAsDDl9sP6cql4baT5EFF5F9X7H7TgLC/rEfIVUN00e9mNkdz3ZFu/vcHstGUWjxdAXQLmqfqaq\ndQDmARjWwPX5AIqi8L5E1ExOnc3jc87GwB6dXcjGXXMK+iJFzLHaOrAzGtEpDF0A7Ao5/sKI2YhI\nOoAMAO+FhNuJSImIrBeR68K9iYiMMa4rqa6ujkLaRMnlnqJNtli3tHYYl9PdhWzc50tNwfqHhtji\naypqkn7ZjFh3PucBWKCqR0Ni6caOQiMBPCMijuPkVHW2qgZUNeD3+2ORK1HCKNpQaZvEdjyAhXf1\ndyehOOFLTXFcrvvueWVJ3RkdjcJQBaBryPGZRsxJHiy3kVS1yvj6GYBVAC6IQk5EFOKBRZttsQ2T\nhiRFZ3Nj8vulY3zO2bb41c8mb2d0NArDRgDniEiGiKQg+OFvG10kIj0ApAFYFxJLE5ETjOedAFwK\nYGsUciIiQ/+pxbZYYUEfFoUQ43K649STzP89Dh0B/mfJRy5l5K6IC4OqHgFwF4B3AHwMYL6qbhGR\nR0QkdJRRHoB5qhq6pu+5AEpE5AMAKwFMU1UWBqIombigDLu+Ne+tUHDxWUnZ2dyYt39tX3Bv8Qdf\nJeW+0WL+nPaGQCCgJSUlbqdBFNdKK/fi+plrTTF/altsnHSlSxnFv6INlbbbbie2PQ5bpzivteQ1\nIrLJ6NNtEGc+EyWoGyxFAQDeGf8TFzLxjvx+6cgLmAdVHjhyLOmWzGBhIEpA+bP+hqOW2NgBGexX\naAKnpcafKi5Pqv0bWBiIEkxp5V6s27nPFDv/jJMSZhntWHBaT+nWufblyRMVCwNRAqmprbP1KwDA\nK7de5EI23jWwR2fkdO9kih1V4PFlyTE2hoWBKIHcNvfvttiMvCzeQmqBObf0g/W/2sw1O1zJJdZY\nGIgSRGnlXmz6/FtTrN1xwaWmqWXWT7IvmZE/628uZBJbLAxECSJvlv0W0lv32MfmU9P5UlMwdkCG\nKbZu576E74hmYSBKAJMXf4Q6y5SkQd39yPR3cCehBDIhtycsk6Id98lOJCwMRAmgcN3npuMTjgOe\nvNE+7JJaZvUE+y2liQvKXMgkNlgYiDzO6Z73snsuZ4dzFPlSU9Dz1FRTbF5JVcKuwMrCQORhNbV1\ntjkLOd078RZSK/jjmEtssauesm98lAhYGIg8rP80+1aUc27p50Imic+XmoIZeebbc3WKhOyIZmEg\n8qiiDZWo/d4cWzjW/lctRc81WV1w4vHmWCJ2RLMwEHlQRfV+2yqgZ6W1Q3Z6mksZJY+l4+xDgEe/\nssGFTFoPCwORB90w097h/P+SfJvOWMn0d8Agy3IZxdu/cSmb1hGVwiAiQ0Vku4iUi8hEh/MFIlIt\nImXGY3TIuVEi8qnxGBWNfIgS2aptu1Fz4IgpNj7nbI5CiqEnb7TvQDzi+fddyKR1RFwYRKQNgOcB\nXAWgJ4B8EXFaxvENVc0yHnOM1/oAPAygH4C+AB4WEbaFiRrgdE97XE53FzJJXr7UFJRalsso2fVd\nwnRER6PF0BdAuap+pqp1AOYBGNbE114JYLmq1qjqXgDLAQyNQk5ECcnpg8c6UoZiw5eagvZtzLFE\n6YiORmHoAmBXyPEXRszqZyLyoYgsEJGuzXwtEcH+wXPGSSlcJM9F/+uwT3QizIiOVefzUgDdVPU8\nBFsFc5v7DURkjIiUiEhJdXV11BMkindz1lTYYmsftC/VQLGT6e9g2wo0EWZER6MwVAHoGnJ8phH7\ngaruUdXDxuEcABc29bUh32O2qgZUNeD3+6OQNpG3TFm2zXTcrxu74+KB01agI2evcyGT6IlGYdgI\n4BwRyRCRFAB5AJaEXiAip4ccXgvgY+P5OwCuEJE0o9P5CiNGRIaa2joMfuI9W3zmTQEXsiEn1n6e\nr/9Vh9LKvS5lE7mIC4OqHgFwF4If6B8DmK+qW0TkERG51rhsnIhsEZEPAIwDUGC8tgbAowgWl40A\nHjFiRGSYtuxjVOw5aIoVFvTh8NQ44tTP83OHLVa9QlS18aviTCAQ0JKSErfTIIqJjIlvwfpbunPa\n1a7kQuEVbai0zUYvLOiDgT06u5SRnYhsUtVGm5qc+UwUxyqq99uKQmFBH1dyoYbl90tHL8vS3F4d\nvsrCQBTHrpxuXta5XRvE1V+gZPbWPQNtMS9OemNhIIpTRRsqccQSe8th3DzFF+vwVS+2GlgYiOKU\n9X41N+DxBqfhq16b9MbCQBSHHl+21RbjBjzecfNF6abjeSWO07PiFgsDURyauWaH6Xh8ztkuZUIt\n8eshP7LFnIp9vGJhIIozTh8gXD3VW3ypKRh2/umm2Mw1O1BTW+dSRs3DwkAUZ6ythYdye7iUCUXi\n4Wt722JOGyzFIxYGojhi7aRsC2D0gEx3kqGI+FJTbHNOKr454FI2zcPCQBRHrJ2U79zL4aleNrBH\nZ5zcTkwxp3Wv4g0LA1GcuPrpVbYYh6d638r7c0zHFXsOxv2y3CwMRHGgtHIvtnxda4otHHuJS9lQ\nNPlSU9D15BNMsZ8+uzrM1fGBhYEoDlhX4jz5BEF2OvdbSBTvP2BuNRw8grhelpuFgSgOfG85Xvmb\nHMfryLusrYb8ON7Mh4WByGX5s8xDGNu3Ee61kIAWjxtgOj58VOO2ryEqhUFEhorIdhEpF5GJDufH\ni8hWEflQRFaISHrIuaMiUmY8llhfS5TIijZUYt3OfabY62Mudikbak2+1BRclGG+PTh0enz2NURc\nGESkDYDnAVwFoCeAfBHpabnsHwACqnoegAUAfh9y7qCqZhmPa0GURKwL5fU6NZV9CwnshV+Y98j5\nHvHZ1xCNFkNfAOWq+pmq1gGYB2BY6AWqulJV62d2rAdwZhTel8jTnNbpd1rPnxKHLzUFqSnmj93r\n43AL0GgUhi4AdoUcf2HEwrkVwNshx+1EpERE1ovIdVHIh8gTrOv0D8j0uZQJxdJrt15kiy0ti6/V\nV2Pa+SwivwAQAPBESDjd2IN0JIBnRMRx/r+IjDEKSEl1dXUMsiVqPc8Vb7fFXr2NfQvJIDs9Df0z\nO5pid8+Lr/0aolEYqgB0DTk+04iZiEgOgAcBXKuqh+vjqlplfP0MwCoAFzi9iarOVtWAqgb8fn8U\n0iZyz1PF5abjnO6dXMqE3PDsyGxbLJ5aDdEoDBsBnCMiGSKSAiAPgGl0kYhcAOBFBIvC7pB4moic\nYDzvBOBSAN5ZtJyoBYo2VNpi3IQnufhSU2x/DMRTqyHiwqCqRwDcBeAdAB8DmK+qW0TkERGpH2X0\nBIAOAP5sGZZ6LoASEfkAwEoA01SVhYESmnUk0tgBGS5lQm5y+mMgXloNoqpu59BsgUBAS0pK3E6D\nqNnmrKnAlGXbTLGd0652KRtyW6z/PYjIJqNPt0Gc+UwUQ9YPgbxAQwP4KNE57bUxefFHLmRixsJA\nFCNO8xamjchyIROKJ13T2puOC9d97voWoCwMRDFinbfA1gIBQOEv+9pi9813tyOahYEoBthaoHAy\n/R0wI8/8b+G97e7O1WJhIIoBa2uBI5Eo1DVZ9tajm30NLAxErcyptTAh17rOJCU7a6uhcN3nLmXC\nwkDU6qythat6nepSJhTPnFoNTkunxAILA1ErctqI5bHrz3MhE/KCqcN7m46tS6fECgsDUSuybsQy\ndXhv7s5GYeX3S7fFRr+yIeZ5sDAQtZLSyr22vZydfvGJQt14ofmWUvH2b2KeAwsDUSuxbsCycOwl\nLmVCXjLRYWBCrNdQYmEgagVOI5G4ZSc1hS81BYUFfUyxu+eVxXQ2NAsDUSvgvAWKxMAenW2x+2M4\nG5qFgSjKOG+BosE6r2HF9uqYtRpYGIiizNpaGJ9ztkuZkJc5zWu4b/4/YvLeUSkMIjJURLaLSLmI\nTHQ4f4KIvGGc3yAi3ULOPWDEt4vIldHIh8gtjy+z7zM1Lqe7C5lQIrCvoRSbEUoRFwYRaQPgeQBX\nAegJIF9ErO3mWwHsVdWzATwN4HHjtT0R3Aq0F4ChAF4wvh+RJ81cs8N0nNvLfq+YqKmcWg1z1lS0\n+vtGo8XQF0C5qn6mqnUA5gEYZrlmGIC5xvMFAAaLiBjxeap6WFV3ACg3vl+rqKmtw4urK1xf65wS\nk9Nezi/c1MfhSqKmK7j4LNOxdbOn1hCNwtAFwK6Q4y+MmOM1xh7R3wLo2MTXRs3I2Wsx9e1tGDl7\nbeMXEzWTdS9n6y80UUtMHvbjmL+nZzqfRWSMiJSISEl1dcvWKt/2de0PX2/+w7popkdJzmkCkhu/\n0JSYHsrtEdP3i0ZhqALQNeT4TCPmeI2ItAVwMoA9TXwtAEBVZ6tqQFUDfr8/4qTXVNRE/D2I6t09\nzzzG3NppSBSJ0QMycbzx/PgGr4yOaBSGjQDOEZEMEUlBsDN5ieWaJQBGGc9HAHhPVdWI5xmjljIA\nnAPg71HIydF5Z5xkOu4/tbi13oqSiNMKqk6dhkSReGPsJcj0p+KNGCytEnFhMPoM7gLwDoCPAcxX\n1S0i8oiIXGtc9hKAjiJSDmA8gInGa7cAmA9gK4C/ALhTVY9GmlM4hbdeZDre9e1hdkRTxHKfNq+g\nGutmPyWH7PQ0rLh3YEyWVmkbjW+iqssALLPE/jvk+SEAN4R57WMAHotGHo3xpaagyyntULXv0A+x\n659/H6t+MzgWb08JqGhDJQ4fM8dGD8h0JxmiKPFM53O0vHprP9PxzppDKK3c61I25HXWkUhsLVAi\nSLrCkOnvgMxO7U2xfI5QohZw+oOCrQVKBElXGADgz2MvMx0fPqJsNVCz/fxF83yY4eef5lImRNGV\nlIXBl5qCi7udYor91x846Y2abvLij/C9pW/h6fwL3UmGKMqSsjAAQNHtl5qODx5xXi6ZyEnhus9N\nx+xboESStIUBAAZk+kzHt87dGOZKon9zWkGVfQuUSJK6MLx628Wm46MKzmugRllXUM0LcDIbJZak\nLgwAkNO9k+l44O85G5rCc1pBddoILn9BiSXpC8OcW8zzGr47rC5lQl5gnbfA3dkoESV9YQDswwxH\nPP++S5lQPHManMDd2SgRsTDAPsywZNd3jgujUXKz7uXMvgVKVCwMBusWjD99dnWYKykZOe23wL4F\nSlQsDAbrFowHj3CEEv0b91ugZMLCEKKbr53pePCT77mUCcUTp5FI3G+BEhkLQ4iFd/Y3He89eJSt\nBrKNRBo7IMOlTIhig4UhhC81BRk+88qrl0wtZnFIYk6thQm5PV3IhCh2IioMIuITkeUi8qnx1ba1\nkIhkicg6EdkiIh+KyM9DzhWKyA4RKTMert+4ffNO88qrh44o5q7dEeZqSnSct0DJKNIWw0QAK1T1\nHAArjGOrAwBuVtVeAIYCeEZEQpc2vV9Vs4xHmcPrY8qXmoKMjuZWw6ssDEnpueLtthjnLVAyiLQw\nDAMw13g+F8B11gtU9RNV/dR4/k8AuwH4I3zfVvXmHeZWw96DR7lfQxJ6qrjcdMwVVClZRFoYTlXV\nL43nXwE4taGLRaQvgBQAFSHhx4xbTE+LyAkNvHaMiJSISEl1dXWEaTfMl5pim7z0s5lr2deQRJxa\nC1xBlZJFo4VBRIpFZLPDY1jodaqqAMIuNCQipwN4DcAtqlq/xckDAHoA6APAB2BCuNer6mxVDahq\nwO9v/QaHdfKSApi1usL5YkooNbV1ttbC1OG9XcqGKPYaLQyqmqOqvR0eiwF8bXzg13/wO+50IyL/\nAeAtAA+q6vqQ7/2lBh0G8AqAvtH4oaLF2mp4ac1nLmVCsVTwkn0P8Px+6S5kQuSOSG8lLQEwyng+\nCsBi6wUikgJgEYBXVXWB5Vx9UREE+yc2W1/vJmur4Sichy9SYvnwn+Z1sjjLmZJNpIVhGoAhIvIp\ngBzjGCISEJE5xjU3AhgAoMBhWOrrIvIRgI8AdAIwJcJ8os668qp1+CIllnuKNtlinOVMyaZtJC9W\n1T0ABjvESwCMNp7/EcAfw7x+UCTvHwtP51+IRR+8ZYrNWVPBjsgEVFNbh0UffGWK9T+7o0vZELmH\nM5+bwLrL25Rl21zKhFrT9S/81RZ7Nj/bhUyI3MXC0ATWXd4A9jUkmpraOuzcc9AUKyzoA19qiksZ\nEbmHhaGJrAunsa8hsfz0GfP+G+3aCgb26BzmaqLExsLQRE4Lp01e/JELmVC01dTW4Z//Mk9efOtX\nA1zKhsh9LAzNYJ3kVLjuc5cyoWjKsey7cTyATH8Hd5IhigMsDM3gNMnJaXgjeceqbbtRc/CoKfbG\n2EtcyoYoPrAwNJO11WAd3kje8svCjabjAZk+ZKfbVo8nSiosDM3k1GoY/AS3APWi54q345gl9upt\nF7uSC1E8YWFogYWWWw0Vew5i1TbHZaIojlkXyrPOVyFKViwMLZCdnoZ2bcyxAsstCYpvS8uqbDGn\n+SpEyYiFoYX+NMbeQclWg3fcPc+8WWDPU1NdyoQo/rAwtFB2ehq6Wz5M2Grwhjtes/9/+qNDoSdK\nViwMEShy+DDh8NX4t2yLuWX3UG4PLn1BFIKFIQJOW4By+Gp8y5/1N9OxgFt2ElmxMETIupkPAExc\nUOZwJbmtprYO63buM8Xe5GQ2IpuICoOI+ERkuYh8anx1nBkkIkdDNulZEhLPEJENIlIuIm8Yu715\nzvics03H80qqUFNbF+ZqcssV01eajlMEnMxG5CDSFsNEACtU9RwAK4xjJwdVNct4XBsSfxzA06p6\nNoC9AG6NMB9XjMvpjjZijl3z7Bp3kiFHpZV78c2BI6bY2+MvdykbovgWaWEYBmCu8Xwugvs2N4mx\nz/MgAPX7QDfr9fHmXcuHTNV3hx3HylPsVVTvx/Uz15piJ7c7jgvlEYURaWE4VVW/NJ5/BeDUMNe1\nE5ESEVkvIvUf/h0B7FPV+j/jvgDg2c11M/0dkHq8OWYdK0/uGO0wjHjhnf1dyITIGxotDCJSLCKb\nHR7DQq9TVQWgYb5NuqoGAIwE8IyINHsYiIiMMYpLSXV1dXNfHhPvTxxii7HV4L4dew6YjsfnnM3W\nAlEDGi0Mqpqjqr0dHosBfC0ipwOA8dVx6q+qVhlfPwOwCsAFAPYAOEVE2hqXnQkg7Keoqs5W1YCq\nBvx+fzN+xNjxpaag1+knmWJsNbjLaTLbuJzuLmRC5B2R3kpaAmCU8XwUgMXWC0QkTUROMJ53AnAp\ngK1GC2MlgBENvd5rXht9kS3m9OFErW9pWZVtMpt12XQisou0MEwDMEREPgWQYxxDRAIiMse45lwA\nJSLyAYKFYJqqbjXOTQAwXkTKEexzeCnCfFznS01B/7M7mmLLtuzmOkoucGqtOS2bTkRmEvzD3VsC\ngYCWlJS4nUZYNbV1yH50uS2+c9rVLmSTnIo2VOKBRZtNsRl5Wbgmy7PjG4giJiKbjP7eBnHmcyvw\npabY9mwAgmPpKTasRSGneycWBaImYmFoJdnpaTjVMn71+plrOSM6Bka/ssEW414LRE3HwtCK3h4/\n0BbLf/Fv9gspapaWVaF4+zem2NBe4abXEJETFoZW5EtNsa2jtH33Ad5SakXWDufjAPzu+vPcSYbI\no1gYWtm4nO6wLKNkW56BosNpWPDyey/nXgtEzcTCEAPF99oXa5uzpsKFTBKbdc7Cxd1O4QxnohZg\nYYiBTH8H5PbqbIpNWbYNFdX7Xcoo8Qx8fIUtVnT7pS5kQuR9LAwx8sJNfWyxETPZER0NS8uqsHPv\nIVNsRp59AyUiahoWhhh6KLeH6XjvgSMo2lDpUjaJoaa2ztbhfHK7NpyzQBQBFoYYGj0gE5kd25ti\nDyzazLkNEfjJE8W22Mr7B7mQCVHiYGGIsRUOH1oXTVnO4tACS8uq8O0h85Iul2V25CgkogixMLjA\nusJnnQJjX4vftZ/ikdMtJAB4bmS2C9kQJRYWBhfk90uH9Y/aDTs56a05rnxqlS02Iy+LrQWiKGBh\ncMn7E+y7vfWfar9fTnaTF3+E6trvTbG8QBd2OBNFCQuDS3ypKeh5aqoptuvbw3iueLtLGXlDaeVe\nFK773BRr1waYNoLDU4miJaLCICI+EVkuIp8aX9McrvmJiJSFPA6JyHXGuUIR2RFyLql+u/84xr40\n91PF5eyIDqOmts5xOZG3fm2fWU5ELRdpi2EigBWqeg6AFcaxiaquVNUsVc0CMAjAAQDvhlxyf/15\nVU2qDZJ9qSlY4bBcRvajy7njm4OfPrvaFps6vDeXvSCKskgLwzAAc43ncwFc18j1IwC8raoHInzf\nhJHp7+C4D/EvC7lPdKilZVX453fmllSHlOO4VSdRK4i0MJyqql8az78C0NjC93kAiiyxx0TkQxF5\nWkROiDAfT8rvl472bc2xY+BCe/XCDU0d2a9b7JMhSgKNFgYRKRaRzQ6PYaHXaXDz6LAbSIvI6QB+\nDOCdkPADAHoA6APAB2BCA68fIyIlIlJSXV3dWNqe87+/st9SmrJsG/duAHCRw/7ZN190Fm4fmOlC\nNkSJr9HCoKo5qtrb4bEYwNfGB379B39DN8ZvBLBIVX8YZ6iqX2rQYQCvAOjbQB6zVTWgqgG/39/U\nn88zMv0dbGspAcG9G5J5Fdb8WX+DtSv+odweeOS6H3POAlErifRW0hIAo4znowAsbuDafFhuI4UU\nFUGwf2Kzw+uSxugBmRg7IMMWv/oZe6drMpi4oAzrdu6zxUcPYEuBqDVFWhimARgiIp8CyDGOISIB\nEZlTf5GIdAPQFYD1E+51EfkIwEcAOgGYEmE+njchtydOsvwhfOgo0G/Ku0k1jHXOmgrMK6myxQsL\n7MuXE1F0SbBrwFsCgYCWlCQXLrrhAAAIvklEQVTu2kI1tXXIdrivfmHXk/HmnZe5kFFslVbudZyv\nUFjQBwN7dHZ4BRE1hYhsUtVAY9dx5nMc8qWmYOFY++S3Tbu+TYr+hhEORWHsgAwWBaIYYWGIU9np\naY7zGwZPX52wI5Vqautw/uRlOGaJn39GB0zI7elKTkTJiIUhjuX3S3fsjL5+5tqELA7D/u/7tv0V\nTm53HBaP45IXRLHEwhDnJuT2RKcT29riiTaM9eY/rMMuy77NALDy/sEuZEOU3FgYPODde3+CNg7x\nwdNXJ8RIpSunr8SaihpbvLCgD+cqELmAhcEDfKkp+LNDZzQQXHBvaZl9WKdX3FO0Cdur7UtnPZTb\ng53NRC5hYfCI7PQ0x5FKAHD3vDLP3VaqqN6PnpPewqIPvrKdGzsgg5PYiFzEwuAh2elp2Dntajjd\nXBk8fbVnikNNbR0GT1+NA9/bz43POZsjkIhcxsLgQesn2bcFBYLFYeKC+N7SoqJ6v+PkPSB4+2hc\nTvcYZ0REViwMHhRugx8AmFdShdtfK4nLTunSyr0YPN153aec7p14+4goTrAweFSmv0PY4vCXLV+j\n/7QVcXVrqaJ6v+MyF0CwpTDnln4xzoiIwmFh8LBMfwfMyHPeJrv2+2PIiZPhrHe8tjFsS2Hq8N5s\nKRDFGfvMKfKUa7K6oEvaifj5rLX43rIeoiI4nHX4+afh6fwLY5pXTW0dnln+Cf60vhJHwlxTOmkI\n5ykQxSGurppAJi/+CIXrPg97fkZeFq7J6tKqOdTU1mHu2p14ff0OfFPrXBIEQPG9lyPT36FVcyEi\ns6aursrCkGCeK96Op4rLw54XAK+00vLVq7btxi2FG8Pv7wqga1o7vD+By1wQuYGFIYnV1Nbhkt8t\nx6Gj4a/J6Ngeb95xWcS3cmpq6zBrVTnmbazEd4es66Ka9TqtA1677WLePiJySUwKg4jcAGAygHMB\n9FVVx09rERkK4FkAbQDMUdX6nd4yAMwD0BHAJgA3qWqjvaUsDE1zx2sbsWxLQ9twB3XreCJeKujT\npFs7NbV1+HPJLpxxcjvcO78MdQ3Xgh9wkx0i98WqMJwL4BiAFwHc51QYRKQNgE8ADAHwBYCNAPJV\ndauIzAewUFXnicgsAB+o6szG3peFoelWbduNgsKNzX7diSmA6nEQKDp2OAFf7juEIy34pzKoux9P\n3pjFVgJRHIjJDm6q+rGqbm/ksr4AylX1M6M1MA/AMBERAIMALDCumwvgukjyIbuBPTqjdNIQDDv/\n9Ga97kAdcPD7YzjwvWLX3uYXhfPOOAmlk4bg5Vv6sigQeUws5jF0AbAr5PgLI9YRwD5VPWKJOxKR\nMSJSIiIl1dXVrZZsIvKlpuDZ/GzsnHa148Y/0XRRRhpKJw3BknEDWBCIPKrReQwiUgzgNIdTD6rq\n4uin5ExVZwOYDQRvJcXqfRPNhNyemJDbE6WVezHyxbVopL+4yQZ174Qnb7yAxYAoATRaGFQ1J8L3\nqALQNeT4TCO2B8ApItLWaDXUxykGstPTsO13V6Oiej9+u/BD7D1Qhx1f18JhwVNH3Xzt8dItfTkX\ngSgBxWLm80YA5xgjkKoA5AEYqaoqIisBjECw32EUgJi1QCgo098Bb/yff+/zUD/q6IZAV/hSU36Y\nsAYoRl2SwRYBURKIdFTScAAzAPgB7ANQpqpXisgZCA5LzTWuywXwDILDVV9W1ceM+H8iWBR8AP4B\n4Beqerix9+WoJCKi5uMENyIiMonJcFUiIko8LAxERGTCwkBERCYsDEREZMLCQEREJiwMRERk4snh\nqiJSDaCyFb51JwDftML3jRWv5w94/2fwev6A938G5h9euqr6G7vIk4WhtYhISVPG+MYrr+cPeP9n\n8Hr+gPd/BuYfOd5KIiIiExYGIiIyYWEwm+12AhHyev6A938Gr+cPeP9nYP4RYh8DERGZsMVAREQm\nLAwWIvKoiHwoImUi8q6xhLhniMgTIrLN+BkWicgpbufUXCJyg4hsEZFjIuKZ0SUiMlREtotIuYhM\ndDuf5hKRl0Vkt4hsdjuXlhCRriKyUkS2Gv9+fuV2Ts0hIu1E5O8i8oGR//+4lgtvJZmJyH+o6nfG\n83EAeqrq7S6n1WQicgWA91T1iIg8DgCqOsHltJpFRM4FcAzAiwDuU9W4X2NdRNoA+ATAEAT3L98I\nIF9Vt7qaWDOIyAAA+wG8qqq93c6nuUTkdACnq2qpiJwEYBOA67zy/0BEBECqqu4XkeMB/BXAr1R1\nfaxzYYvBor4oGFIBeKpyquq7xlapALAewS1TPUVVP1bV7W7n0Ux9AZSr6meqWofgBlTDXM6pWVR1\nDYAat/NoKVX9UlVLjef/AvAxgC7uZtV0GrTfODzeeLjy+cPC4EBEHhORXQD+C8B/u51PBH4J4G23\nk0gSXQDsCjn+Ah76UEo0ItINwAUANribSfOISBsRKQOwG8ByVXUl/6QsDCJSLCKbHR7DAEBVH1TV\nrgBeB3CXu9naNZa/cc2DAI4g+DPEnab8DEQtISIdALwJ4NeWOwBxT1WPqmoWgi39viLiyi29tm68\nqdtUNaeJl74OYBmAh1sxnWZrLH8RKQDwUwCDNU47kZrx/8ArqgB0DTk+04hRDBn35t8E8LqqLnQ7\nn5ZS1X0ishLAUAAxHwyQlC2GhojIOSGHwwBscyuXlhCRoQB+A+BaVT3gdj5JZCOAc0QkQ0RSAOQB\nWOJyTknF6Lx9CcDHqvqU2/k0l4j460cRikh7BAcyuPL5w1FJFiLyJoDuCI6KqQRwu6p65i8/ESkH\ncAKAPUZovZdGVQGAiAwHMAOAH8A+AGWqeqW7WTVORHIBPAOgDYCXVfUxl1NqFhEpAjAQwdU9vwbw\nsKq+5GpSzSAilwF4H8BHCP7+AsBvVXWZe1k1nYicB2Augv9+jgMwX1UfcSUXFgYiIgrFW0lERGTC\nwkBERCYsDEREZMLCQEREJiwMRERkwsJAREQmLAxERGTCwkBERCb/Hz1DrNDqOhoJAAAAAElFTkSu\nQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4NfjBP6X4q-I",
        "colab_type": "text"
      },
      "source": [
        "## Training Procedure"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBWqCuKTUFSx",
        "colab_type": "text"
      },
      "source": [
        "### Weight Update"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "G2Z7LnOkDARK",
        "colab": {}
      },
      "source": [
        "def update_weights(model, gradients, lr = 0.001):\n",
        "  for i, grad in enumerate(gradients):\n",
        "    model[i]['W'] -= lr*grad['W']\n",
        "    model[i]['b'] -= lr*grad['b']\n",
        "      \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGT8WBA4FWMU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_rmsprop_params(model):\n",
        "  rmsprop_params = []\n",
        "  for layer in model:\n",
        "    rmsprop_param = {}\n",
        "    rmsprop_param['W'] = np.zeros_like(layer['W'])\n",
        "    rmsprop_param['b'] = np.zeros_like(layer['b'])\n",
        "    rmsprop_params.append(rmsprop_param)\n",
        "  \n",
        "  return rmsprop_params"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SzuLxvJIDA8B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def update_weights_rmsprop(model, gradients, rmsprop_params, lr = 0.001, beta=0.9):\n",
        "  for i, grad in enumerate(gradients):\n",
        "    v = beta*rmsprop_params[i]['W'] + (1.-beta)*np.square(grad['W'])\n",
        "    model[i]['W'] -= (lr*grad['W'] / np.sqrt(v + 1e-14))\n",
        "    rmsprop_params[i]['W'] = v\n",
        "    \n",
        "    v = beta*rmsprop_params[i]['b'] + (1.-beta)*np.square(grad['b'])\n",
        "    model[i]['b'] -= (lr*grad['b'] / np.sqrt(v + 1e-14))\n",
        "    rmsprop_params[i]['b'] = v\n",
        "      \n",
        "  return model, rmsprop_params"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d4QVJ_HbS7Ff",
        "colab_type": "text"
      },
      "source": [
        "### Gradient check"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fIeDu9MNM10z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def rel_error(x, y):\n",
        "    \"\"\" returns relative error \"\"\"\n",
        "    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "at3QukeGZ_oc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def grad_check(model, gradients, x_batch, y_batch):\n",
        "  layer = 0\n",
        "  layer_x = 0\n",
        "  layer_y = 0\n",
        "  grad_num = gradients[layer]['W'][layer_x][layer_y]\n",
        "\n",
        "  h = 0.00001\n",
        "  oldval = model[layer]['W'][layer_x][layer_y]\n",
        "  model[layer]['W'][layer_x][layer_y] = oldval - h\n",
        "  _ , y_pred1 = forward(x_batch, model)\n",
        "  loss1 = ms_loss(y_pred1, y_batch)\n",
        "  model[layer]['W'][layer_x][layer_y] = oldval + h\n",
        "  _ , y_pred2 = forward(x_batch, model)\n",
        "  loss2 = ms_loss(y_pred2, y_batch)\n",
        "  grad_ana = (loss2 - loss1) / (2.*h)\n",
        "  grad_ana = np.sum(grad_ana)\n",
        "  \n",
        "  model[layer]['W'][layer_x][layer_y] = oldval\n",
        "\n",
        "  error = rel_error(grad_num, grad_ana)\n",
        "  return error"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NhaDwRn8TCQK",
        "colab_type": "text"
      },
      "source": [
        "### Training/Test Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aIg3R-NWX68l",
        "colab_type": "code",
        "outputId": "52fbb7b2-0cae-4128-b57a-1fd331825864",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        }
      },
      "source": [
        "D_in, D_out = 1, 1\n",
        "hidden_layers = [100]\n",
        "model = create_model(hidden_layers, D_in, D_out)\n",
        "rmsprop_params = init_rmsprop_params(model)\n",
        "net_size = len(hidden_layers)\n",
        "\n",
        "train_size = x_train.shape[0]\n",
        "test_size = x_test.shape[0]\n",
        "\n",
        "batch_start = 0\n",
        "batch_end = 0\n",
        "\n",
        "epochs = 500\n",
        "batch_size = 1000\n",
        "num_batches = int(round(train_size / batch_size))\n",
        "\n",
        "should_grad_check = True\n",
        "if should_grad_check:\n",
        "  x_batch = np.expand_dims(x_train[0:batch_size], axis=1)\n",
        "  y_batch = np.expand_dims(y_train[0:batch_size], axis=1)\n",
        "  activations, y_pred = forward(x_batch, model)\n",
        "  loss = ms_loss(y_pred, y_batch).mean()\n",
        "  loss_grad = ms_loss_back(y_pred, y_batch)\n",
        "  gradients = backward(loss_grad, activations, model)\n",
        "  err = grad_check(model, gradients, x_batch, y_batch)\n",
        "  print (\"Gradient Check - Rel Error = {} \\n\".format(err))\n",
        "\n",
        "loss_summary = 0\n",
        "for epoch in range(epochs):\n",
        "  loss = 0\n",
        "  for i in range(num_batches):\n",
        "    batch_start = (i%num_batches)*batch_size\n",
        "    batch_end = min(train_size, batch_start+batch_size)\n",
        "    x_batch = np.expand_dims(x_train[batch_start:batch_end], axis=1)\n",
        "    y_batch = np.expand_dims(y_train[batch_start:batch_end], axis=1)\n",
        "\n",
        "    activations, y_pred = forward(x_batch, model)\n",
        "\n",
        "    loss += ms_loss(y_pred, y_batch).mean()\n",
        "\n",
        "    loss_grad = ms_loss_back(y_pred, y_batch)\n",
        "\n",
        "    gradients = backward(loss_grad, activations, model)\n",
        "        \n",
        "    model, rmsprop_params = update_weights_rmsprop(model, gradients, rmsprop_params, lr=0.0001)\n",
        "  \n",
        "  if (epoch+1)%20 == 0:\n",
        "    print (\"Epoch {} Loss = {}\".format(epoch+1, loss/num_batches))\n",
        "    loss_summary = 0\n",
        "  else:\n",
        "    loss_summary += loss"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Gradient Check - Rel Error = 4.406373888546297e-12 \n",
            "\n",
            "Epoch 20 Loss = 0.16657028565528886\n",
            "Epoch 40 Loss = 0.12172470833960478\n",
            "Epoch 60 Loss = 0.0813841244265466\n",
            "Epoch 80 Loss = 0.0558919694905186\n",
            "Epoch 100 Loss = 0.04158196896866\n",
            "Epoch 120 Loss = 0.028978750969706302\n",
            "Epoch 140 Loss = 0.01757701242794659\n",
            "Epoch 160 Loss = 0.00882320230309065\n",
            "Epoch 180 Loss = 0.0036351146952212235\n",
            "Epoch 200 Loss = 0.0012984278416177768\n",
            "Epoch 220 Loss = 0.000510946875716817\n",
            "Epoch 240 Loss = 0.00024393252506375849\n",
            "Epoch 260 Loss = 0.00015341435018136849\n",
            "Epoch 280 Loss = 0.00010897929033152397\n",
            "Epoch 300 Loss = 8.82935623451998e-05\n",
            "Epoch 320 Loss = 7.496269085299053e-05\n",
            "Epoch 340 Loss = 6.717626744786651e-05\n",
            "Epoch 360 Loss = 6.206806740602548e-05\n",
            "Epoch 380 Loss = 5.861803391409044e-05\n",
            "Epoch 400 Loss = 5.615937002595734e-05\n",
            "Epoch 420 Loss = 5.429361564631993e-05\n",
            "Epoch 440 Loss = 5.2886008616538616e-05\n",
            "Epoch 460 Loss = 5.1731525178403746e-05\n",
            "Epoch 480 Loss = 5.0958913873490035e-05\n",
            "Epoch 500 Loss = 5.013253656245042e-05\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avMnM5g4YAAi",
        "colab_type": "code",
        "outputId": "d207456c-d6a8-45c4-809a-4488fc0d97e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "x_fun = np.expand_dims(np.linspace(-1.0*np.pi, 1.0*np.pi, num=2000), axis=1)\n",
        "activations, y_pred = forward(x_fun, model)\n",
        "plot_data(x_fun.squeeze(), None, y_pred.squeeze())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt0lfWd7/H3N8GEaSKa1HBVo4fa\n4qUSYsTWmWOZKi26jmK1VLBW8Ea1Ryx26hRrV51x6pQZZwqV1lu1gkcFRHHENVpLrdTOsigxBOWm\ngjOpIJBUopAoiUm+54/nyWbvmJDL3tnXz2utvdjPs59n728Q9zff39XcHRERkU55qQ5ARETSixKD\niIjEUGIQEZEYSgwiIhJDiUFERGIoMYiISAwlBhERiaHEICIiMZQYREQkxpBUBzAQRx11lB933HGp\nDkNEJKO8+uqrf3H3st6uy8jEcNxxx1FdXZ3qMEREMoqZ1fXlOjUliYhIDCUGERGJocQgIiIxlBhE\nRCRGQhKDmf3azOrNbGMPr5uZ3Wlm28zsNTOrjHptppm9FT5mJiIeEREZuERVDIuBKYd4/VzghPAx\nG7gbwMxKgVuBM4CJwK1mVpKgmEREZAASkhjc/UVg7yEumQo85IG1wJFmNgr4KrDa3fe6eyOwmkMn\nGBERGWTJmscwBngn6nhHeK6n8yKS5dZsrefah9dxoC04LirI55eXVjJp3PDUBiaZM8HNzGYTNENx\n7LHHpjgaEemLrl/+h9Lc2s6sxesix586LI+7vnmaEkUKJCsx7ASOiTo+Ojy3E5jU5fya7t7A3e8D\n7gOoqqrywQhSROK3vaGJaxa/wtvvfRTX+3z4cUckUaiaSK5kJYZVwPVmtoygo/kDd99lZs8B/xzV\n4fwV4OYkxSQiCTSQhPCZsiLuvbyKDz78mGuWrOO9Dz/u9rrOasKAW84bx9VnjU1Q1NIdc4//l28z\nW0rwm/9RwB6CkUaHAbj7PWZmwC8IOpY/BK5w9+rw3iuBH4Zvdbu7P9jb51VVVbnWShJJD3ubW/n+\nY+v5/Rt/6fb1zi//sWXFfX7P3pLMYXnGz74xnvMr1CXZH2b2qrtX9XpdIhJDsikxiKSHpS/XcfOT\nn5y+lMj+gZq6xh6riYEknVymxCAig6amrpGrFr9C40exvcqD3WF8/4vb+ckzW2POGXDn9ApVD32g\nxCAig6K7KiHP4OeXJOfLuaemq8+NKOauy05T9XAISgwiklDbG5q49qF1vNXwYcz5s8eVcce0CkqL\nCpIaz5qt9Vy5ZB0dUV9hRQV5rJrzv5UcetDXxKBF9ESkV9sbmrhg0R9jkkJRQR4rrzuTB2ZNTHpS\nAJg0bjirv/clTij7VORcc2sHUxa+SE1dY9LjySZKDCJySGu21jP5Z3+gubUjcu7ck0fwxx+cTWV5\napc2G1tWzOq/+1sWTa+InPu43bn4npdYs7U+hZFlNiUGEenRmq31zFoc21yzaHoFd3+rKiVVQk/O\nrxgTkxzcYdbidTxduzOFUWUuJQYR6VZnUuhkwOJZp6ft6J/zK8aw8rozybeD5+Ysq2Xpy33a5lii\nKDGIyCc8XbszJinkGzxx3ZlpvyRFZXkJK649k6LD8iPnbn5yo5JDPykxiEiMNVvrmbOsNnJcmG+s\nuPbMlPcn9FVleQl/nPdlPjeiKHJOyaF/lBhEJKJr81FRQR7PzD0rY5JCp9KiApbOPpMpJx+scJQc\n+k6JQUSAYEjqVQ8dnB90eGF+Rs8JKC0q4J5vnc65J4+InLv5yY3qkO4DJQYRYXtDExf+4r9oD4cf\nFeQb/3H932RsUoh2+0WnxjQrzVlWq6GsvVBiEBHmPLqe/S3tAJjBstlfzIqkAAeblU4adXjk3BVL\n1mkS3CEoMYjkuKUv17F5177I8YMzT8+4PoXelBYV8PDVX+DwwmC0kjvM+NVatjc0pTiy9KTEIJLD\n1mytj1kQb9H0irQfkjpQpUUFLLnyDPLzgokOLW0dzLj3JfY2t6Y4svSjxCCSo7qOQJpy8vC0nbyW\nKJXlJaz49hfpnANX3/Qx/7jq9ZTGlI4SkhjMbIqZvWFm28xsXjevLzCz2vDxppm9H/Vae9RrqxIR\nj4gcWtcRSCOHFfLPF41PYUTJU1lewp1Ry2c8tWG3Rip1Efeez2aWD/wSmAzsANaZ2Sp339x5jbvf\nGHX9HGBC1Ft85O4ViEhS7G1u5Zu/WhszAumRa76QVmsfDbbOyqhzIt+cZbWMKflU1vWtDFQiKoaJ\nwDZ3f9vdW4FlwNRDXD8DWJqAzxWRAZj/7BZ272sBsm8EUn+cXzGGk0YNixxf/uuX1d8QSkRiGAO8\nE3W8Izz3CWZWDhwP/D7q9FAzqzaztWZ2YQLiEZEe1NQ1sqJ6R+Q4G0cg9ceiSydEOqObWtrV3xBK\ndufzdOBxd2+POlce7ih0KbDQzMZ2d6OZzQ4TSHVDQ0MyYhXJKtsbmpjxq7V0rqD9+dGHZ+0IpL4a\nW1bMA5cf3NDsqQ27NfmNxCSGncAxUcdHh+e6M50uzUjuvjP8821gDbH9D9HX3efuVe5eVVZWFm/M\nIjlnzqPraWkLNtspHJLHwhmVKY4oPUwaNzxmL4erHlqX8/MbEpEY1gEnmNnxZlZA8OX/idFFZjYO\nKAH+FHWuxMwKw+dHAX8NbO56r4jEZ83W+phJbEuv+UJO9iv05PyKMYw8YigA7R0wd2lNiiNKrbgT\ng7u3AdcDzwFbgMfcfZOZ3WZmF0RdOh1Y5u5Re0FxIlBtZhuAF4D50aOZRCR+NXWNXLnk4HyFRdMr\ncrpfoSd3XVpJ2N3A6+/uz+khrBb7PZ0ZqqqqvLq6uvcLRXLc3uZWvvSvv4+sg/T50Yfz9A1npTiq\n9NV10t/K6zJnH4q+MLNXwz7dQ9LMZ5EsNv/ZLZGkkJ+H+hV6MWnc8JghrLMefCUnh7AqMYhkqa5D\nUx+4/HT1K/TBoksnUDgk+Grcd6AtJ4ewKjGIZCENTR24sWXFLL3mC5HjVRt259wS3UoMIlnoxuW1\nGpoah8ryEqaOHw2AA9c9/GpqA0oyJQaRLFNT18hrOz4AgiUvNDR1YG694ORIk9Ke/S05NUpJiUEk\ni+xtbmXWg69EjqdVjsmqUTXJVFpUwL2XnRY5vmFZbc40KSkxiGSR+c9uYd+BNgCGDR3CvPNOSnFE\nmW3SuOE52aSkxCCSJbY3NPHEq8EoJDNYfMXEnFpKe7B0bVLKhbWUlBhEssSNy2tpD4chnTLqcDUh\nJUjXJqVcWEtJiUEkC6zZWh/pcNZEtsSbNG44px59BJAbaykpMYhkuK7bdGoi2+BYcEkF+eFaSht3\n7c/qjmglBpEMd+Py2sg2nSMOL9REtkEytqyYi087GgD37F4uQ4lBJIN1nbNwd1RbuCTevHNPZNjQ\nIUCwXMb8Z7JzMWglBpEMpTkLyVdaVMDiKyZGjp9YvzMrO6KVGEQy1MLVb2rOQgpUlpdkfUe0EoNI\nBtrb3MrK9cGchXzNWUi6bO+ITkhiMLMpZvaGmW0zs3ndvD7LzBrMrDZ8XB312kwzeyt8zExEPCLZ\nbv6zW2gK91kYMWyompCSLNs7ouNODGaWD/wSOBc4CZhhZt3VtMvdvSJ83B/eWwrcCpwBTARuNTP9\nCxc5hOh9FvIMfnGp5iykQteO6IWr30hxRImTiIphIrDN3d9291ZgGTC1j/d+FVjt7nvdvRFYDUxJ\nQEwiWWvO0prIPgtfV4dzynR2RIctSqxcvzNrqoZEJIYxwDtRxzvCc11dbGavmdnjZnZMP+/FzGab\nWbWZVTc0NCQgbJHMU1PXyLvvHwCguDBfHc4pVllewugjhwLQ1NKeNcNXk9X5/DRwnLufSlAVLOnv\nG7j7fe5e5e5VZWVlCQ9QJN11Dk/trBYumjBGHc5pYNGMyqiq4d2sGL6aiMSwEzgm6vjo8FyEu7/n\n7i3h4f3AaX29V0QCXZfUnjv5cymOSCCoGqZVBR3RbR2eFcNXE5EY1gEnmNnxZlYATAdWRV9gZqOi\nDi8AtoTPnwO+YmYlYafzV8JzIhJFS2qnt3nnnkhhOH518+79GV81xJ0Y3L0NuJ7gC30L8Ji7bzKz\n28zsgvCyG8xsk5ltAG4AZoX37gX+iSC5rANuC8+JSJSbVmzQktpprLSogKkTgu7RbJj0Zu7e+1Vp\npqqqyqurq3u/UCQLbG9o4twFf6C1AwryjWfnnqXVU9PQ3uZWzvzp8xxo6yDP4PFrz0y7BG5mr7p7\nVW/XaeazSJq7acUGWjuC5yePHqakkKZKiwq4J1zEsMMzextQJQaRNFZT10jtn98HoHBIHv/2jYoU\nRySHMmnc8Mikt/qmloxdKkOJQSSNzVlaQ1gscEnV0aoWMsCd0ycAwVIZ1z+amX0NSgwiaarrZDYN\nT80Mk8YNZ+SwQgD27DuQkVWDEoNImope+kKT2TLLV04aCUB7hlYNSgwiaSi6WtBktswzd/JnI30N\nu/YdYM3W+hRH1D9KDCJp6DuPBtVCfp5pMlsGit7pzR2uX5ZZI5SUGETSTE1dI7s/CKqFvyqwtBsL\nL31TWV7CyCOCBfY+avWM6mtQYhBJI9H7OJvBL6af1ssdks7uujRYYK+9wzNqMx8lBpE0Er2P86hh\nQ5k0bniKI5J4RC/LnUmb+SgxiKSR327eDQT7OGtntuwQuyx3Zmzmo8QgkibWbK1n975gdXrt45w9\num7mkwlVgxKDSJq4Ydl6IOxbULWQVRbNqIx82S6v3pH2y3IrMYikgTVb6yN9C8OLC1UtZJnK8hIq\njj0SgJa2Dr7/WG2KIzo0JQaRFNvb3Mq14UqceQZ3X6aRSNnojmnjKQi/cd+qb0rrvoaEJAYzm2Jm\nb5jZNjOb183r3zOzzWb2mpk9b2blUa+1m1lt+FjV9V6RbLdw9ZscaAuWyjtZm/BkrbFlxZxydFA1\nNLW0M/+ZzSmOqGdxJwYzywd+CZwLnATMMLOTuly2Hqhy91OBx4F/jXrtI3evCB8XIJJD9ja3snJ9\nsGVnYb6xcIb6FrLZHdPGR750n9rwbtpWDYmoGCYC29z9bXdvBZYBU6MvcPcX3P3D8HAtcHQCPlck\n4y1c/SZNLe0AfLq4UMtqZ7mgajgCgJY2T9uqIRGJYQzwTtTxjvBcT64Cno06Hmpm1Wa21swuTEA8\nIhlhe0MTy9f9GQjWRNJIpNyw4JIK8sOJDSvXv5uWI5SS2vlsZpcBVcAdUafLwz1ILwUWmtnYHu6d\nHSaQ6oaGhiREKzK4blqxgZb2YGHtU8cMU99CjhhbVszFpwWNJm0dnpYjlBKRGHYCx0QdHx2ei2Fm\n5wC3ABe4e0vneXffGf75NrAGmNDdh7j7fe5e5e5VZWVlCQhbJHX2Nrfyxp59ABTkm7bszDHzzj2R\nosLg63fTu/vSrmpIRGJYB5xgZsebWQEwHYgZXWRmE4B7CZJCfdT5EjMrDJ8fBfw1kJ6NbiIJtHD1\nmzS3hCORRg9T30KOKS0q4HMjhgHQ2p5+VUPcicHd24DrgeeALcBj7r7JzG4zs85RRncAxcCKLsNS\nTwSqzWwD8AIw392VGCSrRfctFA7JU7WQo+6YNp7CsLPhtZ370mpZ7iGJeBN3fwZ4psu5H0c9P6eH\n+14CPp+IGEQyRXTfwkmjDle1kKPGlhVzyenH8tDaOto7nOsfreGlm89OdViAZj6LJJX6FiTa3Mmf\npbgwH4A9+w6kTdWgxCCSROpbkGilRQVcNCEYodTucP2jNSmOKKDEIJIkXWc5q1oQCKqGYUODVv09\n+1vSompQYhBJEs1ylu6UFhWw+IqJQLAF6HXhgoqppMQgkgR7m1t5sjaoFjTLWbqqLC+JVA31Tamv\nGpQYRJJg4eo32X8gqBY0y1m6c+f0YG6vOymvGpQYRAaZ+hakLyaNG542VYMSg8ggU9+C9FV01ZDK\nEUpKDCKDSH0L0h+Txg1nzJFDAdi17wBrttb3csfgUGIQGUTzn92ivgXpl0XhZk3ucP2y1PQ1KDGI\nDJK9za08tT5YaDg/D/UtSJ9Ulpcw8oigamhu6UhJ1aDEIDJIFq5+8+CaSCO1JpL03V2XHqwavv1w\nddK3AFViEBkE0X0LhUPytJez9EtleQmnpnALUCUGkUEQ3begFVRlIBZcUhH5gn5qw7tJrRqUGEQS\nTH0Lkghjy4o5JapqWLj6jaR9thKDSIKpb0ESZcElFRSE39LLq3ckbQvQhCQGM5tiZm+Y2TYzm9fN\n64Vmtjx8/WUzOy7qtZvD82+Y2VcTEY9IKv12824gmOWsvgWJR1A1HAlAS1tH0rYAjTsxmFk+8Evg\nXOAkYIaZndTlsquARnf/DLAA+Jfw3pMI9og+GZgC3BW+n0hGqqlrZM++FkCznCUx7pg2PlI1bHp3\nX1KqhkRUDBOBbe7+tru3AsuAqV2umQosCZ8/DpxtZhaeX+buLe7+38C28P1EMtKcpTU4muUsiRNd\nNbS2e1KqhkQkhjHAO1HHO8Jz3V7j7m3AB8Cn+3ivSEa4/8Xt7Hz/AAAjDi/ULGdJmDumjacw3wAi\nFelgypjOZzObbWbVZlbd0NAwoPeoqWtk0h0v8OP/2Jj0CSOS3dZsrecnz2yNHKtakEQaW1bMJacf\nC8A5J44Y9M8bkoD32AkcE3V8dHiuu2t2mNkQ4AjgvT7eC4C73wfcB1BVVeUDCfSmxzfwP+99yP+8\nVwc4t134+YG8jcgnfHf5+sjzH503TtWCJNzcyZ9lTMlfMa3qmN4vjlMiKoZ1wAlmdryZFRB0Jq/q\ncs0qYGb4/OvA793dw/PTw1FLxwMnAK8kIKZu3fH18Vj4/LlNewbrYyTHbG9ooq0jmMz2+dGHc/VZ\nY1MckWSj0qICvv2lsZQWFQz6Z8WdGMI+g+uB54AtwGPuvsnMbjOzC8LLHgA+bWbbgO8B88J7NwGP\nAZuB3wD/193b442pJ5XlJYwOl7Stb2pJ2ZK2kl1uWrGB5hZn2NAhGp4qWcGCX9wzS1VVlVdXVw/o\n3pq6Ri66+yUACocYf7r5nKRkYMleZ9z+O/bsb2HE4YW8fMs5qQ5HpEdm9qq7V/V2XcZ0PidKqhen\nkuxSU9dIY3MwSqSzGhXJdDmXGCC1i1NJdvnusvW0dsDQIXlaE0myRk4mhq6LU6lqkIHY3tDE3rBa\n+OyIYs1ylqyRk4kBgqohnC/CyvXvJm1xKskeN63YQHNrBwBf+F+fTnE0IomTs4lhbFkxF592NABt\nHc7cpTUpjkgyyd7mVt7Ysw+A4sJ8rp30mRRHJJI4OZsYAOade2Jkmvnm3ftVNUifzX92C80tQbVw\nwvBijWyTrJLTiaG0qICpE4Klmdo7UNUgfaKNeCTb5XRigKBqGDok+GvYtGs/NXWNKY5I0p024pFs\nl/OJobSogHsuOw2ADofrH1XVID3b3tDE8nV/BqBwSJ5mOktWyvnEADBp3HBGDisEYM++A6oapEc3\nrdhwsFoYpWpBspMSQ+grJ40EoF1Vg/QgeiRSQb6pb0GylhJDaO7kz1JcGOwqqqpBuhM9Eunk0cNU\nLUjWUmIIlRYVcNGEYF6DqgbpantDE0+8ugOAIXmqFiS7KTFEmTv5swwbGuxdtGvfAS3LLRE3Lq8l\n7FrgogmjVS1IVlNiiFJaVMDiKyYC4A7XL3s1xRFJOtje0MTGHR8AwVLt8847KcURiQwuJYYuKstL\nGHlEsHxyc0uHqgbhphUb6AifTx0/WrOcJevFlRjMrNTMVpvZW+Gfn9jo1swqzOxPZrbJzF4zs0ui\nXltsZv9tZrXhIy0abu8KN3J3h28/XK1luXNYTV0jtX9+H4BhQ4eoWpCcEG/FMA943t1PAJ4Pj7v6\nELjc3U8GpgALzezIqNdvcveK8FEbZzwJoc18pNOcpTWRauHCClULkhviTQxTgSXh8yXAhV0vcPc3\n3f2t8Pm7QD1QFufnDjpt5iM1dY28+/4BIFhBde7kz6U4IpHkiDcxjHD3XeHz3cCIQ11sZhOBAmB7\n1OnbwyamBWZWGGc8CdN1M5+Fq99IcUSSbHOW1tC5I/pFE8aoWpCc0WtiMLPfmdnGbh5To69zd4fI\n/0fdvc8o4P8BV7h7Z3V+MzAOOB0oBX5wiPtnm1m1mVU3NDT0/pMlwIJLKigI/4aWV+/Qstw5RNWC\n5LJeE4O7n+Pup3TzeArYE37hd37xdzuEx8yGAf8J3OLua6Pee5cHWoAHgYmHiOM+d69y96qysuS0\nRAVVQ9Ad0tLWwfcfS4suEEkCVQuSy+JtSloFzAyfzwSe6nqBmRUATwIPufvjXV7rTCpG0D+xMc54\nEu6OaeMjVcOmd/epasgBa7bWszOsFoYNHaJqQXJOvIlhPjDZzN4CzgmPMbMqM7s/vOYbwFnArG6G\npT5iZq8DrwNHAT+JM56Ei64aWttdVUMOuGHZegDMYPEVE1UtSM4ZEs/N7v4ecHY356uBq8PnDwMP\n93D/l+P5/GS5Y9p4zl3wB1o7DlYNWhIhO63ZWs++A20ADC8upLL8E1NzRLKeZj73gaqG3LC3uZVr\nHw6WQckzuDvcwEkk1ygx9NEd08ZTmG8AvLZzn5blzkLzn93CgbZwWe1Rh6takJylxNBHY8uKueT0\nYwFo73Aty51l9ja38tT6nQDk56EtOyWnKTH0Q/RmPu81tWiEUhaZ/+yWg1t2jtSWnZLblBj6IXoz\nn5Z2Z+5SVQ3ZoOsmPKoWJNcpMfTT3MmfZeiQ4K9t06796mvIAtqERySWEkM/lRYVcE84WqXD4bqH\ntZlPJtMmPCKfpMQwAJPGDY9sAVrf1KKqIYPduLxWm/CIdKHEMEB3Tp8ABJv5zHrwFS3LnYFULYh0\nT4lhgCaNG86YI4MtQPcdaNOy3BlI1YJI95QY4rBoRiUWPl+5fqeqhgxSU9fIa6oWRLqlxBCHyvIS\nRodVQ1NLu6qGDPKdcIKiGdx7WZWqBZEoSgxxUtWQeWrqGtn9QbCsdlFhHpPGDU9xRCLpRYkhTqoa\nMsve5lZmPfgKEFQLv5iuhfJEulJiSIBFMyojf5H/UfuuqoY0tnD1m5FltUcNG6pqQaQbSgwJUFle\nQsWxwbLc+w60Mf+ZzSmOSLqzt7mVleuDpS/yDX5xqZa+EOlOXInBzErNbLWZvRX+2e06xWbWHrV7\n26qo88eb2ctmts3MlofbgGakO6aNj/xlPrVBVUM6Wrj6TZpa2gEYMWyoltUW6UG8FcM84Hl3PwF4\nPjzuzkfuXhE+Log6/y/AAnf/DNAIXBVnPCkTbOZzBAAtba6qIc1sb2hi+bo/A5CfZ6oWRA4h3sQw\nFVgSPl8CXNjXG83MgC8Djw/k/nS04JIKwr18WLn+XS3LnUZuWrEhsqz2qWOGqVoQOYR4E8MId98V\nPt8NjOjhuqFmVm1ma82s88v/08D77t4WHu8AxvT0QWY2O3yP6oaGhjjDHhxjy4q5+LRgWe62Di3L\nnS6CpS/eB6Ag3/i3b1SkOCKR9NZrYjCz35nZxm4eU6Ovc3cHvIe3KXf3KuBSYKGZje1voO5+n7tX\nuXtVWVlZf29PmnnnnhjZAnTz7v2qGtLATSs20BqufXHy6GFaVlukF70mBnc/x91P6ebxFLDHzEYB\nhH/W9/AeO8M/3wbWABOA94AjzWxIeNnRwM64f6IUKy0qYOqEoPBp70BVQ4qpWhDpv3ibklYBM8Pn\nM4Gnul5gZiVmVhg+Pwr4a2BzWGG8AHz9UPdnonnnnhjZzGfju/tZs7XbfClJcOPyWlULIv0Ub2KY\nD0w2s7eAc8JjzKzKzO4PrzkRqDazDQSJYL67dw7Z+QHwPTPbRtDn8ECc8aSF6M18HJj9cLWGr6bA\nmq31kYXy8vNQtSDSR0N6v6Rn7v4ecHY356uBq8PnLwGf7+H+t4GJ8cSQriaNG87II4ay+4MDtIbD\nV/91mr6YkmVvcyvfjtpd74HLT1e1INJHmvk8iO6KGiv/xPqd6ohOovnPbqGlLWhDGnF4oZa+EOkH\nJYZBVFlewqnhpDd1RCdPTV0jK6qDpS/yDO6+TAvlifSHEsMgW3BJBUPyguGr6ohOju88WhMZN/31\nyjGazCbST0oMg2xsWTH3X14FqCM6GZ6u3RnZa6FAO7OJDIgSQxJ0dkQDkY5oSbyaukZuWFYbOb5P\nO7OJDIgSQ5LcdenBnd4er9lJTV1jSuPJRtFNSFPHj1SHs8gAKTEkSWV5CdOqgnWUOhyuixpKKfHr\n2oR06wXdjpAWkT5QYkiieeeeSGE4I3rP/hZ1RCfI9oYmvrtcTUgiiaLEkESlRQXcGzV08qqH1mlu\nQwLcuLyWjrAN6cQRRWpCEomTEkOSTRo3PGZuw/UPV6c4oszWddmLX1xWleKIRDKfEkMKLLikgnBq\nA1v2NKtJaYC2NzRx1UMHE6uWvRBJDCWGFBhbVszPLzm4bpLmNgzMjctraQ/bkLTshUjiKDGkyPkV\nY2LmNvzjqtdTHFFmiW5CMi17IZJQSgwpFD234akNu3m6NuP3KUqKrk1ID848XcteiCSQEkMKVZaX\ncOf0g01KNyyr1SilPlATksjgiisxmFmpma02s7fCPz/xa5uZ/a2Z1UY9DpjZheFri83sv6Ney7kN\nC86vGMNJo4YBwVpK3/zVWvU3HIKakEQGX7wVwzzgeXc/AXg+PI7h7i+4e4W7VwBfBj4Efht1yU2d\nr7t7bdf7c8GiSydEVmDdva9F/Q09UBOSSHLEmximAkvC50uAC3u5/uvAs+7+YZyfm1WiV2AFWLVh\nt9ZS6oaakESSI97EMMLdd4XPdwMjerl+OrC0y7nbzew1M1tgZoVxxpOxJo0bztTxo4GgSWnGr9aq\nvyGKmpBEkqfXxGBmvzOzjd08pkZf5+4OkcUtu3ufUQR7Pz8XdfpmYBxwOlAK/OAQ9882s2ozq25o\naOgt7Ix06wUnM2xosA13S1uHZkWHauoauXLJusjxNG2+IzKoek0M7n6Ou5/SzeMpYE/4hd/5xX+o\nKbzfAJ5094+j3nuXB1qAB4GJh4jjPnevcveqsrKyvv58GaW0qIDFV0yMDGHVrOjAdx6tiayFNHJY\noTbfERlk8TYlrQJmhs9nAk8d4toZdGlGikoqRtA/sTHOeDJe1yGsVyxZl9P9DdHLaefnwSPXfEEr\np4oMsngTw3xgspm9BZwTHmMqvMpmAAAItklEQVRmVWZ2f+dFZnYccAzwhy73P2JmrwOvA0cBP4kz\nnqwQPSvaHabd+6ecTA5rttYzJ2pHNq2FJJIcFnQNZJaqqiqvrs7u9veaukam3funyCic4sJ8Xvz7\nL+fMb8s1dY1cfM9LdP7znDp+JD+foQ5nkXiY2avu3usSxJr5nKYqy0tY8e0vYmGHQ1NLO5fdnxuT\n3zqTYmdSKC7M145sIkmkxJDGKstLeHDm6ZHjzbv288OVG1IY0eDb29zKzF+/HKmU8vOMh648I2cq\nJZF0oMSQ5iaNG86iqM7o32yqz9rF9vY2tzLjvpfY39IOBPMVVnz7ixqaKpJkSgwZ4PyKMfz0a6dE\njucsq8265LC3uZXL7l/LG3uaI+e05IVIaigxZIgZZ5RHZkZDkByyaY7DLStfY/Ou/ZHjn37tFC15\nIZIiSgwZ5NYLTmb44QdXDcmWOQ5LX67j2U17IseLplcw44zyFEYkktuUGDJIaVEBS2d/gcL8YKiS\nO1x8z0sZXTnc/+J2bn7y4LzGn37tFM6vGJPCiEREiSHDjC0rZunsL5KfdzA5zFq8LuOSw/aGJib/\n+wv85JmtkXM//dopqhRE0oASQwbqnOOQbwfPzVq8jqUv16UuqH7Y3tDEBYv+yFsNB1df/9F545QU\nRNKEEkOGqiwvYcW1Z1IQlR1ufnJj2ieHmrpGpix4kebWjsi5n37tFK4+a2wKoxKRaEoMGayyvIRn\n557FUUWHRc7d/ORG7n9xewqj6tnTtTu56O6X+Ljj4DIs6mgWST9KDBlubFkxv/3eJKacfHBo50+e\n2co5/74mrTb6WfpyXcyCeBAkBXU0i6QfJYYsUFpUwD3fOp0fnTcucm5bQzNn//sfUt60tLe5lSsf\nfDlm5NFh+cbK685UUhBJU0oMWeTqs8bGJAcImpZO/8nqlMx3eLp2J1X/tJrfv/GXyLnPjSjmN3PP\n0oxmkTSmxJBlrj5rLItnnc6QqP+yDU2tXHT3S0mtHu5/cTtzltXSEXXuR+eN47kbv6Q9FUTSnPZj\nyFJ7m1v54coN/GZT7PyGw/KMn31j/KA149z/4vaYuQkABtyp/gSRlOvrfgxxJQYzmwb8A3AiMNHd\nu/22NrMpwM+BfOB+d+/c6e14YBnwaeBV4Fvu3uuGA0oMfbdmaz3XPbKOjz6OPV9UkM8vL61MyHpE\n2xuauGbxK7z93kefeO0zZUXce3mVqgSRNJCsxHAi0AHcC3y/u8RgZvnAm8BkYAewDpjh7pvN7DFg\npbsvM7N7gA3ufndvn6vE0H/d/SbfKc/g9gv7N+v4UMlgoO8pIoMrKYkh6sPW0HNi+CLwD+7+1fD4\n5vCl+UADMNLd27pedyhKDAOzvaGJax9aFzPjuDuH5Rs/mxY0Nx0qoXRHCUEkffU1MQxJQixjgHei\njncAZxA0H73v7m1R59UIPYjGlhWz+u/+lpq6Rq5Zso73Pvy42+s+bnfmLKv9xLyDQ/nUYXnc9c3T\ntFS2SBboNTGY2e+Akd28dIu7P5X4kHqMYzYwG+DYY49N1sdmpcryEl798Vcix/2tCjopGYhkp14T\ng7ufE+dn7ASOiTo+Ojz3HnCkmQ0Jq4bO8z3FcR9wHwRNSXHGJFGuPmtsZK2irs1NahoSyT3JaEpa\nB5wQjkDaCUwHLnV3N7MXgK8TjEyaCSStApHudTY3iUjuimuCm5l9zcx2AF8E/tPMngvPjzazZwDC\nauB64DlgC/CYu28K3+IHwPfMbBtBn8MD8cQjIiLx0wQ3EZEc0ddRSVoSQ0REYigxiIhIDCUGERGJ\nocQgIiIxlBhERCRGRo5KMrMGYDA2FzgK+EuvV6WvTI8fMv9nyPT4IfN/BsXfs3J3L+vtooxMDIPF\nzKr7MpQrXWV6/JD5P0Omxw+Z/zMo/vipKUlERGIoMYiISAwlhlj3pTqAOGV6/JD5P0Omxw+Z/zMo\n/jipj0FERGKoYhARkRhKDF2Y2T+Z2WtmVmtmvzWz0amOqT/M7A4z2xr+DE+a2ZGpjqm/zGyamW0y\nsw4zy5jRJWY2xczeMLNtZjYv1fH0l5n92szqzWxjqmMZCDM7xsxeMLPN4b+f76Y6pv4ws6Fm9oqZ\nbQjj/8eUxaKmpFhmNszd94XPbwBOcvdrUxxWn5nZV4Dfh/to/wuAu/8gxWH1i5mdCHQA99LDXuLp\nxszygTeByQTb1K4DZrj75pQG1g9mdhbQBDzk7qekOp7+MrNRwCh3rzGzw4FXgQsz5b+BmRlQ5O5N\nZnYY8F/Ad919bbJjUcXQRWdSCBUBGZU53f23UftoryXYGS+juPsWd38j1XH000Rgm7u/7e6tBJtP\nTU1xTP3i7i8Ce1Mdx0C5+y53rwmf7yfY/yVj9pH3QFN4eFj4SMn3jxJDN8zsdjN7B/gm8ONUxxOH\nK4FnUx1EjhgDvBN1vIMM+lLKNmZ2HDABeDm1kfSPmeWbWS1QD6x295TEn5OJwcx+Z2Ybu3lMBXD3\nW9z9GOARgt3n0kpv8YfX3AK0EfwMaacvP4PIQJhZMfAEMLdLC0Dac/d2d68gqPQnmllKmvSSsedz\n2nH3c/p46SPAM8CtgxhOv/UWv5nNAv4PcLanaSdSP/4bZIqdwDFRx0eH5ySJwrb5J4BH3H1lquMZ\nKHd/38xeAKYASR8MkJMVw6GY2QlRh1OBramKZSDMbArw98AF7v5hquPJIeuAE8zseDMrAKYDq1Ic\nU04JO28fALa4+89SHU9/mVlZ5yhCM/srgoEMKfn+0aikLszsCeBzBKNi6oBr3T1jfvMzs21AIfBe\neGptJo2qAjCzrwGLgDLgfaDW3b+a2qh6Z2bnAQuBfODX7n57ikPqFzNbCkwiWN1zD3Cruz+Q0qD6\nwcz+Bvgj8DrB/78AP3T3Z1IXVd+Z2anAEoJ/P3nAY+5+W0piUWIQEZFoakoSEZEYSgwiIhJDiUFE\nRGIoMYiISAwlBhERiaHEICIiMZQYREQkhhKDiIjE+P9zA/fOBrSiRAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}