00_xor_from_scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "00_xor_from_scratch.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/redpony/seaml_intro_nns/blob/master/00_xor_from_scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "or24uUbiwh2j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6cK_99VZwri3",
        "colab_type": "text"
      },
      "source": [
        "# Point of this notebook\n",
        "\n",
        "* See the limits of linear models for solving XOR\n",
        "* See the difficulty of computing derivatives by hand\n",
        "* See that MLPs with ReLU activations can solve XOR\n",
        "* See the difficulties with experimenting with MLPs written from scratch by changing the ReLU activation function to a tanh activation function\n",
        "* See that MLPs can solve XOR with different nonlinear activation functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VSru1flJw0Xr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "3a39fe0f-71d8-4b00-d2b4-d70e15ad2545"
      },
      "source": [
        "X = np.array([[-1., -1.],\n",
        "              [-1., 1.],\n",
        "              [1., -1.],\n",
        "              [1., 1.]])\n",
        "y = np.array([[-1.],\n",
        "              [1.],\n",
        "              [1.],\n",
        "              [-1.]])\n",
        "\n",
        "plt.scatter(X[:,0], X[:,1], c=np.where(np.reshape(y, (4, )) < 0, 0, 1))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.collections.PathCollection at 0x7f05c22d53c8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGHVJREFUeJzt3X+QXeV93/H3h12tBAaMhNYghITE\noBRw0gj7BnvixD+wAEFaidQKFhNiYeOqdoLTxHUGUdKJh0KL0xljpyFjVAKWnRhIlDLeTGqrAkFd\nTyyiq1pGP6jQIuoiWZZkBDhY6Od++8d9lp5nde/+OufuatHnNXNnz3nOc8758uzlfu75sTqKCMzM\nzPqdNt4FmJnZycXBYGZmGQeDmZllHAxmZpZxMJiZWcbBYGZmGQeDmZllHAxmZpZxMJiZWaZzvAsY\njenTp8ecOXPGuwwzswll48aNP4mI7qH6TchgmDNnDvV6fbzLMDObUCT9cDj9fCrJzMwyDgYzM8s4\nGMzMLONgMDOzjIPBzMwyDgYzM8tUEgySHpK0T9KWFssl6U8k9Up6VtK7CsuWSdqRXsuqqKeVOPRt\n+n7ya/TtfTd9L99MHPlBO3dnZlbKoYOHefCOv+DGC/4lH3nHJ/jyb6/kpy//Y9v3qyoe7Snp/cDr\nwNci4uebLL8e+AxwPfAe4MsR8R5J04A6UAMC2Ai8OyJeGWx/tVotRvp3DH0/ewT+8V7gjULrFDTt\n66jrF0e0LTOzdosIfu9X/x07/tdOjh46CkBnVyfvmDWd/7Lli3RNnjTibUraGBG1ofpVcsQQEd8B\nDgzSZTGN0IiIWA+cI2kGcC2wNiIOpDBYCyysoqa8vuPw+hfJQwHgEPH6F6venZlZaVu++7/Z+ewP\n3wwFgGNHjnFg76t8978+09Z9j9U1hpnAS4X5XamtVXu1+g5AHGq+7Ohzle/OzKys3u+/yPGjx05o\nP/T6IbZv6G3rvifMxWdJyyXVJdX3798/spVPezst/1M7Lihdm5lZ1c6f+w46u078V4umnDGZmZfM\naOu+xyoYdgOzCvMXprZW7SeIiJURUYuIWnf3kP8GVEbqgjN+Ezh9wJIp6MzfHdG2zMzGwpXXXcFZ\n087ktI7//zEtQefkTj78m7/S1n2PVTD0AB9Ldye9F3gtIvYAa4BrJE2VNBW4JrVVTmf9Gzjjt0Cn\nA5NA0+Dsz6MpV7Vjd2ZmpXR0dvDl797NL37wnXRM6qBjUgc/V7uEL333bt729re1dd9V3ZX0CPBB\nYDqwF/gjYBJARHxFkoA/pXFh+SDw8Yiop3U/AfzbtKl7IuLhofY3mruS+kUchfgZ6GykCXMmzcxO\nYW/87BB9x/t429lnlNrOcO9KqiQYxlqZYDAzO1WN6e2qZmb21uFgMDOzjIPBzMwyDgYzM8s4GMzM\nLONgMDOzjIPBzMwyDgYzM8s4GMzMLONgMDOzjIPBzMwyDgYzM8s4GMzMLONgMDOzjIPBzMwylQSD\npIWStkvqlbSiyfL7JG1Kr+clvVpYdrywrKeKeszMbPROfNL0CEnqAO4HrgZ2ARsk9UTEtv4+EfH7\nhf6fAa4obOKNiJhftg4zM6tGFUcMVwK9EbEzIo4AjwKLB+l/E/BIBfs1M7M2qCIYZgIvFeZ3pbYT\nSLoImAusKzRPkVSXtF7SDRXUY2ZmJZQ+lTRCS4HVEXG80HZRROyWdDGwTtLmiHhh4IqSlgPLAWbP\nnj021ZqZnYKqOGLYDcwqzF+Y2ppZyoDTSBGxO/3cCTxNfv2h2G9lRNQiotbd3V22ZjMza6GKYNgA\nzJM0V1IXjQ//E+4uknQpMBX4XqFtqqTJaXo68D5g28B1zcxs7JQ+lRQRxyTdBqwBOoCHImKrpLuA\nekT0h8RS4NGIiMLqlwEPSOqjEVL3Fu9mMjOzsaf8c3piqNVqUa/Xx7sMM7MJRdLGiKgN1c9/+Wxm\nZhkHg5mZZRwMZmaWcTCYmVnGwWBmZhkHg5mZZRwMZmaWcTCYmVnGwWBmZhkHg5mZZRwMZmaWcTCY\nmVnGwWBmZhkHg5mZZRwMZmaWcTCYmVmmkmCQtFDSdkm9klY0WX6LpP2SNqXXJwvLlknakV7LqqjH\nzMxGr/SjPSV1APcDVwO7gA2Sepo8ovOxiLhtwLrTgD8CakAAG9O6r5Sty8zMRqeKI4Yrgd6I2BkR\nR4BHgcXDXPdaYG1EHEhhsBZYWEFNZmY2SlUEw0zgpcL8rtQ20EckPStptaRZI1wXScsl1SXV9+/f\nX0HZZmbWzFhdfP5bYE5E/FMaRwWrRrqBiFgZEbWIqHV3d1deoJmZNVQRDLuBWYX5C1PbmyLi5Yg4\nnGYfBN493HXNzGxsVREMG4B5kuZK6gKWAj3FDpJmFGYXAc+l6TXANZKmSpoKXJPazMxsnJS+Kyki\njkm6jcYHegfwUERslXQXUI+IHuB3JS0CjgEHgFvSugck/Xsa4QJwV0QcKFuTmZmNniJivGsYsVqt\nFvV6fbzLMDObUCRtjIjaUP38l89mZpZxMJiZWcbBYGZmGQeDmZllHAxmZpZxMJiZWcbBYGZmGQeD\nmZllHAxmZpZxMJiZWcbBYGZmGQeDmZllHAxmZpZxMJiZWcbBYGZmmUqCQdJCSdsl9Upa0WT5ZyVt\nk/SspCclXVRYdlzSpvTqGbiumZmNrdJPcJPUAdwPXA3sAjZI6omIbYVu3wdqEXFQ0qeBPwY+mpa9\nERHzy9ZhZmbVqOKI4UqgNyJ2RsQR4FFgcbFDRDwVEQfT7Hrgwgr2a2ZmbVBFMMwEXirM70ptrdwK\nfKswP0VSXdJ6STdUUI+ZmZVQ+lTSSEi6GagBHyg0XxQRuyVdDKyTtDkiXmiy7nJgOcDs2bPHpF4z\ns1NRFUcMu4FZhfkLU1tG0gLgTmBRRBzub4+I3ennTuBp4IpmO4mIlRFRi4had3d3BWWbmVkzVQTD\nBmCepLmSuoClQHZ3kaQrgAdohMK+QvtUSZPT9HTgfUDxorWZmY2x0qeSIuKYpNuANUAH8FBEbJV0\nF1CPiB7gPwFnAn8tCeD/RsQi4DLgAUl9NELq3gF3M5mZ2RhTRIx3DSNWq9WiXq+PdxlmZhOKpI0R\nURuqn//y2czMMg4GMzPLOBjMzCzjYDAzs4yDwczMMg4GMzPLOBjMzCzjYDAzs4yDwczMMg4GMzPL\nOBjMzCzjYDAzs4yDwczMMg4GMzPLOBjMzCxTSTBIWihpu6ReSSuaLJ8s6bG0/BlJcwrL7kjt2yVd\nW0U9ZmY2eqWDQVIHcD9wHXA5cJOkywd0uxV4JSIuAe4DvpDWvZzGo0DfCSwE/ixtz8zMxkkVRwxX\nAr0RsTMijgCPAosH9FkMrErTq4EPq/GMz8XAoxFxOCJeBHrT9szMbJxUEQwzgZcK87tSW9M+EXEM\neA04d5jrmpnZGJowF58lLZdUl1Tfv3//eJdjZvaWVUUw7AZmFeYvTG1N+0jqBN4OvDzMdQGIiJUR\nUYuIWnd3dwVlm5lZM1UEwwZgnqS5krpoXEzuGdCnB1iWppcA6yIiUvvSdNfSXGAe8A8V1GRmZqPU\nWXYDEXFM0m3AGqADeCgitkq6C6hHRA/w58DXJfUCB2iEB6nfXwHbgGPA70TE8bI1mZnZ6KnxxX1i\nqdVqUa/Xx7sMM7MJRdLGiKgN1W/CXHw2M7Ox4WAwM7OMg8HMzDIOBjMzyzgYzMws42AwM7OMg8HM\nzDIOBjMzyzgYzMws42AwM7OMg8HMzDIOBjMzyzgYzMws42AwM7OMg8HMzDIOBjMzy5QKBknTJK2V\ntCP9nNqkz3xJ35O0VdKzkj5aWPZVSS9K2pRe88vUY2Zm5ZU9YlgBPBkR84An0/xAB4GPRcQ7gYXA\nlySdU1j+BxExP702lazHzMxKKhsMi4FVaXoVcMPADhHxfETsSNM/AvYB3SX3a2ZmbVI2GM6LiD1p\n+sfAeYN1lnQl0AW8UGi+J51iuk/S5EHWXS6pLqm+f//+kmWbmVkrQwaDpCckbWnyWlzsFxEBxCDb\nmQF8Hfh4RPSl5juAS4FfAqYBt7daPyJWRkQtImrd3T7gMDNrl86hOkTEglbLJO2VNCMi9qQP/n0t\n+p0N/B1wZ0SsL2y7/2jjsKSHgc+NqHozM6tc2VNJPcCyNL0M+ObADpK6gMeBr0XE6gHLZqSfonF9\nYkvJeszMrKSywXAvcLWkHcCCNI+kmqQHU58bgfcDtzS5LfUvJW0GNgPTgbtL1mNmZiWpcWlgYqnV\nalGv18e7DDOzCUXSxoioDdXPf/lsZmYZB4OZmWUcDGZmlnEwmJlZxsFgZmYZB4OZmWUcDGZmlnEw\nmJlZxsFgZmYZB4OZmWUcDGZmlnEwmJlZxsFgZmYZB4OZmWUcDGZmlikVDJKmSVoraUf6ObVFv+OF\nh/T0FNrnSnpGUq+kx9LT3szMbByVPWJYATwZEfOAJ9N8M29ExPz0WlRo/wJwX0RcArwC3FqyHjMz\nK6lsMCwGVqXpVTSe2zws6TnPVwH9z4Ee0fpmZtYeZYPhvIjYk6Z/DJzXot8USXVJ6yX1f/ifC7wa\nEcfS/C5gZsl6zMyspM6hOkh6Aji/yaI7izMREZJaPUD6oojYLeliYJ2kzcBrIylU0nJgOcDs2bNH\nsqqZmY3AkMEQEQtaLZO0V9KMiNgjaQawr8U2dqefOyU9DVwB/A1wjqTOdNRwIbB7kDpWAisBarVa\nqwAyM7OSyp5K6gGWpellwDcHdpA0VdLkND0deB+wLSICeApYMtj6ZmY2tsoGw73A1ZJ2AAvSPJJq\nkh5MfS4D6pJ+QCMI7o2IbWnZ7cBnJfXSuObw5yXrMTOzktT44j6x1Gq1qNfr412GmdmEImljRNSG\n6ue/fDYzs4yDwczMMg4GMzPLOBjMzCzjYDAzs4yDwczMMg4GMzPLOBjMzCzjYDAzs4yDwczMMg4G\nMzPLOBjMzCzjYDAzs4yDwczMMg4GMzPLOBjMzCxTKhgkTZO0VtKO9HNqkz4fkrSp8Dok6Ya07KuS\nXiwsm1+mHjMzK6/sEcMK4MmImAc8meYzEfFURMyPiPnAVcBB4L8XuvxB//KI2FSyHjMzK6lsMCwG\nVqXpVcANQ/RfAnwrIg6W3K+ZmbVJ2WA4LyL2pOkfA+cN0X8p8MiAtnskPSvpPkmTW60oabmkuqT6\n/v37S5RsZmaDGTIYJD0haUuT1+Jiv4gIIAbZzgzgF4A1heY7gEuBXwKmAbe3Wj8iVkZELSJq3d3d\nQ5VtZmaj1DlUh4hY0GqZpL2SZkTEnvTBv2+QTd0IPB4RRwvb7j/aOCzpYeBzw6zbzMzapOyppB5g\nWZpeBnxzkL43MeA0UgoTJInG9YktJesxM7OSygbDvcDVknYAC9I8kmqSHuzvJGkOMAv4HwPW/0tJ\nm4HNwHTg7pL1mJlZSUOeShpMRLwMfLhJex34ZGH+/wAzm/S7qsz+zcysev7LZzMzyzgYzMws42Aw\nM7OMg8HMzDIOBjMzyzgYzMws42AwM7OMg8HMzDIOBjMzyzgYzMws42AwM7OMg8HMzDIOBjMzyzgY\nzMws42AwM7NMqWCQ9BuStkrqk1QbpN9CSdsl9UpaUWifK+mZ1P6YpK4y9Qxl3Tf+J5+47F+z6Ozf\n4vd+5Q/Z+vfb27k7M7NSou8gfT/9D/TtfQ99e2v0vXYn0Xeg7fste8SwBfgXwHdadZDUAdwPXAdc\nDtwk6fK0+AvAfRFxCfAKcGvJelrq+bNv88XlD/DS9h/xxuuH2Pr327n9mrvYtv75du3SzGzUIoI4\nsAwOfgPiFYifwhuPEy8vIeJIW/ddKhgi4rmIGOpr95VAb0TsjMZ/zaPA4vSc56uA1anfKhrPfa7c\n8WPHefgPH+XwwcNZ++GDR3jozm+0Y5dmZuUc3QDHdwDFEDgGfQfg0LfbuuuxuMYwE3ipML8rtZ0L\nvBoRxwa0V+61n/yUI4eaJ+zOZ3/Yjl2amZVz9Dl48+OxIA4SR7e0dddDPvNZ0hPA+U0W3RkR36y+\npJZ1LAeWA8yePXtE65417UxO62iegTPmvKN0bWZmleuYDZoEJ5w2Oh11zm3rroc8YoiIBRHx801e\nww2F3cCswvyFqe1l4BxJnQPaW9WxMiJqEVHr7u4e5q4bJnVN4obPXMfkMyZn7ZPP6OJjn79xRNsy\nMxsTk38VTpsKdBQaBeqCKf+8rbsei1NJG4B56Q6kLmAp0BMRATwFLEn9lgFtOwL5+N03seSz/4zT\nz5pCx6QOpp1/Dr+/8lO859fe3a5dmpmNmtSJpj0CXb9M4+ROB0yaj859DJ12Znv33fh8HuXK0q8D\n/xnoBl4FNkXEtZIuAB6MiOtTv+uBL9GIvoci4p7UfjGNi9HTgO8DN0fE4RP3lKvValGv10dV8/Hj\nxzn0s8OccdbpNK5/m5md3Bofi31Ip5fajqSNEdHyTwve7FcmGMZLmWAwMztVDTcY/JfPZmaWcTCY\nmVnGwWBmZhkHg5mZZRwMZmaWcTCYmVlmQt6uKmk/UOYfOZoO/KSicqrkukbmZKzrZKwJXNdInIw1\nQTV1XRQRQ/7TERMyGMqSVB/OvbxjzXWNzMlY18lYE7iukTgZa4KxrcunkszMLONgMDOzzKkaDCvH\nu4AWXNfInIx1nYw1gesaiZOxJhjDuk7JawxmZtbaqXrEYGZmLbxlg0HSb0jaKqlPUssr+ZIWStou\nqVfSikL7XEnPpPbH0rMkqqhrmqS1knakn1Ob9PmQpE2F1yFJN6RlX5X0YmHZ/LGqK/U7Xth3T6G9\n8vEa5ljNl/S99Lt+VtJHC8sqHatW75XC8snpv703jcWcwrI7Uvt2SdeWqWOENX1W0rY0Nk9Kuqiw\nrOnvcozqukXS/sL+P1lYtiz9zndIWjbGdd1XqOl5Sa8WlrVlvCQ9JGmfpKbP61TDn6San5X0rsKy\n9oxVRLwlX8BlwD8BngZqLfp0AC8AFwNdwA+Ay9OyvwKWpumvAJ+uqK4/Blak6RXAF4boPw04AJyR\n5r8KLGnDeA2rLuD1Fu2Vj9dwagJ+DpiXpi8A9gDnVD1Wg71XCn1+G/hKml4KPJamL0/9JwNz03Y6\nxqimDxXeO5/ur2mw3+UY1XUL8Kct3u8708+paXrqWNU1oP9naDw/pt3j9X7gXcCWFsuvB74FCHgv\n8Ey7x+ote8QQEc9FxPYhul0J9EbEzog4QuOhQYslCbgKWJ36rQJuqKi0xWl7w93uEuBbEXGwov23\nMtK63tTG8Rqypoh4PiJ2pOkfAftoPDiqak3fK4PUuxr4cBqbxcCjEXE4Il4EetP22l5TRDxVeO+s\np/EI3XYbzli1ci2wNiIORMQrwFpg4TjVdRPwSEX7bikivkPjy18ri4GvRcN6Go9EnkEbx+otGwzD\nNBN4qTC/K7WdC7waEccGtFfhvIjYk6Z/DJw3RP+lnPjmvCcdUt4naXKzldpY1xRJdUnr+09v0b7x\nGtFYSbqSxjfBFwrNVY1Vq/dK0z5pLF6jMTbDWbddNRXdSuObZ79mv8sqDLeuj6TfzWpJ/c+Fb9dY\njWjb6ZTbXGBdobld4zWUVnW3baw6q9jIeJH0BHB+k0V3RkTbnh89lMHqKs5EREhqeVtY+lbwC8Ca\nQvMdND4ku2jcvnY7cNcY1nVRROxW47Gs6yRtpvEBOCoVj9XXgWUR0ZeaRz1WbzWSbgZqwAcKzSf8\nLiPiheZbqNzfAo9ExGFJ/4rGkdZVY7Tv4VgKrI6I44W28RyvMTWhgyEiFpTcxG5gVmH+wtT2Mo3D\ntc70za+/vXRdkvZKmhERe9KH2b5BNnUj8HhEHC1su/8b9GFJDwOfG8u6ImJ3+rlT0tPAFcDfMMrx\nqqImSWcDf0fjC8H6wrZHPVZNtHqvNOuzS1In8HYa76XhrNuumpC0gEbQfiAKz1Rv8bus4oNuyLoi\n4uXC7IM0rif1r/vBAes+XUFNw6qrYCnwO8WGNo7XUFrV3baxOtVPJW0A5qlxR00XjTdDTzSu7DxF\n4/w+wDKgqiOQnrS94Wz3hHOc6QOy/7z+DUDTOxnaUZekqf2nYyRNB94HbGvjeA2npi7gcRrnYFcP\nWFblWDV9rwxS7xJgXRqbHmCpGnctzQXmAf9QopZh1yTpCuABYFFE7Cu0N/1dVlDTcOuaUZhdBDyX\nptcA16T6pgLXkB8xt7WuVNulNC7mfq/Q1s7xGkoP8LF0d9J7gdfSl572jVVVV9ZPthfw6zTOuR0G\n9gJrUvsFwH8r9LseeJ5G8t9ZaL+Yxv+8vcBfA5Mrqutc4ElgB/AEMC2114AHC/3m0PhGcNqA9dcB\nm2l8yP0FcOZY1QX8ctr3D9LPW9s5XsOs6WbgKLCp8JrfjrFq9l6hcWpqUZqekv7be9NYXFxY9860\n3nbgugrf50PV9ER6//ePTc9Qv8sxqus/AlvT/p8CLi2s+4k0hr3Ax8eyrjT/eeDeAeu1bbxofPnb\nk97Hu2hcC/oU8Km0XMD9qebNFO6ybNdY+S+fzcwsc6qfSjIzswEcDGZmlnEwmJlZxsFgZmYZB4OZ\nmWUcDGZmlnEwmJlZxsFgZmaZ/wdyVId2OlT4CwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NBlCegKwxkfm",
        "colab_type": "text"
      },
      "source": [
        "# SGD Configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gS5tQxmaw6Yj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ETA = 0.15  # learning rate for SGD\n",
        "EPOCHS = 35"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zbWyE1A8xsaQ",
        "colab_type": "text"
      },
      "source": [
        "# Linear model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwmA50BSxqMj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "outputId": "f9cd6cdb-4290-43b1-fe38-ecd7c8ceb0a0"
      },
      "source": [
        "W = np.random.normal(size=(2, 1)) / np.sqrt(2)\n",
        "\n",
        "lin_losses = []\n",
        "\n",
        "for _ in range(EPOCHS):\n",
        "    y_pred = X @ W\n",
        "    loss = 0.5 * np.sum((y_pred - y)**2)\n",
        "    lin_losses.append(loss)\n",
        "    \n",
        "    # loss = 0.5 * \\sum_i (yp_i - y_i)^2\n",
        "    #      = \\sum_i yp_i^2 - 2 * yp_i * y_i + y_i^2\n",
        "    # dloss/dyp_i = 0.5 * (2 * yp_i - 2 * y_i) = yp_i - y_i\n",
        "    dloss_dy_pred = y_pred - y\n",
        "    \n",
        "    # dloss/dW = dy_pred/dW * dloss_dy_pred\n",
        "    #          = X^T * dloss_dy_pred\n",
        "    dloss_dW = X.T @ dloss_dy_pred\n",
        "    \n",
        "    W -= ETA * dloss_dW\n",
        "\n",
        "print(\"Predictions after training:\\n{}\".format(y_pred))\n",
        "plt.plot(lin_losses)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions after training:\n",
            "[[ 1.61281734e-14]\n",
            " [ 3.27125935e-14]\n",
            " [-3.27125935e-14]\n",
            " [-1.61281734e-14]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f05c2249240>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGIFJREFUeJzt3X2QneV93vHvteecPSvtLsigRRYg\nRcGAscyLIGuiGuIQbGzFzkAzdVK3NXE9dtV6aAda2rFLJy7gyUyJHZLpOIYqJjVNSR1qiE2xjaPW\nwjZJEFkRISEJsHixY1ug5V0v0Uq7++sfz7PS6ui8PCvt7jnPc67PzM6el3vP/vyMufbWfe5z/xQR\nmJlZsfS0uwAzM5t9DnczswJyuJuZFZDD3cysgBzuZmYF5HA3MyugluEuqU/SY5KekLRN0i0Nxv2m\npO3pmD+d/VLNzCwrtdrnLklAf0TslVQBHgGuj4hHp405B7gXuDIiXpN0WkTsnsvCzcyssXKrAZGk\n/970biX9qv2L8C+AP4yI19KfcbCbmbVRy3AHkFQCNgFnk4T4xpoh56bj/hIoATdHxEPNXnPx4sWx\nYsWKGRdsZtbNNm3a9HJEDLUalyncI2ICWCVpEfDnks6PiCdrXucc4ArgTOD7ki6IiNenv46ktcBa\ngOXLlzMyMpLpf4yZmSUk/SjLuBntlknDegOwpuapnwAPRMShiHgeeIYk7Gt/fl1EDEfE8NBQyz88\nZmZ2nLLslhlKZ+xIWgBcBTxVM+zrJLN2JC0mWaZ5blYrNTOzzLIsyywF7k7X3XuAeyPiQUm3AiMR\n8QDwHeD9krYDE8B/iIhX5qxqMzNrquVWyLkyPDwcXnM3M5sZSZsiYrjVOH9C1cysgBzuZmYF5HA3\nMyug3IX70y/u4QvfeZpX9x1sdylmZh0rd+H+/Mt7+eKGnbz05oF2l2Jm1rFyF+4D1QoAe8fG21yJ\nmVnnyl2491dLAOw94HA3M2skd+E+2Jd87sozdzOzxnIX7l6WMTNrLX/hPjVz97KMmVlDuQv3hZUS\nEuzxzN3MrKHchXtPj+jvLXvmbmbWRO7CHWCgWmafZ+5mZg3lM9z7yn5D1cysiXyGe7XsNXczsyZy\nGe6DfWX2HjjU7jLMzDpWLsO9v9fLMmZmzWTpodon6TFJT0jaJumWJmP/kaSQ1LJLyIkY6PNuGTOz\nZrL0UB0DroyIvZIqwCOSvh0Rj04fJGkQuB7YOAd1HmWg6pm7mVkzLWfukdib3q2kX/Uar34OuA2Y\n87N4B9PdMu3q/2pm1ukyrblLKknaDOwG1kfExprnLwGWRcQ356DGYwxUy0wG/P2hifn4dWZmuZMp\n3CNiIiJWAWcCl0o6f+o5ST3A7cCNrV5H0lpJI5JGRkdHj7dm+qs+X8bMrJkZ7ZaJiNeBDcCaaQ8P\nAucDD0t6AVgNPFDvTdWIWBcRwxExPDQ0dNxFTx37673uZmb1ZdktMyRpUXp7AXAV8NTU8xHxRkQs\njogVEbECeBS4OiJG5qhmBtKZu48gMDOrL8vMfSmwQdIW4G9I1twflHSrpKvntrz6BrwsY2bWVMut\nkBGxBbi4zuOfbTD+ihMvq7kBL8uYmTWVy0+oeuZuZtZcvsPdM3czs7ryGe5ukm1m1lQuw71aLtFb\n6nG4m5k1kMtwBx8eZmbWTG7Dvb9a8szdzKyB3Ib7QLXCHs/czczqym24D7pJtplZQ7kNdzfJNjNr\nLL/h7oYdZmYN5Tbc+6tlr7mbmTWQ23BPujEdancZZmYdKbfhPlAtc+DQJOMTk+0uxcys4+Q63AH2\njbnVnplZrfyG++Fjf700Y2ZWK7/h7pMhzcwayn+4e8eMmdkxsvRQ7ZP0mKQnJG2TdEudMf9O0nZJ\nWyT9P0k/NzflHuFjf83MGssycx8DroyIi4BVwBpJq2vG/C0wHBEXAl8Dfnd2yzzWoJdlzMwaahnu\nkdib3q2kX1EzZkNE7E/vPgqcOatV1nF45u5lGTOzY2Rac5dUkrQZ2A2sj4iNTYZ/Avh2g9dZK2lE\n0sjo6OjMq52m3zN3M7OGMoV7RExExCqSGfmlks6vN07SR4Fh4PMNXmddRAxHxPDQ0NDx1gxAf2+6\nFdIzdzOzY8xot0xEvA5sANbUPifpfcB/Aq6OiLHZKa+xUo/o7y352F8zszqy7JYZkrQovb0AuAp4\nqmbMxcB/Iwn23XNRaD0+9tfMrL5yhjFLgbsllUj+GNwbEQ9KuhUYiYgHSJZhBoD/LQngxxFx9VwV\nPWWgWmaPw93M7Bgtwz0itgAX13n8s9Nuv2+W68pkoOom2WZm9eT2E6rgZRkzs0byHe7uo2pmVlfO\nw73irZBmZnXkOtwHvSxjZlZXrsO9v1pi79g4EdF6sJlZF8l1uA9UK0xMBgcOudWemdl0+Q53H/tr\nZlZXrsPdx/6amdWX63B3NyYzs/pyHe5Tx/66SbaZ2dFyHe6DbthhZlZXrsN9allm30GHu5nZdPkO\nd8/czczqyne4H15zd7ibmU2X63Cvlnso98gzdzOzGrkOd0k+9tfMrI5chzu4YYeZWT1Zeqj2SXpM\n0hOStkm6pc6YqqQ/k7RT0kZJK+ai2HoGqp65m5nVyjJzHwOujIiLgFXAGkmra8Z8AngtIs4Gfh+4\nbXbLbMzH/pqZHatluEdib3q3kn7VnrF7DXB3evtrwHuVdsqea/2euZuZHSPTmrukkqTNwG5gfURs\nrBlyBvB3ABExDrwBnFrnddZKGpE0Mjo6emKVp7zmbmZ2rEzhHhETEbEKOBO4VNL5x/PLImJdRAxH\nxPDQ0NDxvMQxBvvK3uduZlZjRrtlIuJ1YAOwpuapnwLLACSVgZOBV2ajwFbcJNvM7FhZdssMSVqU\n3l4AXAU8VTPsAeBj6e0PA9+Neep9N1CtsP/gBBOTbrVnZjalnGHMUuBuSSWSPwb3RsSDkm4FRiLi\nAeAu4E8k7QReBT4yZxXX6K+WgKRhx8kLKvP1a83MOlrLcI+ILcDFdR7/7LTbB4DfmN3Sshmc1mrP\n4W5mlijAJ1STQPeOGTOzI/If7m6SbWZ2jPyHu5tkm5kdozjh7mUZM7PD8h/uh5dl3CTbzGxK/sN9\nqhuTZ+5mZocVJtz3jU20uRIzs86R+3Av9YiFvSUvy5iZTZP7cAcf+2tmVqsQ4T5YLXvN3cxsmkKE\nu5tkm5kdrRjh7mN/zcyOUphw97KMmdkRhQl3L8uYmR1RjHD3mruZ2VGKEe5pk+x5av5kZtbxihHu\nfWXGJ4Ox8cl2l2Jm1hGy9FBdJmmDpO2Stkm6vs6YkyX9H0lPpGM+Pjfl1jfoY3/NzI6SZeY+DtwY\nESuB1cB1klbWjLkO2B4RFwFXAL8nqXdWK22i38f+mpkdpWW4R8SuiHg8vb0H2AGcUTsMGJQkYICk\nSfa8Ja0bdpiZHW1Ga+6SVpA0y95Y89QXgXcAPwO2AtdHxDEL4JLWShqRNDI6OnpcBdczdaa797qb\nmSUyh7ukAeA+4IaIeLPm6Q8Am4HTgVXAFyWdVPsaEbEuIoYjYnhoaOgEyj7aYNok259SNTNLZAp3\nSRWSYL8nIu6vM+TjwP2R2Ak8D5w3e2U25ybZZmZHy7JbRsBdwI6IuL3BsB8D703HLwHeDjw3W0W2\n0l8tAbDH4W5mBkA5w5jLgGuBrZI2p4/dBCwHiIg7gc8BX5G0FRDw6Yh4eQ7qrWtqWca7ZczMEi3D\nPSIeIQnsZmN+Brx/toqaqb5KD6UeuRuTmVmqEJ9QlZQe++s+qmZmUJBwBx/7a2Y2XaHC3csyZmaJ\n4oS7j/01MzusOOGeHvtrZmZFCnfP3M3MDitMuA+61Z6Z2WGFCfd+L8uYmR1WmHAfqJbZd3CCiUm3\n2jMzK0y4D6aHh+076Nm7mVlhwn2qYYeP/TUzK1K497nVnpnZlMKE+1QfVR/7a2ZWoHAfdJNsM7PD\nChPu7sZkZnZEccLdM3czs8MKE+6HuzF55m5mlqmH6jJJGyRtl7RN0vUNxl0haXM65nuzX2pzU31U\nHe5mZtl6qI4DN0bE45IGgU2S1kfE9qkBkhYBXwLWRMSPJZ02R/U2VC710FfpcbibmZFh5h4RuyLi\n8fT2HmAHcEbNsH8K3B8RP07H7Z7tQrMYqFbcjcnMjBmuuUtaAVwMbKx56lzgLZIelrRJ0m81+Pm1\nkkYkjYyOjh5PvU0N+thfMzNgBuEuaQC4D7ghIt6seboM/ALwIeADwG9LOrf2NSJiXUQMR8Tw0NDQ\nCZRdX9Ik2+FuZpZlzR1JFZJgvyci7q8z5CfAKxGxD9gn6fvARcAzs1ZpBu7GZGaWyLJbRsBdwI6I\nuL3BsG8Al0sqS1oI/CLJ2vy86q+WffyAmRnZZu6XAdcCWyVtTh+7CVgOEBF3RsQOSQ8BW4BJ4MsR\n8eRcFNxMsuZ+aL5/rZlZx2kZ7hHxCKAM4z4PfH42ijpeXpYxM0sU5hOqkJwvs29sot1lmJm1XbHC\nvVrm4MQkY+MOeDPrboULd/DhYWZmxQx375gxsy5XrHBPz3T3EQRm1u0KFe6DbpJtZgYULNzdjcnM\nLFGocO/3mruZGVCwcJ9alvGau5l1u0KFu5dlzMwShQr3BZUSPfIbqmZmhQp3SQxUy16WMbOuV6hw\nh/TwMM/czazLFS/c+3wypJlZ8cLdM3czswKGe1/F4W5mXa9w4T7ombuZWaYeqsskbZC0XdI2Sdc3\nGfsuSeOSPjy7ZWbXXy15zd3Mul6WHqrjwI0R8bikQWCTpPURsX36IEkl4DbgL+agzswGql6WMTNr\nOXOPiF0R8Xh6ew+wAzijztB/A9wH7J7VCmdooC9ZlpmcjHaWYWbWVjNac5e0ArgY2Fjz+BnArwN3\ntPj5tZJGJI2Mjo7OrNKMps6X2X/IrfbMrHtlDndJAyQz8xsi4s2ap/8A+HRETDZ7jYhYFxHDETE8\nNDQ082ozOHy+jNfdzayLZVlzR1KFJNjviYj76wwZBr4qCWAx8EFJ4xHx9VmrNKMjx/4eAvrm+9eb\nmXWEluGuJLHvAnZExO31xkTEz08b/xXgwXYEO/jYXzMzyDZzvwy4FtgqaXP62E3AcoCIuHOOajsu\nPvbXzCxDuEfEI4CyvmBE/PMTKehEDbiPqplZ8T6hOuBlGTOz4oa7l2XMrJsVLtwP75bxzN3Muljh\nwr233EO13OOZu5l1tcKFO8Bgn0+GNLPuVshwd8MOM+t2hQz3/qpb7ZlZdytkuA9Uy+zxzN3Mulgh\nw33QTbLNrMsVMtwHqmX2HXS4m1n3Kma4e+ZuZl2ukOHe7zV3M+tyhQz3wWqZg+OTjI27G5OZdadC\nhvuRkyEd7mbWnQoZ7ktOSjow7dy9t82VmJm1RyHD/ZfOHaK33MO3tu5qdylmZm1RyHAfqJb5lbcP\n8a2tu5iYjHaXY2Y271qGu6RlkjZI2i5pm6Tr64z5Z5K2SNoq6a8kXTQ35Wb3axeezu49Y4y88Gq7\nSzEzm3dZZu7jwI0RsRJYDVwnaWXNmOeBX46IC4DPAetmt8yZu/K80+ir9PBNL82YWRdqGe4RsSsi\nHk9v7wF2AGfUjPmriHgtvfsocOZsFzpT/dUyV553Gt/a+qKXZsys68xozV3SCuBiYGOTYZ8Avt3g\n59dKGpE0Mjo6OpNffVw+dMHpvLx3jI3PvzLnv8vMrJNkDndJA8B9wA0R8WaDMb9CEu6frvd8RKyL\niOGIGB4aGjqeemfkyvNOY0GlxDe3eGnGzLpLpnCXVCEJ9nsi4v4GYy4EvgxcExEdMVVe0Fvive84\njYeefJHxicl2l2NmNm+y7JYRcBewIyJubzBmOXA/cG1EPDO7JZ6YX7twKa/sO8jG571rxsy6RznD\nmMuAa4Gtkjanj90ELAeIiDuBzwKnAl9K/hYwHhHDs1/uzF3x9tPo7y3x4JafcdnZi9tdjpnZvGgZ\n7hHxCKAWYz4JfHK2ippNfZUS71u5hIeefJFbrzmfSqmQn9syMztKVyTdhy5Yymv7D/HXz3bEWwFm\nZnOuK8L9PecOMVAte9eMmXWNrgj3vkqJq1Yu4aFtL3Jw3LtmzKz4uiLcIdk188bfH+Ivn3253aWY\nmc25rgn3y89ZzGCfl2bMrDt0TbhXyyXev/KtfMdLM2bWBbom3CFZmtlzYJwf/HDuz7UxM2unrgr3\ny85ezMkLKl6aMbPC66pw7y338IF3LmH99pc4cMjNs82suLoq3AE+dOHp7Bkb5wc/9K4ZMyuurgv3\nd7/tVBYtrPDglp+1uxQzsznTdeFeKfWw5p1v5f96acbMCqzrwh2S5tn7Dk7w8NPeNWNmxdSV4b76\nrFM4pb/XzbPNrLC6MtzLpR7WnP9W1m9/kT/56xe8PGNmhdOV4Q7wqV9+GyuXnsRvf2Mbl9+2gTse\nfpY9Bw61uywzs1mRpc3eMkkbJG2XtE3S9XXGSNJ/lbRT0hZJl8xNubNn2SkLue9T7+ara1fzjqWD\n3PbQU7z7v3yXL3znaV7ZO9bu8szMTkiWNnvjwI0R8bikQWCTpPURsX3amF8Fzkm/fhG4I/3e0SSx\n+qxTWX3WqWz9yRt86eGd/OHDO/nyI8/xkXctZ+17zuL0RQvaXaaZ2YxlabO3C9iV3t4jaQdwBjA9\n3K8B/kdEBPCopEWSlqY/mwsXnHkyd3z0F9i5ey93fu9Z/uejP+KejT/ife9YwumLFnBKfy+LFlY4\nZWEvixb2ckp/L29ZWGHRwl56y127umVmHSrLzP0wSSuAi4GNNU+dAfzdtPs/SR/LTbhPOfu0Ab7w\nGxfxb686lz/6/nOs3/4S33tmlP0HG7/p2lfpoSTR0yNKPTpyW8n9nh7okRDJvxZgWlNaHfXt8PMn\nanZexczmwj9+1zI++UtnzenvyBzukgaA+4AbIuLN4/llktYCawGWL19+PC8xb85YtICbr34nN1/9\nTgDGxid4ff8hXt13kNf2H+S1fYd4bf9BXt9/kD0HxpmYDCYimEy/T0zCxOQkE5MwGcFkBBHJa6ff\niPSBqftHbpyYmK0XMrM5sXigOue/I1O4S6qQBPs9EXF/nSE/BZZNu39m+thRImIdsA5geHg4VwlU\nLZdYclKJJSf1tbsUM7OWsuyWEXAXsCMibm8w7AHgt9JdM6uBN/K03m5mVjRZZu6XAdcCWyVtTh+7\nCVgOEBF3At8CPgjsBPYDH5/9Us3MLKssu2UeocX7c+kumetmqygzMzsx3sNnZlZADnczswJyuJuZ\nFZDD3cysgBzuZmYFpKlPSc77L5ZGgR8d548vBvLW4do1z4+81Zy3esE1z5dGNf9cRAy1+uG2hfuJ\nkDQSEcPtrmMmXPP8yFvNeasXXPN8OdGavSxjZlZADnczswLKa7iva3cBx8E1z4+81Zy3esE1z5cT\nqjmXa+5mZtZcXmfuZmbWRO7CXdIaSU+nzbg/0+56spD0gqStkjZLGml3PfVI+mNJuyU9Oe2xUySt\nl/TD9Ptb2lnjdA3qvVnST9PrvFnSB9tZY61GzeY7/Do3qrkjr7WkPkmPSXoirfeW9PGfl7QxzY0/\nk9Tb7lqnNKn5K5Ken3aNV83ohSMiN19ACXgWOAvoBZ4AVra7rgx1vwAsbncdLWp8D3AJ8OS0x34X\n+Ex6+zPAbe2us0W9NwP/vt21Nal5KXBJensQeAZY2eHXuVHNHXmtSU6wHUhvV0hagq4G7gU+kj5+\nJ/CpdteaoeavAB8+3tfN28z9UmBnRDwXEQeBr5I057YTFBHfB16tefga4O709t3AP5zXoppoUG9H\ni4hdEfF4ensPMNVsvpOvc6OaO1Ik9qZ3K+lXAFcCX0sf77Rr3KjmE5K3cG/UiLvTBfAXkjalfWTz\nYkkc6aj1IrCkncVk9K8lbUmXbTpmeaNWTbP5XFznmpqhQ6+1pFLaWGg3sJ7kX/uvR8R4OqTjcqO2\n5oiYusa/k17j35c0o8areQv3vLo8Ii4BfhW4TtJ72l3QTEXyb8ZO31p1B/A2YBWwC/i99pZTX7Nm\n8516nevU3LHXOiImImIVSS/nS4Hz2lxSS7U1Szof+I8ktb8LOAX49ExeM2/hnqkRd6eJiJ+m33cD\nf07yf7g8eEnSUoD0++4219NURLyU/kcyCfwRHXidGzSb7+jrXK/mPFzriHgd2AD8A2CRpKnOcx2b\nG9NqXpMuiUVEjAH/nRle47yF+98A56TvfPcCHyFpzt2xJPVLGpy6DbwfeLL5T3WMB4CPpbc/Bnyj\njbW0NBWQqV+nw65zk2bzHXudG9Xcqdda0pCkRentBcBVJO8TbAA+nA7rtGtcr+anpv3BF8l7BDO6\nxrn7EFO65eoPSHbO/HFE/E6bS2pK0lkks3VIetb+aSfWLOl/AVeQnET3EvCfga+T7DJYTnKC529G\nREe8idmg3itIlgmCZIfSv5y2lt12ki4HfgBsBSbTh28iWcPu1OvcqOZ/Qgdea0kXkrxhWiKZvN4b\nEbem/x1+lWR542+Bj6Yz4rZrUvN3gSGS3TSbgX817Y3X1q+bt3A3M7PW8rYsY2ZmGTjczcwKyOFu\nZlZADnczswJyuJuZFZDD3cysgBzuZmYF5HA3Myug/w/0pWZWnIZLVgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5WfCFImsx7Po",
        "colab_type": "text"
      },
      "source": [
        "# 1 Layer MLP with a ReLU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97YgqpL-x-uM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "outputId": "82d7ac8a-bd77-492d-b572-b82c8a91ac43"
      },
      "source": [
        "HIDDEN = 32\n",
        "\n",
        "W = np.random.normal(size=(2, HIDDEN)) / np.sqrt(HIDDEN)\n",
        "a = np.zeros(shape=(HIDDEN, ))\n",
        "V = np.random.normal(size=(HIDDEN, 1)) / np.sqrt(HIDDEN)\n",
        "\n",
        "mlp_losses = []\n",
        "\n",
        "def relu(x):\n",
        "    return np.where(x > 0.0, x, 0.0)\n",
        "\n",
        "for _ in range(EPOCHS):\n",
        "    p = X @ W + a\n",
        "    h = relu(p)\n",
        "    y_pred = h @ V\n",
        "    loss = 0.5 * np.sum((y_pred - y)**2)\n",
        "    mlp_losses.append(loss)\n",
        "\n",
        "    # loss = 0.5 * \\sum_i (yp_i - y_i)^2\n",
        "    #      = \\sum_i yp_i^2 - 2 * yp_i * y_i + y_i^2\n",
        "    # dloss/dyp_i = 0.5 * (2 * yp_i - 2 * y_i) = yp_i - y_i\n",
        "    dloss_dy_pred = y_pred - y\n",
        "    \n",
        "    # dloss/dV = dy_pred/dV * dloss/dy_pred\n",
        "    #          = h^T * dloss/dy_pred\n",
        "    dloss_dV = h.T @ dloss_dy_pred\n",
        "    \n",
        "    # dloss/dh = dloss/dy_pred * dy_pred/dh\n",
        "    dloss_dh = dloss_dy_pred @ V.T\n",
        "    \n",
        "    # dloss/dp = dloss/dh * dh/dp\n",
        "    #          = dloss/dh * is_non_zero(dh)\n",
        "    dloss_dp = np.where(h > 0.0, dloss_dh, 0.0)\n",
        "\n",
        "    # dloss/da = dloss/dp * dp/da\n",
        "    #          =          * ones(len(p))\n",
        "    #          = sum(dloss/dp, axis=0)\n",
        "    dloss_da = np.sum(dloss_dp, axis=0)\n",
        "    \n",
        "    # dloss/dW = dp/dW * dloss/dp\n",
        "    #            X^T * dloss/dp\n",
        "    dloss_dW = X.T @ dloss_dp\n",
        "\n",
        "    a -= ETA * dloss_da\n",
        "    V -= ETA * dloss_dV\n",
        "    W -= ETA * dloss_dW\n",
        "\n",
        "print(\"Predictions after training:\\n{}\".format(y_pred))\n",
        "plt.plot(lin_losses)\n",
        "plt.plot(mlp_losses)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions after training:\n",
            "[[-1.        ]\n",
            " [ 0.99999999]\n",
            " [ 1.        ]\n",
            " [-1.        ]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f05c21bb320>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGcxJREFUeJzt3XuYXHWd5/H3p28kHUIupIGYTgiX\nIHINoeUyosYgF3UeYFcccJARlY3MoMB6WVFnUZiZZ8d5nkGHBWEiF0FRQUAmw4KSFVxlV4HOlSSA\nRIdLIpCQkITc08l3/zgnUHSquyrdVX2qTn1ez3OoU6d+fc73OSSfPvnV7/yOIgIzM8uXpqwLMDOz\nynO4m5nlkMPdzCyHHO5mZjnkcDczyyGHu5lZDjnczcxyyOFuZpZDDnczsxxqyerA48aNi8mTJ2d1\neDOzujR37tzXIqKjVLvMwn3y5Ml0d3dndXgzs7ok6YVy2rlbxswshxzuZmY55HA3M8shh7uZWQ45\n3M3McsjhbmaWQyXDXdIwSU9IWihpiaSri7S5SNIqSQvS5eLqlGtmZuUo58p9KzAjIo4FpgJnSjqp\nSLu7ImJqutxc0SoLPPvKG/zTz59h3ebt1TqEmVndKxnukdiQvm1Nl8wevPrC6o1891d/4IXVG7Mq\nwcys5pXV5y6pWdICYCUwJyIeL9Lso5IWSbpH0sSKVlmgc0w7AMtf31ytQ5iZ1b2ywj0idkTEVKAT\nOEHSUb2a/DswOSKOAeYAtxfbj6SZkrolda9atWpABU8YMxyA5a9vGtDPm5k1gj0aLRMRa4FHgTN7\nbV8dEVvTtzcDx/fx87Mioisiujo6Ss57U9So4a3sM6zFV+5mZv0oZ7RMh6TR6fpw4DTgmV5txhe8\nPQt4upJF9tY5pt3hbmbWj3JmhRwP3C6pmeSXwd0R8YCka4DuiJgNXCbpLKAHWANcVK2CATrHDOd5\nf6FqZtankuEeEYuA44psv6pg/avAVytbWt86x7Tz2LLXiAgkDdVhzczqRl3eodo5Zjibtu3g9U0e\n625mVkxdhrtHzJiZ9a8uw70zDfcV/lLVzKyoOg1338hkZtafugz3UcNbGTmsxd0yZmZ9qMtwB491\nNzPrTx2H+3CHu5lZH+o83DcRkdkElWZmNauOw72djdt2sNZj3c3MdlO34T5h9K6x7u6aMTPrrW7D\nvdM3MpmZ9aluw31iOtZ9xVpfuZuZ9Va34b7P8BZG7uV53c3MiqnbcJfEhHTEjJmZvV3dhjv4RiYz\ns77UebgnNzJ5rLuZ2dvVfbhv2NrDus0e625mVqjOw92zQ5qZFVPn4e6x7mZmxZQMd0nDJD0haaGk\nJZKuLtJmL0l3SVom6XFJk6tRbG9vhbuv3M3MCpVz5b4VmBERxwJTgTMlndSrzWeA1yPiUODbwLcq\nW2Zxo4a3srfHupuZ7aZkuEdiQ/q2NV16D085G7g9Xb8HOFWSKlZlHyR56l8zsyLK6nOX1CxpAbAS\nmBMRj/dqMgF4CSAieoB1wL6VLLQvnb6RycxsN2WFe0TsiIipQCdwgqSjBnIwSTMldUvqXrVq1UB2\nsZvOMe2s8Fh3M7O32aPRMhGxFngUOLPXRyuAiQCSWoBRwOoiPz8rIroioqujo2NgFffSOWY4b2zt\nYf3mnorsz8wsD8oZLdMhaXS6Phw4DXimV7PZwCfT9XOBR2KILqV3jZh5yV0zZmZvKufKfTzwqKRF\nwJMkfe4PSLpG0llpm1uAfSUtA74AXFmdcnfnG5nMzHbXUqpBRCwCjiuy/aqC9S3AxypbWnneeiKT\nr9zNzHap6ztUAUa3tzKirdlX7mZmBeo+3JOx7u1+IpOZWYG6D3fANzKZmfWSo3B3n7uZ2S45Cfd2\n3tjied3NzHbJSbh7xIyZWaGchLvHupuZFcpFuE/wvO5mZm+Ti3Af095Ke1uzu2XMzFK5CHfP625m\n9na5CHd4a+pfMzPLVbh7rLuZ2S65Cvf1HutuZgbkKtyT4ZDumjEzy1W4+0YmM7NdchTuvpHJzGyX\n3IT7mPZWhrd6XnczM8hRuL811t3dMmZmuQl38LzuZma7lAx3SRMlPSppqaQlki4v0ma6pHWSFqTL\nVcX2VW1+IpOZWaLkA7KBHuCLETFP0khgrqQ5EbG0V7vfRMSfV77E8nWOGc66zdtZv2U7+wxrzbIU\nM7NMlbxyj4iXI2Jeuv4G8DQwodqFDYTHupuZJfaoz13SZOA44PEiH58saaGkhyQdWYHa9linp/41\nMwPK65YBQNLewL3AFRGxvtfH84ADI2KDpA8D9wNTiuxjJjATYNKkSQMuui++kcnMLFHWlbukVpJg\nvzMi7uv9eUSsj4gN6fqDQKukcUXazYqIrojo6ujoGGTpuxs7os1j3c3MKG+0jIBbgKcj4to+2hyQ\ntkPSCel+V1ey0HJIYoLHupuZldUt8x7gQuApSQvSbV8DJgFExE3AucBfS+oBNgPnR0RUod6SPNbd\nzKyMcI+IxwCVaHM9cH2lihqMzjHDmf/i2qzLMDPLVK7uUIVkOOS6zdt5Y4vndTezxpXDcE9GzPhO\nVTNrZDkM93Tq3zUOdzNrXDkMd491NzPLXbjvO6KNYa1NHjFjZg0td+EuiQmjPRzSzBpb7sIdkn73\n5WvdLWNmjSun4e4rdzNrbDkN93bWbtrOuk0e625mjSmX4X7CQWOR4JoHlpLRLAhmZpnKZbgff+AY\nLpsxhXvnLedHT7yYdTlmZkMul+EOcPmpU3j/YR1cPXspC17yXDNm1lhyG+5NTeI7502lY+Re/M0P\n57Jm47asSzIzGzK5DXeAMSPauOkTx/Paxm1c9uP57Njp/nczawy5DneAoztH8XdnH8ljy17j23N+\nn3U5ZmZDIvfhDnDeuydxXtdErn90GXOWvpp1OWZmVdcQ4Q5w9dlHcvSEUXzh7gU8/9rGrMsxM6uq\nhgn3Ya3NfPeCaTQ3iUt+OJfN23ZkXZKZWdU0TLgDTBzbznfOm8qzr77B13/2lG9wMrPcKhnukiZK\nelTSUklLJF1epI0kXSdpmaRFkqZVp9zBm/7O/bji1MO4b/4K7vjtCw54M8ulkg/IBnqAL0bEPEkj\ngbmS5kTE0oI2HwKmpMuJwI3pa036/IxDWfDS63xj9hL+5ZfPcfSEURzbOYqjO0dzbOco9ttnWNYl\nmpkNSslwj4iXgZfT9TckPQ1MAArD/Wzgjkgug38nabSk8enP1pymJnHDBdP42fwVLHxpLYuWr+OG\nX7325jj4A/YZxtGdSeBP2X8kI9pa2Ku1iWEtzUVf25qbkJK55M3MakE5V+5vkjQZOA54vNdHE4CX\nCt4vT7fVZLgDtLe1cMGJB3LBiQcCsHnbDpb8aR2Llq9j0fIk8AcybFICkQR9k0AkG5R+9ra27P7L\nwL8fzPLv4lMO4gunv7Oqxyg73CXtDdwLXBER6wdyMEkzgZkAkyZNGsguqmZ4WzNdk8fSNXnsm9vW\nb9nO869tZGvPTrZs38HW7TvZ0rP767aenURAAESwMyAIInhznV5d+8V6+t3/b9YYpk4aXfVjlBXu\nklpJgv3OiLivSJMVwMSC953ptreJiFnALICurq6aT7J9hrVyTGf1/yeYmVVaOaNlBNwCPB0R1/bR\nbDbwV+momZOAdbXa325m1gjKuXJ/D3Ah8JSkBem2rwGTACLiJuBB4MPAMmAT8KnKl2pmZuUqZ7TM\nY1Dkm7+3twng0koVZWZmg9NQd6iamTUKh7uZWQ453M3McsjhbmaWQw53M7MccribmeWQw93MLIcc\n7mZmOeRwNzPLIYe7mVkOOdzNzHLI4W5mlkMOdzOzHHK4m5nlkMPdzCyHHO5mZjnkcDczyyGHu5lZ\nDtVfuK+YB/f+F9i2MetKzMxqVslwl3SrpJWSFvfx+XRJ6yQtSJerKl9mgS3r4Km74T9+XdXDmJnV\ns3Ku3L8PnFmizW8iYmq6XDP4svpx4HugbW/4/S+qehgzs3pWMtwj4tfAmiGopTwtbXDwdHjuYYjI\nuhozs5pUqT73kyUtlPSQpCMrtM++HXYGrF8Bry6p+qHMzOpRJcJ9HnBgRBwL/E/g/r4aSpopqVtS\n96pVqwZ+xCmnJ6/PuWvGzKyYQYd7RKyPiA3p+oNAq6RxfbSdFRFdEdHV0dEx8IOOPADGHwu/f3jg\n+zAzy7FBh7ukAyQpXT8h3efqwe63pClnwPInYFPtfB1gZlYryhkK+WPgt8A7JS2X9BlJl0i6JG1y\nLrBY0kLgOuD8iCH4pvOwMyB2wrJfVv1QZmb1pqVUg4j4eInPrweur1hF5XrHNGgfl/S7H/OxIT+8\nmVktq787VHdpaoIpp8Gy/w07erKuxsysptRvuEPSNbP5dVj+ZNaVmJnVlPoO90NmQFOLh0SamfVS\n3+E+bBRMOtlDIs3MeqnvcIfkhqaVS2DtS1lXYmZWM+o/3A87I3l9zlfvZma71H+4jzsMRh/ocDcz\nK1D/4S4lV+9//D+wfXPW1ZiZ1YT6D3dIpiLo2QzPP5Z1JWZmNSEf4T75FGht9wM8zMxS+Qj31mFw\n0PuT8e5+gIeZWU7CHeCw02Hti7DqmawrMTPLXH7CfUo6JNJdM2ZmOQr3URNg/6M9JNLMjDyFOyRd\nMy/+LplMzMysgeUr3KecAbED/vBI1pWYmWUqX+He2QXDx3oiMTNrePkK96ZmOPSDsGwO7NyRdTVm\nZpnJV7hDMhXBptWwYl7WlZiZZaacB2TfKmmlpMV9fC5J10laJmmRpGmVL3MPHDID1OQHeJhZQyvn\nyv37wJn9fP4hYEq6zARuHHxZg9A+Fiae6PHuZtbQSoZ7RPwaWNNPk7OBOyLxO2C0pPGVKnBADjsD\nXlkEr7+QaRlmZlmpRJ/7BKDwMUjL023ZOercpGtm/g8yLcPMLCtD+oWqpJmSuiV1r1q1qnoHGj0x\nefzevDtgx/bqHcfMrEZVItxXABML3nem23YTEbMioisiujo6Oipw6H50fRo2vArPPljd45iZ1aBK\nhPts4K/SUTMnAesi4uUK7HdwDv0gjJoI3bdmXYmZ2ZBrKdVA0o+B6cA4ScuBbwCtABFxE/Ag8GFg\nGbAJ+FS1it0jTc1w/Cfhkb+H1X+AfQ/JuiIzsyFTMtwj4uMlPg/g0opVVEnHXQi/+keYexuc/vdZ\nV2NmNmTyd4dqoZEHwOEfgfl3Qs/WrKsxMxsy+Q53gOM/BZvXwNLZWVdiZjZk8h/uB70fxh7sL1bN\nrKHkP9ybmpKr9xf/H6x8OutqzMyGRP7DHWDqBdDcBt23ZV2JmdmQaIxwH7EvHHEOLPwJbNuYdTVm\nZlXXGOEOyR2rW9fB4vuyrsTMrOoaJ9wnnQQd7/IXq2bWEBon3KXk6v1P8+BPC7Kuxsysqhon3AGO\nPQ9a25M7Vs3Mcqyxwn3YKDjqP8Oin8KW9VlXY2ZWNY0V7pB0zWzfCE/dnXUlZmZV03jh/o5pMP5Y\nePJWiMi6GjOzqmi8cN/1xerKJbD8yayrMTOrisYLd0iesdo20sMizSy3GjPc99o7GTmz+D5Y+2LW\n1ZiZVVxjhjvAey6H5laYfZn73s0sdxo33EdPgg9+E/74KMz/QdbVmJlVVOOGO0DXZ2Dye+EXX4d1\nK7KuxsysYsoKd0lnSnpW0jJJVxb5/CJJqyQtSJeLK19qFTQ1wVnXwc4eeOAKd8+YWW6UDHdJzcAN\nwIeAI4CPSzqiSNO7ImJqutxc4TqrZ+zBcOpV8NzDyZTAZmY5UM6V+wnAsoj4Y0RsA34CnF3dsobY\nCZ+FiSfBz78Cb7ySdTVmZoNWTrhPAF4qeL883dbbRyUtknSPpIkVqW6oNDXB2TdAz1Z44AvunjGz\nulepL1T/HZgcEccAc4DbizWSNFNSt6TuVatWVejQFTLuUPjA1+HZ/wWL7826GjOzQSkn3FcAhVfi\nnem2N0XE6ojYmr69GTi+2I4iYlZEdEVEV0dHx0Dqra6TL4UJXfDgl2HDyqyrMTMbsHLC/UlgiqSD\nJLUB5wOzCxtIGl/w9izg6cqVOISampPumW0b4MEvZV2NmdmAlQz3iOgBPgf8giS0746IJZKukXRW\n2uwySUskLQQuAy6qVsFVt9/hMP1KWPpvsOT+rKsxMxsQRUZfHnZ1dUV3d3cmxy5pRw/cfCqsWw6X\nPgEj9s26IjMzACTNjYiuUu0a+w7VvjS3wDnfhS3r4KEve/SMmdUdh3tf9j8Spn8lGTnzy2sc8GZW\nV1qyLqCmvfdLyZwzj10LLXslffFmZnXA4d4fCT5yLezYDr/6H8kUwe/9YtZVmZmV5HAvZdfkYju2\nJd0zzW3wZ5/Puiozs3453MvR1Azn3JgE/MN/mwT8iZ/Nuiozsz453MvV3AIfvTmZHvih/5Z00XR9\nOuuqzMyK8miZPdHcCufeBlPOgAf+K8zzE5zMrDY53PdUSxv8xR1wyAyY/XlYeFfWFZmZ7cbhPhCt\nw+D8H8HkU+D+SzyLpJnVHIf7QLUOh7+8K3nIxz2fTp7Dun1L1lWZmQEO98FpGwGfuBfefTH89nr4\n3gx4ZXHWVZmZOdwHra0dPvLP8Jc/hY2r4HsfgP97HezcmXVlZtbAHO6Vctjp8De/hSmnw5z/Dnec\nBWtfKv1zZmZV4HCvpBHj4LwfwlnXw5/mw43vgUU/zboqM2tADvdKk2DahXDJY8mDP+67OPnCdfPr\nWVdmZg3E4V4tYw+Cix6EGX+bPNXpuuPgoSvhlaeyrszMGoDDvZqaW+B9X4aLfwkHT4fuW+CmU+Bf\n3wePz4JNa7Ku0Mxyyo/ZG0qb1sBT98D8H8Ari5IJyA7/CBz3CTj4A8kEZWZm/Sj3MXtlhbukM4F/\nAZqBmyPiH3t9vhdwB3A8sBo4LyKe72+fDRnuhV5eBAvuhEV3w+Y1sM8EeNdZ8I6pcMDRMO6wZC4b\nM7MC5YZ7yVkhJTUDNwCnAcuBJyXNjoilBc0+A7weEYdKOh/4FnDewEpvEOOPSZbTroFnH0qCfu5t\n8Hh6l2tzG+z3riTo9z86eT3gKBg2Ktu6zawulDPl7wnAsoj4I4CknwBnA4XhfjbwzXT9HuB6SYqs\n+nzqSctecOQ5ybKjB1YvS750fWVR8vrsz2H+D99qP2K/ZMhl+77QPjZ9LVzGwl6jkgnOWoYl+28Z\nlvyy2PXe3T9muVdOuE8ACu/GWQ6c2FebiOiRtA7YF3itEkU2jOaWZPjkfofDMR9LtkXAhleToH95\nIax9Iem737QGVj4Dm1Yn3TqxB3fENrUmXT5qevvS1JyuN7+1DUDpf6Tir72p97YibQZit/2a1anj\nLoQ/+1xVDzGkD+uQNBOYCTBp0qShPHT9kmDkAcky5bTibXbuhC1r09B/DbZugB1boWcL9GwtWLYk\nT5Pavjl56EjsTJadO9L1HQXrO5NfLESJ1956bavYP978j0DLkb33q/ohygn3FcDEgved6bZibZZL\nagFGkXyx+jYRMQuYBckXqgMp2Ipoakq7aMYCh2ZdjZnVgHLGuT8JTJF0kKQ24Hxgdq82s4FPpuvn\nAo+4v93MLDslr9zTPvTPAb8gGQp5a0QskXQN0B0Rs4FbgB9IWgasIfkFYGZmGSmrzz0iHgQe7LXt\nqoL1LcDHKluamZkNlKcfMDPLIYe7mVkOOdzNzHLI4W5mlkMOdzOzHMpsyl9Jq4AXBvjj46i/qQ1c\n89Cot5rrrV5wzUOlr5oPjIiOUj+cWbgPhqTucqa8rCWueWjUW831Vi+45qEy2JrdLWNmlkMOdzOz\nHKrXcJ+VdQED4JqHRr3VXG/1gmseKoOquS773M3MrH/1euVuZmb9qLtwl3SmpGclLZN0Zdb1lEPS\n85KekrRAUk0+FVzSrZJWSlpcsG2spDmSnktfx2RZY6E+6v2mpBXpeV4g6cNZ1tibpImSHpW0VNIS\nSZen22v5PPdVc02ea0nDJD0haWFa79Xp9oMkPZ7mxl3p9OU1oZ+avy/pPwrO8dQ92nFE1M1CMuXw\nH4CDgTZgIXBE1nWVUffzwLis6yhR4/uAacDigm3/BFyZrl8JfCvrOkvU+03gS1nX1k/N44Fp6fpI\n4PfAETV+nvuquSbPNckzHfdO11uBx4GTgLuB89PtNwF/nXWtZdT8feDcge633q7c33xYd0RsA3Y9\nrNsGKSJ+TTIXf6GzgdvT9duBc4a0qH70UW9Ni4iXI2Jeuv4G8DTJ84dr+Tz3VXNNisSG9G1rugQw\nA7gn3V5r57ivmgel3sK92MO6a/YPWoEAHpY0N32ObL3YPyJeTtdfAfbPspgyfU7SorTbpma6N3qT\nNBk4juQqrS7Oc6+aoUbPtaRmSQuAlcAckn/tr42InrRJzeVG75ojYtc5/of0HH9b0l57ss96C/d6\ndUpETAM+BFwq6X1ZF7SnIvk3Y60PrboROASYCrwM/HO25RQnaW/gXuCKiFhf+FmtnuciNdfsuY6I\nHRExleR5zycAh2dcUkm9a5Z0FPBVktrfDYwFvrIn+6y3cC/nYd01JyJWpK8rgZ+R/IGrB69KGg+Q\nvq7MuJ5+RcSr6V+SncD3qMHzLKmVJCTvjIj70s01fZ6L1VwP5zoi1gKPAicDoyXtevJczeZGQc1n\npl1iERFbgdvYw3Ncb+FezsO6a4qkEZJG7loHTgcW9/9TNaPwweefBP4tw1pK2hWQqf9EjZ1nSSJ5\n3vDTEXFtwUc1e577qrlWz7WkDkmj0/XhwGkk3xM8CpybNqu1c1ys5mcKfuGL5DuCPTrHdXcTUzrk\n6ju89bDuf8i4pH5JOpjkah2SZ9b+qBZrlvRjYDrJTHSvAt8A7icZZTCJZAbPv4iImvgSs496p5N0\nEwTJCKXPFvRlZ07SKcBvgKeAnenmr5H0Ydfqee6r5o9Tg+da0jEkX5g2k1y83h0R16R/D39C0r0x\nH/hEekWcuX5qfgToIBlNswC4pOCL19L7rbdwNzOz0uqtW8bMzMrgcDczyyGHu5lZDjnczcxyyOFu\nZpZDDnczsxxyuJuZ5ZDD3cwsh/4/Yxj/VvL+zYIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dwlyau3hyLi2",
        "colab_type": "text"
      },
      "source": [
        "# MLP Exercise\n",
        "\n",
        "Let's change the ReLU to `tanh`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "THYzK3vzyP7Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "outputId": "6144a6ee-2323-4973-ea7a-ba0c5693ae69"
      },
      "source": [
        "W = np.random.normal(size=(2, HIDDEN)) / np.sqrt(HIDDEN)\n",
        "a = np.zeros(shape=(HIDDEN, ))\n",
        "V = np.random.normal(size=(HIDDEN, 1)) / np.sqrt(HIDDEN)\n",
        "\n",
        "mlp_losses2 = []\n",
        "\n",
        "def relu(x):\n",
        "    return np.where(x > 0.0, x, 0.0)\n",
        "\n",
        "for _ in range(EPOCHS):\n",
        "    p = X @ W + a\n",
        "    h = relu(p)  # <--- CHANGE THIS\n",
        "    y_pred = h @ V\n",
        "    loss = 0.5 * np.sum((y_pred - y)**2)\n",
        "    mlp_losses2.append(loss)\n",
        "\n",
        "    # loss = 0.5 * \\sum_i (yp_i - y_i)^2\n",
        "    #      = \\sum_i yp_i^2 - 2 * yp_i * y_i + y_i^2\n",
        "    # dloss/dyp_i = 0.5 * (2 * yp_i - 2 * y_i) = yp_i - y_i\n",
        "    dloss_dy_pred = y_pred - y\n",
        "    \n",
        "    # dloss/dV = dy_pred/dV * dloss/dy_pred\n",
        "    #          = h^T * dloss/dy_pred\n",
        "    dloss_dV = h.T @ dloss_dy_pred\n",
        "    \n",
        "    # dloss/dh = dloss/dy_pred * dy_pred/dh\n",
        "    dloss_dh = dloss_dy_pred @ V.T\n",
        "    \n",
        "    # dloss/dp = dloss/dh * dh/dp\n",
        "    #          = dloss/dh * is_non_zero(dh)\n",
        "    dloss_dp = np.where(h > 0.0, dloss_dh, 0.0)\n",
        "\n",
        "    # dloss/da = dloss/dp * dp/da\n",
        "    #          =          * ones(len(p))\n",
        "    #          = sum(dloss/dp, axis=0)\n",
        "    dloss_da = np.sum(dloss_dp, axis=0)\n",
        "    \n",
        "    # dloss/dW = dp/dW * dloss/dp\n",
        "    #            X^T * dloss/dp\n",
        "    dloss_dW = X.T @ dloss_dp\n",
        "\n",
        "    a -= ETA * dloss_da\n",
        "    V -= ETA * dloss_dV\n",
        "    W -= ETA * dloss_dW\n",
        "\n",
        "print(\"Predictions after training:\\n{}\".format(y_pred))\n",
        "plt.plot(lin_losses)\n",
        "plt.plot(mlp_losses)\n",
        "plt.plot(mlp_losses2)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions after training:\n",
            "[[-0.99999999]\n",
            " [ 0.99999998]\n",
            " [ 1.        ]\n",
            " [-0.99999999]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f05c21a7cf8>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X14VOWd//H3d5IJSQBJApEgAUEI\nKgIiBLCrv5Zifaiu0K74WKut9eeyuqtuu1t192e3dfuw3WtrW9dqL9S2Wru2Kq7iUy276lpdKoRH\nBUQDKgZ5iATCMyHJ9/fHGXRIZpIhTGYyJ5/Xdc2VM+fcc+brET453Oc+9zF3R0REwiWS7QJERCT9\nFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhPKz9cWDBg3yESNGZOvr\nRURy0pIlSz5y9/LO2mUt3EeMGEFNTU22vl5EJCeZ2fuptFO3jIhICCncRURCSOEuIhJCCncRkRBS\nuIuIhJDCXUQkhDoNdzMrNLNFZrbCzFaZ2XcStPmKmdWb2fLY69ruKVdERFKRypn7AWCGu58KTATO\nM7PTE7T7nbtPjL3uT2uVcdZu3sW//v4tGvcd7K6vEBHJeZ2Guwd2x95GY6+sPXj1/W17uOfldby/\nbU+2ShAR6fFS6nM3szwzWw5sBRa4++sJml1kZivN7HEzG5bWKuNUlhYDULd9X3d9hYhIzksp3N29\nxd0nApXAVDMb16bJ08AId58ALAAeTLQfM7vOzGrMrKa+vr5LBQ8tLQKgbvveLn1eRKQ3OKLRMu6+\nA3gJOK/N+m3ufiD29n5gcpLPz3X3anevLi/vdN6bhAYURTmmMF9n7iIiHUhltEy5mZXElouAs4G3\n2rQZEvd2JrAmnUW2VVlarHAXEelAKrNCDgEeNLM8gl8Gj7r7M2Z2B1Dj7vOBG81sJtAMNABf6a6C\nASpLi3hPF1RFRJLqNNzdfSVwWoL134pbvg24Lb2lJVdZWsyrtR/h7phZpr5WRCRn5OQdqpWlRext\namH7Xo11FxFJJCfDXSNmREQ6lpPhXhkL9426qCoiklCOhrtuZBIR6UhOhvuAoij9C/PVLSMikkRO\nhjtorLuISEdyONyLFO4iIknkeLjvxT1rE1SKiPRYORzuxexpamGHxrqLiLSTs+E+tOTQWHd1zYiI\ntJWz4V6pG5lERJLK2XAfFhvrvnGHztxFRNrK2XA/piif/n00r7uISCI5G+5mxtDYiBkRETlczoY7\n6EYmEZFkcjzcgxuZNNZdRORwOR/uuw8007hPY91FROLleLhrdkgRkURyPNw11l1EJJFOw93MCs1s\nkZmtMLNVZvadBG36mNnvzKzWzF43sxHdUWxbn4S7ztxFROKlcuZ+AJjh7qcCE4HzzOz0Nm2+Bmx3\n99HAj4EfprfMxAYURemnse4iIu10Gu4e2B17G4292g5PmQU8GFt+HDjLzCxtVSZhZpr6V0QkgZT6\n3M0sz8yWA1uBBe7+epsmQ4EPANy9GWgEBqaz0GQqdSOTiEg7KYW7u7e4+0SgEphqZuO68mVmdp2Z\n1ZhZTX19fVd20U5laTEbNdZdROQwRzRaxt13AC8B57XZtBEYBmBm+cAAYFuCz89192p3ry4vL+9a\nxW1Ulhax60AzO/c1p2V/IiJhkMpomXIzK4ktFwFnA2+1aTYfuDq2PBt40TN0Kn1oxMwH6poREflY\nKmfuQ4CXzGwlsJigz/0ZM7vDzGbG2jwADDSzWuDrwK3dU257upFJRKS9/M4auPtK4LQE678Vt7wf\nuDi9paXmkycy6cxdROSQnL5DFaCkOErfgjyduYuIxMn5cA/GuhfriUwiInFyPtwB3cgkItJGiMJd\nfe4iIoeEJNyL2bVf87qLiBwSknDXiBkRkXghCXeNdRcRiReKcB+qed1FRA4TinAvLY5SXJCnbhkR\nkZhQhLvmdRcROVwowh0+mfpXRERCFe4a6y4ickiown2nxrqLiAChCvdgOKS6ZkREQhXuupFJROSQ\nEIW7bmQSETkkNOFeWhylKKp53UVEIETh/slYd3XLiIiEJtxB87qLiBzSabib2TAze8nMVpvZKjO7\nKUGb6WbWaGbLY69vJdpXd9MTmUREAp0+IBtoBr7h7kvNrD+wxMwWuPvqNu3+6O5/nv4SU1dZWkTj\nvoPs3H+QYwqj2SxFRCSrOj1zd/dN7r40trwLWAMM7e7CukJj3UVEAkfU525mI4DTgNcTbP6Uma0w\ns+fN7JQ01HbEKjX1r4gIkFq3DABm1g+YB9zs7jvbbF4KHO/uu83sfOBJoCrBPq4DrgMYPnx4l4tO\nRjcyiYgEUjpzN7MoQbD/xt2faLvd3Xe6++7Y8nNA1MwGJWg3192r3b26vLz8KEtvr6xvgca6i4iQ\n2mgZAx4A1rj7nUnaVMTaYWZTY/vdls5CU2FmDNVYdxGRlLplzgC+DLxhZstj6/4BGA7g7j8HZgN/\nZWbNwD7gMnf3bqi3UxrrLiKSQri7+6uAddLmbuDudBV1NCpLi1i2YUe2yxARyapQ3aEKwXDIxn0H\n2bVf87qLSO8VwnAPRszoTlUR6c1CGO6xqX8bFO4i0nuFMNw11l1EJHThPrBvAYXRiEbMiEivFrpw\nNzOGlmg4pIj0bqELdwj63et2qFtGRHqvkIa7ztxFpHcLabgXs2PvQRr3aqy7iPROoQz3qSPLMIM7\nnllNlmZBEBHJqlCG++TjS7lxRhXzltbxH4s2ZLscEZGMC2W4A9x0VhWfGVPOd+avZvkHmmtGRHqX\n0IZ7JGL85NKJlPfvw/UPL6FhT1O2SxIRyZjQhjtAad8Cfn7lZD7a08SNjyyjpVX97yLSO4Q63AHG\nVw7gn2edwqu1H/HjBW9nuxwRkYwIfbgDXDplOJdWD+Pul2pZsHpLtssREel2vSLcAb4z6xTGDx3A\n1x9dznsf7cl2OSIi3arXhHthNI97vjSJvIgx5+El7GtqyXZJIiLdJufCffHmxVzzwjU0Hmg84s8O\nKyvmJ5dOZO2WXfzjf76hG5xEJLQ6DXczG2ZmL5nZajNbZWY3JWhjZnaXmdWa2Uozm9Q95YJhLN68\nmCVblnTp89NPPJabzxrDE8s28tDC9xXwIhJKnT4gG2gGvuHuS82sP7DEzBa4++q4Np8HqmKvacC9\nsZ9pN6F8An3y+rB482JmDJ/RpX38zYzRLP9gO/80fxU//e93GD90AKdWDmB8ZQmnVg7g2GMK01y1\niEhmdRru7r4J2BRb3mVma4ChQHy4zwIe8uA0+E9mVmJmQ2KfTauCvAImHjuRRZsXdXkfkYjxsy9N\n4j+XbWTFBztYWdfIz17+6ONx8BXHFDK+Mgj8qsH96VuQT59ohML8vIQ/C/IimAVzyYuI9ASpnLl/\nzMxGAKcBr7fZNBT4IO59XWxd2sMdYFrFNO5adhcN+xsoKyzr0j6KC/L50rTj+dK04wHY19TCqg8b\nWVnXyMq6IPC7MmzSDIwg6CMWdCPx8bo2bWn/y0C/H0TC79ozR/L1c07s1u9IOdzNrB8wD7jZ3Xd2\n5cvM7DrgOoDhw4d3ZRcATKmYAkDN5hrOGXFOl/cTr6ggj+oRZVSP+OSXxc79B3nvoz0caG5l/8EW\nDhxsZX9z+59Nza24gwO40+rgOO58vEybrv1EPf3q/xfpHSYOL+n270gp3M0sShDsv3H3JxI02QgM\ni3tfGVt3GHefC8wFqK6u7nKSnTLoFIryi1i0eVHawj2RYwqjTKjs/v8JIiLplspoGQMeANa4+51J\nms0HroqNmjkdaOyO/vZDopEokwdPZvHmxd31FSIiOS2Vce5nAF8GZpjZ8tjrfDObY2ZzYm2eA9YD\ntcB9wPXdU+4nplZMZX3jeur31nf3V4mI5JxURsu8Cgmu/B3exoEb0lVUKqZWTAWCm5rOP+H8TH61\niEiPl3N3qB5yUtlJ9I/2P6ohkSIiYZWz4Z4XyWNyhfrdRUQSydlwh6BrZsOuDWzesznbpYiI9Cg5\nH+6AumZERNrI6XCvKq2ipE8JizYp3EVE4uV0uEcswpSKKSzavEh3d4qIxMnpcIdgKoJNezZRt7su\n26WIiPQYOR/u8ePdRUQkkPPhfsKAExhYOFAXVUVE4uR8uJsZUyumsnjTYvW7i4jE5Hy4A0wZMoWt\n+7by3s73sl2KiEiPEIpwV7+7iMjhQhHuw/sPZ3DxYPW7i4jEhCLcP+5336x+dxERCEm4QzDevWF/\nA7U7arNdiohI1oUm3KcNmQZonhkREQhRuB/X7ziG9huqi6oiIoQo3IGP+91bvTXbpYiIZFXuhfvG\npTDv/0LTnnabplRMYWfTTtY2rM1CYSIiPUen4W5mvzCzrWb2ZpLt082sMe7h2d9Kf5lx9jfCG4/C\nu6+026T53UVEAqmcuf8KOK+TNn9094mx1x1HX1YHjj8DCvrB2y+02zS472BGHDNC/e4i0ut1Gu7u\n/grQkIFaUpNfACdMh3f+AAnGtE+pmELNlhqaW5szXpqISE+Rrj73T5nZCjN73sxOSdM+kxtzLuzc\nCFtWtds0tWIqew7uYc22Nd1ehohIT5WOcF8KHO/upwL/DjyZrKGZXWdmNWZWU19f3/VvrDon+PlO\n+66Z6opqQP3uItK7HXW4u/tOd98dW34OiJrZoCRt57p7tbtXl5eXd/1L+1fAkFPh7T+02zSoaBCj\nS0ar311EerWjDnczqzAziy1Pje1z29Hut1NV50LdItjb/nLAlIopLN26lIMtB7u9DBGRniiVoZCP\nAAuBE82szsy+ZmZzzGxOrMls4E0zWwHcBVzmmZi9a8y54K1Q+9/tNp1x3Bnsa97Hwk0Lu70MEZGe\nKL+zBu5+eSfb7wbuTltFqTpuEhQPCvrdJ1x82KY/G/pnlPYpZf66+Xy68tMZL01EJNty7w7VQyIR\nqDobav8LWg4f9hiNRDn/hPN5acNLNB5ozFKBIiLZk7vhDkHXzL7tUNf+4umFoy6kqbWJP7zf/qKr\niEjY5Xa4j5oBkfyEQyLHlo1ldMlonl73dBYKExHJrtwO98IBMPxTCYdEmhkXjrqQZVuXsWHnhiwU\nJyKSPbkd7hDc0LR1Fez4oN2mC0ZegGE8vV5n7yLSu+R+uI85N/j5Tvuz98F9B3P6kNN5et3TmuNd\nRHqV3A/3QWOg5PiE4Q4wc/RMNu7eyNItSzNcmIhI9uR+uJsFZ+/r/wcO7mu3ecawGRTnF6trRkR6\nldwPdwimImjeB++92m5TcbSYs48/mxfee4H9zfuzUJyISOaFI9xHnAnR4oQP8ACYNXoWew7u4cUN\nL2a4MBGR7AhHuEcLYeRngvHuCaa1mTx4MkP6DmH++vlZKE5EJPPCEe4AY86BHRug/q12myIW4c9P\n+HMWfriQ+r1HMY+8iEiOCE+4V8WGRCbpmrlw1IW0eivPrn82g0WJiGRHeMJ9wFAYPD7pkMiRA0Yy\noXwCT617ikzMSCwikk3hCXcIumY2/CmYTCyBmSfMpHZHLWu3r81wYSIimRWucK86F7wF1iUeFXPe\nyPPIj+Qzf50urIpIuIUr3Curoags4URiAAP6DGB65XSeXf8sB1v1CD4RCa9whXskD0Z/DmoXQGtL\nwiYzR82kYX8DCz/UI/hEJLzCFe4QTEWwdxtsTDyXzJlDz/z4EXwiImGVygOyf2FmW83szSTbzczu\nMrNaM1tpZpPSX+YRGDUDLJLwAR4A0bwonx/5eT2CT0RCLZUz918B53Ww/fNAVex1HXDv0Zd1FIrL\nYNi0pOPdIZgpUo/gE5Ew6zTc3f0VoKGDJrOAhzzwJ6DEzIakq8AuGXMubF4J299PuHls2VhGDRil\nR/CJSGilo899KBD/GKS62LrsGTc76JpZ9uuEm82MmaNnsmzrMt5qaD9dgYhIrsvoBVUzu87Masys\npr6+G+d4KRkWPH5v6UPQknjI40VVF9Ev2o+5K+d2Xx0iIlmSjnDfCAyLe18ZW9eOu89192p3ry4v\nL0/DV3eg+hrYvQXWPpdw84A+A7ji5CtY8P4C3tn+TvfWIiKSYekI9/nAVbFRM6cDje6+KQ37PTqj\nPwcDhkHNL5I2uWrsVRTnF+vsXURCJ5WhkI8AC4ETzazOzL5mZnPMbE6syXPAeqAWuA+4vtuqPRKR\nPJh8Nax/GbatS9jk0Nn7C++9wLodiduIiOSiVEbLXO7uQ9w96u6V7v6Au//c3X8e2+7ufoO7j3L3\n8e5e0/1lp+i0L0MkH5b8MmmTq8ZeRWF+oc7eRSRUwneHarz+FXDSBbDsN9B8IGGT0sJSLjvpMn7/\n3u95t/HdDBcoItI9wh3uAJO/CvsaYHXy6QauHns1BZEC7lt5XwYLExHpPuEP95GfgbITOrywOrBo\nIJeceAnPvvssG3ZuyGBxIiLdI/zhHokEZ+8b/he2rkna7Kvjvko0EuW+N3T2LiK5L/zhDjDxS5BX\nADXJL6wOKhrExWMu5ul1T1O3qy6DxYmIpF/vCPe+A2HsF2DFb6FpT9JmXx33VfIsj/vfuD+DxYmI\npF/vCHcI7lg90AhvPpG0ybHFx3LRmIt4qvYpPtz9YQaLExFJr94T7sNPh/KTO7ywCnDNuGswMx54\n44EMFSYikn69J9zNgrP3D5fCh8uTNqvoW8EXR3+RJ2qfYPOezRksUEQkfXpPuAOceilEizu8YxXg\n2vHXAujsXURyVu8K98IBMO4vYOVjsH9n0mZD+g1h1qhZzHtnHlv2bMlggSIi6dG7wh2CrpmDe+CN\nRztsdu34a3F3frXqV5mpS0QkjXpfuB83CYacCot/Ae5Jm1X2r+TCURfy2NuPady7iOSc3hfuhy6s\nbl0FdYs7bHr9xOuJWIQfLPoB3sEvAhGRnqb3hTsEz1gt6N/psMiKvhXcMPEGXql7hRc3vJih4kRE\njl7vDPc+/YKRM28+ATs6nijsipOvYEzpGH6w6AfsPbg3QwWKiByd3hnuAGfcBHlRmH9jh33v0UiU\n20+/nS17t3DP8nsyWKCISNf13nAvGQ6f+zasfwmW/brDphOPnchFVRfx8JqHWduwNiPliYgcjd4b\n7gDVX4MR/wde+Edo3Nhh05sn3cwxBcfw3T99l1ZvzVCBIiJdk1K4m9l5ZrbWzGrN7NYE279iZvVm\ntjz2ujb9pXaDSARm3gWtzfDMzR12z5QUlvC3k/+W5fXLebL2yQwWKSJy5DoNdzPLA34GfB4YC1xu\nZmMTNP2du0+MvXJnztyyE+Csb8E7fwimBO7ArNGzmHTsJO5ccifb92/PUIEiIkculTP3qUCtu693\n9ybgt8Cs7i0rw6b+JQw7HX5/C+xKPllYxCLcfvrt7Gnaw51L7sxggSIiRyaVcB8KfBD3vi62rq2L\nzGylmT1uZsPSUl2mRCIw62fQfACe+XqH3TOjS0fz5VO+zJO1T7J0y9IMFikikrp0XVB9Ghjh7hOA\nBcCDiRqZ2XVmVmNmNfX19Wn66jQZNBo++4+w9ll4c16HTedMmMOQvkP45z/9MwdbD2aoQBGR1KUS\n7huB+DPxyti6j7n7Nnc/EHt7PzA50Y7cfa67V7t7dXl5eVfq7V6fugGGVsNzfw+7tyZtVhwt5tap\nt1K7o5aHVz+cwQJFRFKTSrgvBqrMbKSZFQCXAfPjG5jZkLi3M4E16SsxgyJ5QfdM02547u86bDpj\n+AymV07n3hX3smn3pgwVKCKSmk7D3d2bgb8GXiAI7UfdfZWZ3WFmM2PNbjSzVWa2ArgR+Ep3Fdzt\njj0Jpt8Kq5+CVR0Pebxt2m0AfH/R9zWxmIj0KJatUKqurvaampqsfHenWprh/rOgsQ5uWAR9ByZt\n+uCqB/m3mn/jG5O/wVfGfSVzNYpIr2RmS9y9urN2vfsO1WTy8uEL98D+Rnj+7zscPXPV2Ks4+/iz\n+fHSH/PqxlczWKSISHIK92QGnwLTbwlGzvz3HUkD3sz47hnfpaqkim/+zzd5t/HdDBcqItKewr0j\n/+fvYPJX4dU74X9+mLRZcbSYu2bcRX4knxtfvJGdTcmfzyoikgkK946YwQV3wsQr4eUfwB9/lLTp\ncf2O487pd1K3q45vvvJNWlpbMlioiMjhFO6dOTS52PhLgu6Z//33pE2rK6q5bdptvLbxNX669KcZ\nLFJE5HD52S4gJ0Ty4Av3QksT/OH/QV4BTPvLhE0vOfES3t7+Nr9c9UuqSqu4cNSFGS5WREThnrq8\nfLjo/mB64Oe/GTzFqfqahE1vmXoL63as49v/+21GDhjJuEHjMlysiPR26pY5EnlRmP1LqDoXnvlb\nWJr4CU7RSJQfTf8R5cXl3PTiTdTv7WHz6IhI6Cncj1R+AVzyEIyaAfP/Blb8LmGzssIyfvrZn7Lr\n4C5ufulmDrQcSNhORKQ7KNy7IloIl/0HjDgTnpyTdBbJE8tO5Ptnfp+VH63k9tdup6mlKcOFikhv\npXDvqmgRXPG74CEfj18TPIf14P52zT53/Oe4adJNPP/u81zx7BWs27EuC8WKSG+jcD8aBX3hynkw\n5VpYeDfcNwM2v9mu2bXjr+XuGXdTv6+eS5+5lN++9VtNNCYi3UrhfrQKiuGCH8EVj8Geerjvs/Da\nXdDaelizzwz7DPNmzqO6oprvvf49bnzxRhr2N2SpaBEJO4V7uow5B65fCFXnwILb4aGZsOODw5oM\nKhrEPWfdwy1TbuG1D1/jL576C17b+FqWChaRMFO4p1PfQXDpwzDzbvhwGdx7Bqx87LAmEYtw5dgr\neeSCRygtLGXOf83hh4t+qNE0IpJWCvd0M4NJX4Y5rwYP/nji2uCC677thzU7sexEHrngES4/6XIe\nXvMwVzx7BW9vfztLRYtI2OhhHd2ppRle+zG8/C/Qpz9MuAxO+xJUjD+s2St1r3D7a7fTsL+BaUOm\ncfGYi5kxbAbRvGiWCheRnirVh3Uo3DPhw+Xw2k/grWeD+WmGnBrMNDl+NhSXAdCwv4HH1j7GvHfm\nsWnPJsoKy5g1ehazq2Yz/JjhWf4PEJGeQuHeE+1tgDceh2W/hs0rgwnITroATrsSTvgsRPJoaW1h\n4aaFPP7247z8wcu0eAvTKqYx+8TZnDXsLJ3Ni/RyaQ13MzsP+CmQB9zv7v/SZnsf4CFgMrANuNTd\n3+ton70y3ONtWgnLfwMrH4V9DXDMUDh5Jhw3Mei2GTSG+gM7eLL2Sea9M4+NuzdSVljGWcPPYtyg\ncZxcdjKjS0Yr7EV6mbSFu5nlAW8DZwN1wGLgcndfHdfmemCCu88xs8uAL7r7pR3tt9eH+yHNB2Dt\n80HQv/sKNMfucs0rgGNPhorxtB47jj/1yeex7StYuKWGPQf3AJAfyaeqpIqxA8cyduBYTi47marS\nKgrzC7P4HyQi3Smd4f4p4Nvufm7s/W0A7v6DuDYvxNosNLN8YDNQ7h3sXOGeQEszbKuFzW8E3Tab\n3wheez/6uElr32P5oF8Za4qKWR2N8JY1s7plN42twbw1eUSoKCyjrGAAJX0GUNqnhJLCUkoLyygp\nGkRpcTklRWX0i/YjmhelIFJAQV7BJz/zCsiPaCZokZ4q1XBP5W/xUCD+bpw6YFqyNu7ebGaNwEDg\nIyR1efnB8MljT4IJFwfr3GH3liDkN60gsuN9jt/bwPF7Gzhv+zbYuw3f18DmiLG6TwGrCwr4MLqL\n7ZEIDXkR1kfy2J4XYW8k9VGvEXcKgHw3jGC8bPzr0Lq8uM9Y7BUsW9wykOBXvHXyPqEUGplmdZAc\n8MXBp3P1BXO79TsyeopmZtcB1wEMH64RICkxg/4Vwavq7MRNWlsZsn8HQ/Y2cNbej+DAbmg5EHTx\nNB+A5gMcaNrDjqZGtjftoqFpJ/taD9DU2kKTN9PU2kyTH1puoclbOOitNNOK47S4B0vutOA40IrT\n6sGyx9YBH78n7n1b3mZtuvJY0/VIrhjYd3C3f0cq4b4RGBb3vjK2LlGbuli3zACCC6uHcfe5wFwI\numW6UrAkEIkEQyqLy4DRCZv0AQbHXiISfqn8W30xUGVmI82sALgMmN+mzXzg6tjybODFjvrbRUSk\ne3V65h7rQ/9r4AWCbtZfuPsqM7sDqHH3+cADwK/NrBZoIPgFICIiWZJSn7u7Pwc812bdt+KW9wMX\np7c0ERHpKk0cJiISQgp3EZEQUriLiISQwl1EJIQU7iIiIZS1KX/NrB54v4sfH0TuTW2gmjMj12rO\ntXpBNWdKspqPd/fyzj6ctXA/GmZWk8rEOT2Jas6MXKs51+oF1ZwpR1uzumVEREJI4S4iEkK5Gu7d\nO1dm91DNmZFrNedavaCaM+Woas7JPncREelYrp65i4hIB3Iu3M3sPDNba2a1ZnZrtutJhZm9Z2Zv\nmNlyM+uRzxY0s1+Y2VYzezNuXZmZLTCzd2I/S7NZY7wk9X7bzDbGjvNyMzs/mzW2ZWbDzOwlM1tt\nZqvM7KbY+p58nJPV3COPtZkVmtkiM1sRq/c7sfUjzez1WG78LjZ9eY/QQc2/MrN3447xxCPasbvn\nzItgyuF1wAlAAbACGJvtulKo+z1gULbr6KTGTwOTgDfj1v0rcGts+Vbgh9mus5N6vw38XbZr66Dm\nIcCk2HJ/ggfPj+3hxzlZzT3yWBM8jLFfbDkKvA6cDjwKXBZb/3Pgr7Jdawo1/wqY3dX95tqZ+1Sg\n1t3Xu3sT8FtgVpZrCgV3f4VgLv54s4AHY8sPAl/IaFEdSFJvj+bum9x9aWx5F7CG4PnDPfk4J6u5\nR/LA7tjbaOzlwAzg8dj6nnaMk9V8VHIt3BM9rLvH/kGL48AfzGxJ7DmyuWKwu2+KLW8mN57S99dm\ntjLWbdNjujfaMrMRwGkEZ2k5cZzb1Aw99FibWZ6ZLQe2AgsI/rW/w92bY016XG60rdndDx3j78WO\n8Y/NrM+R7DPXwj1Xnenuk4DPAzeY2aezXdCR8uDfjD19aNW9wChgIrAJ+FF2y0nMzPoB84Cb3X1n\n/LaeepwT1Nxjj7W7t7j7RILnPU8FTspySZ1qW7OZjQNuI6h9ClAG3HIk+8y1cE/lYd09jrtvjP3c\nCvwnwR+4XLDFzIYAxH5uzXI9HXL3LbG/JK3AffTA42xmUYKQ/I27PxFb3aOPc6Kac+FYu/sO4CXg\nU0CJmR168lyPzY24ms+LdYnbfgoKAAABN0lEQVS5ux8AfskRHuNcC/dUHtbdo5hZXzPrf2gZOAd4\ns+NP9RjxDz6/Gngqi7V06lBAxnyRHnaczcwInje8xt3vjNvUY49zspp76rE2s3IzK4ktFwFnE1wn\neAmYHWvW045xoprfivuFbwTXCI7oGOfcTUyxIVc/4ZOHdX8vyyV1yMxOIDhbh+CZtf/RE2s2s0eA\n6QQz0W0B/gl4kmCUwXCCGTwvcfcecREzSb3TCboJnGCE0l/G9WVnnZmdCfwReANoja3+B4I+7J56\nnJPVfDk98Fib2QSCC6Z5BCevj7r7HbG/h78l6N5YBlwZOyPOug5qfhEoJxhNsxyYE3fhtfP95lq4\ni4hI53KtW0ZERFKgcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhP4/lyADI/D8\nqhgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}