Chapter5 - Generalizing Gradient Descent - Learning Multiple Weights at a Time.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Descent Learning with Multiple Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights:[0.1119, 0.20091, -0.09832]\n",
      "Weight Deltas:[-1.189999999999999, -0.09099999999999994, -0.16799999999999987]\n"
     ]
    }
   ],
   "source": [
    "def w_sum(a,b):\n",
    "    assert(len(a) == len(b))\n",
    "    output = 0\n",
    "\n",
    "    for i in range(len(a)):\n",
    "        output += (a[i] * b[i])\n",
    "\n",
    "    return output\n",
    "\n",
    "weights = [0.1, 0.2, -.1] \n",
    "\n",
    "def neural_network(input,weights):\n",
    "    pred = w_sum(input,weights)\n",
    "    return pred\n",
    "\n",
    "toes =  [8.5, 9.5, 9.9, 9.0]\n",
    "wlrec = [0.65, 0.8, 0.8, 0.9]\n",
    "nfans = [1.2, 1.3, 0.5, 1.0]\n",
    "\n",
    "win_or_lose_binary = [1, 1, 0, 1]\n",
    "\n",
    "true = win_or_lose_binary[0]\n",
    "\n",
    "# Input corresponds to every entry\n",
    "# for the first game of the season.\n",
    "\n",
    "input = [toes[0],wlrec[0],nfans[0]]\n",
    "\n",
    "pred = neural_network(input,weights)\n",
    "error = (pred - true) ** 2\n",
    "delta = pred - true\n",
    "\n",
    "def ele_mul(number,vector):\n",
    "    output = [0,0,0]\n",
    "\n",
    "    assert(len(output) == len(vector))\n",
    "\n",
    "    for i in range(len(vector)):\n",
    "        output[i] = number * vector[i]\n",
    "\n",
    "    return output\n",
    "\n",
    "  \n",
    "\n",
    "alpha = 0.01\n",
    "\n",
    "for i in range(len(weights)):\n",
    "    weights[i] -= alpha * weight_deltas[i]\n",
    "    \n",
    "print(\"Weights:\" + str(weights))\n",
    "print(\"Weight Deltas:\" + str(weight_deltas))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's Watch Several Steps of Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:1\n",
      "Pred:0.8600000000000001\n",
      "Error:0.01959999999999997\n",
      "Delta:-0.1399999999999999\n",
      "Weights:[0.1, 0.2, -0.1]\n",
      "Weight_Deltas:\n",
      "[-1.189999999999999, -0.09099999999999994, -0.16799999999999987]\n",
      "\n",
      "Iteration:2\n",
      "Pred:0.9637574999999999\n",
      "Error:0.0013135188062500048\n",
      "Delta:-0.036242500000000066\n",
      "Weights:[0.1119, 0.20091, -0.09832]\n",
      "Weight_Deltas:\n",
      "[-0.30806125000000056, -0.023557625000000044, -0.04349100000000008]\n",
      "\n",
      "Iteration:3\n",
      "Pred:0.9906177228125002\n",
      "Error:8.802712522307997e-05\n",
      "Delta:-0.009382277187499843\n",
      "Weights:[0.11498061250000001, 0.20114557625, -0.09788509000000001]\n",
      "Weight_Deltas:\n",
      "[-0.07974935609374867, -0.006098480171874899, -0.011258732624999811]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def neural_network(input, weights):\n",
    "  out = 0\n",
    "  for i in range(len(input)):\n",
    "    out += (input[i] * weights[i])\n",
    "  return out\n",
    "\n",
    "def ele_mul(scalar, vector):\n",
    "  out = [0,0,0]\n",
    "  for i in range(len(out)):\n",
    "    out[i] = vector[i] * scalar\n",
    "  return out\n",
    "\n",
    "toes =  [8.5, 9.5, 9.9, 9.0]\n",
    "wlrec = [0.65, 0.8, 0.8, 0.9]\n",
    "nfans = [1.2, 1.3, 0.5, 1.0]\n",
    "\n",
    "win_or_lose_binary = [1, 1, 0, 1]\n",
    "true = win_or_lose_binary[0]\n",
    "\n",
    "alpha = 0.01\n",
    "weights = [0.1, 0.2, -.1]\n",
    "input = [toes[0],wlrec[0],nfans[0]]\n",
    "\n",
    "for iter in range(3):\n",
    "\n",
    "  pred = neural_network(input,weights)\n",
    "\n",
    "  error = (pred - true) ** 2\n",
    "  delta = pred - true\n",
    "\n",
    "  weight_deltas=ele_mul(delta,input)\n",
    "\n",
    "  print(\"Iteration:\" + str(iter+1))\n",
    "  print(\"Pred:\" + str(pred))\n",
    "  print(\"Error:\" + str(error))\n",
    "  print(\"Delta:\" + str(delta))\n",
    "  print(\"Weights:\" + str(weights))\n",
    "  print(\"Weight_Deltas:\")\n",
    "  print(str(weight_deltas))\n",
    "  print(\n",
    "  )\n",
    "\n",
    "  for i in range(len(weights)):\n",
    "    weights[i]-=alpha*weight_deltas[i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Freezing One Weight - What Does It Do?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:1\n",
      "Pred:0.8600000000000001\n",
      "Error:0.01959999999999997\n",
      "Delta:-0.1399999999999999\n",
      "Weights:[0.1, 0.2, -0.1]\n",
      "Weight_Deltas:\n",
      "[0, -0.09099999999999994, -0.16799999999999987]\n",
      "\n",
      "Iteration:2\n",
      "Pred:0.9382250000000001\n",
      "Error:0.003816150624999989\n",
      "Delta:-0.06177499999999991\n",
      "Weights:[0.1, 0.2273, -0.04960000000000005]\n",
      "Weight_Deltas:\n",
      "[0, -0.040153749999999946, -0.07412999999999989]\n",
      "\n",
      "Iteration:3\n",
      "Pred:0.97274178125\n",
      "Error:0.000743010489422852\n",
      "Delta:-0.027258218750000007\n",
      "Weights:[0.1, 0.239346125, -0.02736100000000008]\n",
      "Weight_Deltas:\n",
      "[0, -0.017717842187500006, -0.032709862500000006]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def neural_network(input, weights):\n",
    "  out = 0\n",
    "  for i in range(len(input)):\n",
    "    out += (input[i] * weights[i])\n",
    "  return out\n",
    "\n",
    "def ele_mul(scalar, vector):\n",
    "  out = [0,0,0]\n",
    "  for i in range(len(out)):\n",
    "    out[i] = vector[i] * scalar\n",
    "  return out\n",
    "\n",
    "toes =  [8.5, 9.5, 9.9, 9.0]\n",
    "wlrec = [0.65, 0.8, 0.8, 0.9]\n",
    "nfans = [1.2, 1.3, 0.5, 1.0]\n",
    "\n",
    "win_or_lose_binary = [1, 1, 0, 1]\n",
    "true = win_or_lose_binary[0]\n",
    "\n",
    "alpha = 0.3\n",
    "weights = [0.1, 0.2, -.1]\n",
    "input = [toes[0],wlrec[0],nfans[0]]\n",
    "\n",
    "for iter in range(3):\n",
    "\n",
    "  pred = neural_network(input,weights)\n",
    "\n",
    "  error = (pred - true) ** 2\n",
    "  delta = pred - true\n",
    "\n",
    "  weight_deltas=ele_mul(delta,input)\n",
    "  weight_deltas[0] = 0\n",
    "\n",
    "  print(\"Iteration:\" + str(iter+1))\n",
    "  print(\"Pred:\" + str(pred))\n",
    "  print(\"Error:\" + str(error))\n",
    "  print(\"Delta:\" + str(delta))\n",
    "  print(\"Weights:\" + str(weights))\n",
    "  print(\"Weight_Deltas:\")\n",
    "  print(str(weight_deltas))\n",
    "  print(\n",
    "  )\n",
    "\n",
    "  for i in range(len(weights)):\n",
    "    weights[i]-=alpha*weight_deltas[i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Descent Learning with Multiple Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights:[0.293825, 0.25655, 0.868475]\n",
      "Weight Deltas:[0.061750000000000006, -0.5655, 0.3152500000000001]\n"
     ]
    }
   ],
   "source": [
    "# Instead of predicting just \n",
    "# whether the team won or lost, \n",
    "# now we're also predicting whether\n",
    "# they are happy/sad AND the\n",
    "# percentage of the team that is\n",
    "# hurt. We are making this\n",
    "# prediction using only\n",
    "# the current win/loss record.\n",
    "\n",
    "weights = [0.3, 0.2, 0.9] \n",
    "\n",
    "def neural_network(input, weights):\n",
    "    pred = ele_mul(input,weights)\n",
    "    return pred\n",
    "\n",
    "wlrec = [0.65, 1.0, 1.0, 0.9]\n",
    "\n",
    "hurt  = [0.1, 0.0, 0.0, 0.1]\n",
    "win   = [  1,   1,   0,   1]\n",
    "sad   = [0.1, 0.0, 0.1, 0.2]\n",
    "\n",
    "input = wlrec[0]\n",
    "true = [hurt[0], win[0], sad[0]]\n",
    "\n",
    "pred = neural_network(input,weights)\n",
    "\n",
    "error = [0, 0, 0] \n",
    "delta = [0, 0, 0]\n",
    "\n",
    "for i in range(len(true)):\n",
    "    error[i] = (pred[i] - true[i]) ** 2\n",
    "    delta[i] = pred[i] - true[i]\n",
    "    \n",
    "def scalar_ele_mul(number,vector):\n",
    "    output = [0,0,0]\n",
    "\n",
    "    assert(len(output) == len(vector))\n",
    "\n",
    "    for i in range(len(vector)):\n",
    "        output[i] = number * vector[i]\n",
    "\n",
    "    return output\n",
    "\n",
    "weight_deltas = scalar_ele_mul(input,delta)\n",
    "\n",
    "alpha = 0.1\n",
    "\n",
    "for i in range(len(weights)):\n",
    "    weights[i] -= (weight_deltas[i] * alpha)\n",
    "    \n",
    "print(\"Weights:\" + str(weights))\n",
    "print(\"Weight Deltas:\" + str(weight_deltas))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Descent with Multiple Inputs & Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "            #toes %win #fans\n",
    "weights = [ [0.1, 0.1, -0.3],#hurt?\n",
    "            [0.1, 0.2, 0.0], #win?\n",
    "            [0.0, 1.3, 0.1] ]#sad?\n",
    "\n",
    "def w_sum(a,b):\n",
    "    assert(len(a) == len(b))\n",
    "    output = 0\n",
    "\n",
    "    for i in range(len(a)):\n",
    "        output += (a[i] * b[i])\n",
    "\n",
    "    return output\n",
    "\n",
    "def vect_mat_mul(vect,matrix):\n",
    "    assert(len(vect) == len(matrix))\n",
    "    output = [0,0,0]\n",
    "    for i in range(len(vect)):\n",
    "        output[i] = w_sum(vect,matrix[i])\n",
    "    return output\n",
    "\n",
    "def neural_network(input, weights):\n",
    "    pred = vect_mat_mul(input,weights)\n",
    "    return pred\n",
    "\n",
    "toes  = [8.5, 9.5, 9.9, 9.0]\n",
    "wlrec = [0.65,0.8, 0.8, 0.9]\n",
    "nfans = [1.2, 1.3, 0.5, 1.0]\n",
    "\n",
    "hurt  = [0.1, 0.0, 0.0, 0.1]\n",
    "win   = [  1,   1,   0,   1]\n",
    "sad   = [0.1, 0.0, 0.1, 0.2]\n",
    "\n",
    "alpha = 0.01\n",
    "\n",
    "input = [toes[0],wlrec[0],nfans[0]]\n",
    "true  = [hurt[0], win[0], sad[0]]\n",
    "\n",
    "pred = neural_network(input,weights)\n",
    "\n",
    "error = [0, 0, 0] \n",
    "delta = [0, 0, 0]\n",
    "\n",
    "for i in range(len(true)):\n",
    "    error[i] = (pred[i] - true[i]) ** 2\n",
    "    delta[i] = pred[i] - true[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def outer_prod(a, b):\n",
    "    \n",
    "    # just a matrix of zeros\n",
    "    out = np.zeros((len(a), len(b)))\n",
    "\n",
    "    for i in range(len(a)):\n",
    "        for j in range(len(b)):\n",
    "            out[i][j] = a[i] * b[j]\n",
    "    return out\n",
    "\n",
    "weight_deltas = outer_prod(delta,input)\n",
    "\n",
    "for i in range(len(weights)):\n",
    "    for j in range(len(weights[0])):\n",
    "        weights[i][j] -= alpha * weight_deltas[i][j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0.061324999999999998, 0.097042500000000004, -0.30546000000000001],\n",
       " [0.1017, 0.20013, 0.00023999999999999887],\n",
       " [-0.073525000000000007, 1.2943775, 0.089620000000000005]]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}