Logistic Regression 101.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "from builtins import range\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "N = 100\n",
    "D = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.random.randn(N,D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.38944384e+00,  3.15558742e-01],\n",
       "       [ 1.18937519e+00,  1.40441491e+00],\n",
       "       [-1.48284111e+00, -2.30893793e+00],\n",
       "       [ 1.90735584e-01,  2.85248497e-01],\n",
       "       [-5.34594336e-01,  1.03433020e+00],\n",
       "       [-5.86380782e-01, -7.32276127e-01],\n",
       "       [-2.01839142e+00, -7.16616081e-01],\n",
       "       [ 4.14648474e-02, -2.03888467e+00],\n",
       "       [ 1.41031544e-02,  1.17967243e+00],\n",
       "       [-7.66877981e-01,  2.06079906e-01],\n",
       "       [ 4.29022542e-01,  1.07119084e+00],\n",
       "       [ 3.51330854e-01, -1.41639619e-01],\n",
       "       [-6.87592008e-01, -1.45126160e+00],\n",
       "       [ 1.20961105e+00,  9.10971109e-01],\n",
       "       [-1.89649282e+00, -5.47009099e-01],\n",
       "       [ 2.67821065e-01, -1.27903186e+00],\n",
       "       [ 5.84928027e-01,  4.16495496e-01],\n",
       "       [-1.69335813e+00,  5.86033149e-01],\n",
       "       [ 6.72711342e-01, -2.07218790e-02],\n",
       "       [-6.73441619e-01,  9.55234952e-01],\n",
       "       [-2.32744698e-01,  4.78084506e-01],\n",
       "       [-1.24573499e+00,  5.26029927e-01],\n",
       "       [-6.42541449e-01, -2.96285475e-01],\n",
       "       [ 1.02027762e-01,  1.24278033e+00],\n",
       "       [ 1.26737016e+00,  3.53135544e-01],\n",
       "       [ 1.38913703e+00,  5.45446557e-01],\n",
       "       [-2.45251220e-01,  1.19183812e+00],\n",
       "       [ 3.16333383e-01,  3.13627349e-02],\n",
       "       [-1.23161113e+00,  6.61612466e-01],\n",
       "       [ 2.73693557e-01,  2.03357406e+00],\n",
       "       [-1.17128040e+00,  2.32399292e+00],\n",
       "       [ 1.28196026e+00, -2.79134411e-01],\n",
       "       [ 5.49994268e-01,  2.45119879e-01],\n",
       "       [ 1.33508738e+00, -1.26567476e+00],\n",
       "       [-2.97163858e-02, -1.30388251e+00],\n",
       "       [-7.57160488e-01, -1.64335464e-01],\n",
       "       [-3.46741465e-01,  7.51320174e-01],\n",
       "       [ 3.97846360e-01,  1.36885651e+00],\n",
       "       [-1.37010133e-01,  1.64697952e+00],\n",
       "       [-2.10072621e+00,  4.78605548e-01],\n",
       "       [-1.40998099e-01,  1.20136034e-01],\n",
       "       [-5.50272740e-01,  1.45414917e+00],\n",
       "       [-1.54842226e+00, -5.48468826e-01],\n",
       "       [-7.39883166e-02, -1.23683344e-02],\n",
       "       [ 7.01199846e-01,  5.90539275e-01],\n",
       "       [ 1.08809685e+00,  1.16103376e+00],\n",
       "       [ 5.46408005e-01, -5.25897592e-01],\n",
       "       [-7.07535012e-01,  1.21514920e+00],\n",
       "       [ 1.63363863e-01, -1.42792654e-01],\n",
       "       [ 6.31937528e-01,  1.40756807e+00],\n",
       "       [-2.99644352e-01, -3.09737129e-02],\n",
       "       [-6.99208720e-01, -1.59084057e+00],\n",
       "       [ 1.83466709e-03,  1.62503090e+00],\n",
       "       [-1.46084626e-01,  6.48740586e-01],\n",
       "       [-1.32613373e-02, -2.78027247e+00],\n",
       "       [ 1.87399348e-01,  5.21803552e-01],\n",
       "       [ 1.76134654e-01,  1.27791511e+00],\n",
       "       [-5.25684815e-01,  4.12619850e-01],\n",
       "       [ 4.30732605e-01,  1.00761605e+00],\n",
       "       [-1.05109850e+00,  2.11131729e-01],\n",
       "       [-1.32382884e-01,  7.42028513e-01],\n",
       "       [-1.43690824e+00, -1.24675556e-01],\n",
       "       [-3.64446336e-02, -6.44893660e-01],\n",
       "       [-3.77782693e-01,  8.79239106e-01],\n",
       "       [ 1.03536377e+00,  2.59297619e+00],\n",
       "       [ 1.42546763e-01,  9.44871407e-01],\n",
       "       [ 5.88209221e-01, -2.13744929e-01],\n",
       "       [ 2.87266140e-01, -4.51887632e-01],\n",
       "       [ 1.50551503e+00, -1.50545352e+00],\n",
       "       [-8.77301003e-02,  1.44475801e+00],\n",
       "       [-6.63929564e-01, -1.51833964e+00],\n",
       "       [ 1.51564425e+00,  2.66287837e-01],\n",
       "       [-1.30339929e+00, -1.31923684e+00],\n",
       "       [-1.11312681e+00,  1.28477716e+00],\n",
       "       [-5.96716094e-01,  2.19274558e-01],\n",
       "       [-1.81856830e+00, -3.32593154e-01],\n",
       "       [ 1.09079851e+00, -8.02211083e-02],\n",
       "       [ 2.76396457e+00, -4.84966791e-01],\n",
       "       [ 2.59914309e+00, -9.45114344e-01],\n",
       "       [ 1.91988433e+00,  4.02825249e-01],\n",
       "       [ 1.95194624e-01, -4.42539375e-01],\n",
       "       [ 1.50993317e+00, -5.51765973e-01],\n",
       "       [-8.64580991e-01,  4.54994105e-01],\n",
       "       [-5.60349839e-01, -2.95579494e-01],\n",
       "       [-1.57983614e+00, -2.79599774e-01],\n",
       "       [ 6.35108056e-01,  2.26368483e-01],\n",
       "       [-1.12747245e-01,  2.28940579e-01],\n",
       "       [-4.09754410e-01, -1.97425291e+00],\n",
       "       [ 1.06358411e+00, -1.54432707e+00],\n",
       "       [-1.95320551e+00,  1.90557465e+00],\n",
       "       [ 2.77539034e-01,  3.68815875e-02],\n",
       "       [ 1.04376377e+00, -4.26315411e-02],\n",
       "       [-6.81869999e-01, -5.68929571e-01],\n",
       "       [ 4.47603165e-01,  1.14903707e+00],\n",
       "       [-1.44595215e-01, -1.88961846e-01],\n",
       "       [-2.42974267e-01,  1.47031526e-01],\n",
       "       [-9.81242613e-02, -7.68827076e-01],\n",
       "       [ 3.01404494e-01, -4.04741881e-01],\n",
       "       [-4.12351331e-01, -1.12478606e+00],\n",
       "       [ 1.34044792e-01, -5.12187798e-02]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# center the first 50 points at (-2,-2)\n",
    "X[:50,:] = X[:50,:] - 2*np.ones((50,D))\n",
    "\n",
    "# center the last 50 points at (2, 2)\n",
    "X[50:,:] = X[50:,:] + 2*np.ones((50,D))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-3.38944384, -1.68444126],\n",
       "       [-0.81062481, -0.59558509],\n",
       "       [-3.48284111, -4.30893793],\n",
       "       [-1.80926442, -1.7147515 ],\n",
       "       [-2.53459434, -0.9656698 ],\n",
       "       [-2.58638078, -2.73227613],\n",
       "       [-4.01839142, -2.71661608],\n",
       "       [-1.95853515, -4.03888467],\n",
       "       [-1.98589685, -0.82032757],\n",
       "       [-2.76687798, -1.79392009],\n",
       "       [-1.57097746, -0.92880916],\n",
       "       [-1.64866915, -2.14163962],\n",
       "       [-2.68759201, -3.4512616 ],\n",
       "       [-0.79038895, -1.08902889],\n",
       "       [-3.89649282, -2.5470091 ],\n",
       "       [-1.73217894, -3.27903186],\n",
       "       [-1.41507197, -1.5835045 ],\n",
       "       [-3.69335813, -1.41396685],\n",
       "       [-1.32728866, -2.02072188],\n",
       "       [-2.67344162, -1.04476505],\n",
       "       [-2.2327447 , -1.52191549],\n",
       "       [-3.24573499, -1.47397007],\n",
       "       [-2.64254145, -2.29628548],\n",
       "       [-1.89797224, -0.75721967],\n",
       "       [-0.73262984, -1.64686446],\n",
       "       [-0.61086297, -1.45455344],\n",
       "       [-2.24525122, -0.80816188],\n",
       "       [-1.68366662, -1.96863727],\n",
       "       [-3.23161113, -1.33838753],\n",
       "       [-1.72630644,  0.03357406],\n",
       "       [-3.1712804 ,  0.32399292],\n",
       "       [-0.71803974, -2.27913441],\n",
       "       [-1.45000573, -1.75488012],\n",
       "       [-0.66491262, -3.26567476],\n",
       "       [-2.02971639, -3.30388251],\n",
       "       [-2.75716049, -2.16433546],\n",
       "       [-2.34674147, -1.24867983],\n",
       "       [-1.60215364, -0.63114349],\n",
       "       [-2.13701013, -0.35302048],\n",
       "       [-4.10072621, -1.52139445],\n",
       "       [-2.1409981 , -1.87986397],\n",
       "       [-2.55027274, -0.54585083],\n",
       "       [-3.54842226, -2.54846883],\n",
       "       [-2.07398832, -2.01236833],\n",
       "       [-1.29880015, -1.40946072],\n",
       "       [-0.91190315, -0.83896624],\n",
       "       [-1.45359199, -2.52589759],\n",
       "       [-2.70753501, -0.7848508 ],\n",
       "       [-1.83663614, -2.14279265],\n",
       "       [-1.36806247, -0.59243193],\n",
       "       [ 1.70035565,  1.96902629],\n",
       "       [ 1.30079128,  0.40915943],\n",
       "       [ 2.00183467,  3.6250309 ],\n",
       "       [ 1.85391537,  2.64874059],\n",
       "       [ 1.98673866, -0.78027247],\n",
       "       [ 2.18739935,  2.52180355],\n",
       "       [ 2.17613465,  3.27791511],\n",
       "       [ 1.47431518,  2.41261985],\n",
       "       [ 2.43073261,  3.00761605],\n",
       "       [ 0.9489015 ,  2.21113173],\n",
       "       [ 1.86761712,  2.74202851],\n",
       "       [ 0.56309176,  1.87532444],\n",
       "       [ 1.96355537,  1.35510634],\n",
       "       [ 1.62221731,  2.87923911],\n",
       "       [ 3.03536377,  4.59297619],\n",
       "       [ 2.14254676,  2.94487141],\n",
       "       [ 2.58820922,  1.78625507],\n",
       "       [ 2.28726614,  1.54811237],\n",
       "       [ 3.50551503,  0.49454648],\n",
       "       [ 1.9122699 ,  3.44475801],\n",
       "       [ 1.33607044,  0.48166036],\n",
       "       [ 3.51564425,  2.26628784],\n",
       "       [ 0.69660071,  0.68076316],\n",
       "       [ 0.88687319,  3.28477716],\n",
       "       [ 1.40328391,  2.21927456],\n",
       "       [ 0.1814317 ,  1.66740685],\n",
       "       [ 3.09079851,  1.91977889],\n",
       "       [ 4.76396457,  1.51503321],\n",
       "       [ 4.59914309,  1.05488566],\n",
       "       [ 3.91988433,  2.40282525],\n",
       "       [ 2.19519462,  1.55746062],\n",
       "       [ 3.50993317,  1.44823403],\n",
       "       [ 1.13541901,  2.4549941 ],\n",
       "       [ 1.43965016,  1.70442051],\n",
       "       [ 0.42016386,  1.72040023],\n",
       "       [ 2.63510806,  2.22636848],\n",
       "       [ 1.88725275,  2.22894058],\n",
       "       [ 1.59024559,  0.02574709],\n",
       "       [ 3.06358411,  0.45567293],\n",
       "       [ 0.04679449,  3.90557465],\n",
       "       [ 2.27753903,  2.03688159],\n",
       "       [ 3.04376377,  1.95736846],\n",
       "       [ 1.31813   ,  1.43107043],\n",
       "       [ 2.44760317,  3.14903707],\n",
       "       [ 1.85540479,  1.81103815],\n",
       "       [ 1.75702573,  2.14703153],\n",
       "       [ 1.90187574,  1.23117292],\n",
       "       [ 2.30140449,  1.59525812],\n",
       "       [ 1.58764867,  0.87521394],\n",
       "       [ 2.13404479,  1.94878122]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = np.array([0]*50 + [1]*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ones = np.ones((N, 1))\n",
    "#ones\n",
    "Xb = np.concatenate((ones, X), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        , -3.38944384, -1.68444126],\n",
       "       [ 1.        , -0.81062481, -0.59558509],\n",
       "       [ 1.        , -3.48284111, -4.30893793],\n",
       "       [ 1.        , -1.80926442, -1.7147515 ],\n",
       "       [ 1.        , -2.53459434, -0.9656698 ],\n",
       "       [ 1.        , -2.58638078, -2.73227613],\n",
       "       [ 1.        , -4.01839142, -2.71661608],\n",
       "       [ 1.        , -1.95853515, -4.03888467],\n",
       "       [ 1.        , -1.98589685, -0.82032757],\n",
       "       [ 1.        , -2.76687798, -1.79392009],\n",
       "       [ 1.        , -1.57097746, -0.92880916],\n",
       "       [ 1.        , -1.64866915, -2.14163962],\n",
       "       [ 1.        , -2.68759201, -3.4512616 ],\n",
       "       [ 1.        , -0.79038895, -1.08902889],\n",
       "       [ 1.        , -3.89649282, -2.5470091 ],\n",
       "       [ 1.        , -1.73217894, -3.27903186],\n",
       "       [ 1.        , -1.41507197, -1.5835045 ],\n",
       "       [ 1.        , -3.69335813, -1.41396685],\n",
       "       [ 1.        , -1.32728866, -2.02072188],\n",
       "       [ 1.        , -2.67344162, -1.04476505],\n",
       "       [ 1.        , -2.2327447 , -1.52191549],\n",
       "       [ 1.        , -3.24573499, -1.47397007],\n",
       "       [ 1.        , -2.64254145, -2.29628548],\n",
       "       [ 1.        , -1.89797224, -0.75721967],\n",
       "       [ 1.        , -0.73262984, -1.64686446],\n",
       "       [ 1.        , -0.61086297, -1.45455344],\n",
       "       [ 1.        , -2.24525122, -0.80816188],\n",
       "       [ 1.        , -1.68366662, -1.96863727],\n",
       "       [ 1.        , -3.23161113, -1.33838753],\n",
       "       [ 1.        , -1.72630644,  0.03357406],\n",
       "       [ 1.        , -3.1712804 ,  0.32399292],\n",
       "       [ 1.        , -0.71803974, -2.27913441],\n",
       "       [ 1.        , -1.45000573, -1.75488012],\n",
       "       [ 1.        , -0.66491262, -3.26567476],\n",
       "       [ 1.        , -2.02971639, -3.30388251],\n",
       "       [ 1.        , -2.75716049, -2.16433546],\n",
       "       [ 1.        , -2.34674147, -1.24867983],\n",
       "       [ 1.        , -1.60215364, -0.63114349],\n",
       "       [ 1.        , -2.13701013, -0.35302048],\n",
       "       [ 1.        , -4.10072621, -1.52139445],\n",
       "       [ 1.        , -2.1409981 , -1.87986397],\n",
       "       [ 1.        , -2.55027274, -0.54585083],\n",
       "       [ 1.        , -3.54842226, -2.54846883],\n",
       "       [ 1.        , -2.07398832, -2.01236833],\n",
       "       [ 1.        , -1.29880015, -1.40946072],\n",
       "       [ 1.        , -0.91190315, -0.83896624],\n",
       "       [ 1.        , -1.45359199, -2.52589759],\n",
       "       [ 1.        , -2.70753501, -0.7848508 ],\n",
       "       [ 1.        , -1.83663614, -2.14279265],\n",
       "       [ 1.        , -1.36806247, -0.59243193],\n",
       "       [ 1.        ,  1.70035565,  1.96902629],\n",
       "       [ 1.        ,  1.30079128,  0.40915943],\n",
       "       [ 1.        ,  2.00183467,  3.6250309 ],\n",
       "       [ 1.        ,  1.85391537,  2.64874059],\n",
       "       [ 1.        ,  1.98673866, -0.78027247],\n",
       "       [ 1.        ,  2.18739935,  2.52180355],\n",
       "       [ 1.        ,  2.17613465,  3.27791511],\n",
       "       [ 1.        ,  1.47431518,  2.41261985],\n",
       "       [ 1.        ,  2.43073261,  3.00761605],\n",
       "       [ 1.        ,  0.9489015 ,  2.21113173],\n",
       "       [ 1.        ,  1.86761712,  2.74202851],\n",
       "       [ 1.        ,  0.56309176,  1.87532444],\n",
       "       [ 1.        ,  1.96355537,  1.35510634],\n",
       "       [ 1.        ,  1.62221731,  2.87923911],\n",
       "       [ 1.        ,  3.03536377,  4.59297619],\n",
       "       [ 1.        ,  2.14254676,  2.94487141],\n",
       "       [ 1.        ,  2.58820922,  1.78625507],\n",
       "       [ 1.        ,  2.28726614,  1.54811237],\n",
       "       [ 1.        ,  3.50551503,  0.49454648],\n",
       "       [ 1.        ,  1.9122699 ,  3.44475801],\n",
       "       [ 1.        ,  1.33607044,  0.48166036],\n",
       "       [ 1.        ,  3.51564425,  2.26628784],\n",
       "       [ 1.        ,  0.69660071,  0.68076316],\n",
       "       [ 1.        ,  0.88687319,  3.28477716],\n",
       "       [ 1.        ,  1.40328391,  2.21927456],\n",
       "       [ 1.        ,  0.1814317 ,  1.66740685],\n",
       "       [ 1.        ,  3.09079851,  1.91977889],\n",
       "       [ 1.        ,  4.76396457,  1.51503321],\n",
       "       [ 1.        ,  4.59914309,  1.05488566],\n",
       "       [ 1.        ,  3.91988433,  2.40282525],\n",
       "       [ 1.        ,  2.19519462,  1.55746062],\n",
       "       [ 1.        ,  3.50993317,  1.44823403],\n",
       "       [ 1.        ,  1.13541901,  2.4549941 ],\n",
       "       [ 1.        ,  1.43965016,  1.70442051],\n",
       "       [ 1.        ,  0.42016386,  1.72040023],\n",
       "       [ 1.        ,  2.63510806,  2.22636848],\n",
       "       [ 1.        ,  1.88725275,  2.22894058],\n",
       "       [ 1.        ,  1.59024559,  0.02574709],\n",
       "       [ 1.        ,  3.06358411,  0.45567293],\n",
       "       [ 1.        ,  0.04679449,  3.90557465],\n",
       "       [ 1.        ,  2.27753903,  2.03688159],\n",
       "       [ 1.        ,  3.04376377,  1.95736846],\n",
       "       [ 1.        ,  1.31813   ,  1.43107043],\n",
       "       [ 1.        ,  2.44760317,  3.14903707],\n",
       "       [ 1.        ,  1.85540479,  1.81103815],\n",
       "       [ 1.        ,  1.75702573,  2.14703153],\n",
       "       [ 1.        ,  1.90187574,  1.23117292],\n",
       "       [ 1.        ,  2.30140449,  1.59525812],\n",
       "       [ 1.        ,  1.58764867,  0.87521394],\n",
       "       [ 1.        ,  2.13404479,  1.94878122]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = np.random.randn(D + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.20340221,  0.3666083 , -0.57546686])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3,)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = Xb.dot(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1/(1 + np.exp(-z))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = sigmoid(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.18593239, 0.23905917, 0.49985258, 0.29319624, 0.17123574,\n",
       "       0.35910369, 0.24726449, 0.59936664, 0.18856261, 0.23407887,\n",
       "       0.22359521, 0.35999741, 0.4495203 , 0.29599033, 0.23754806,\n",
       "       0.51213281, 0.30769342, 0.14884017, 0.37118447, 0.17047276,\n",
       "       0.24119611, 0.17579588, 0.29927688, 0.18793873, 0.37185326,\n",
       "       0.35656549, 0.17343731, 0.33453092, 0.16548916, 0.13521659,\n",
       "       0.07226092, 0.4613087 , 0.3262685 , 0.60638048, 0.48844251,\n",
       "       0.27514033, 0.20666225, 0.1934787 , 0.14384732, 0.13808938,\n",
       "       0.28771443, 0.13892657, 0.26158941, 0.30880921, 0.29557585,\n",
       "       0.25828217, 0.42978442, 0.14876107, 0.34442822, 0.20358767,\n",
       "       0.15275597, 0.27648214, 0.0720513 , 0.11425423, 0.49349377,\n",
       "       0.13555645, 0.0917923 , 0.11392222, 0.11475563, 0.1064088 ,\n",
       "       0.10942141, 0.11143611, 0.22039512, 0.09401311, 0.0610128 ,\n",
       "       0.10787793, 0.21712942, 0.22170485, 0.44946157, 0.0769383 ,\n",
       "       0.27076055, 0.2281633 , 0.20754636, 0.05904766, 0.1228121 ,\n",
       "       0.10944476, 0.23594177, 0.4185461 , 0.46894754, 0.240656  ,\n",
       "       0.21502584, 0.32081209, 0.09975804, 0.16023885, 0.11512785,\n",
       "       0.17967499, 0.14255843, 0.34632751, 0.41520438, 0.0312564 ,\n",
       "       0.17644875, 0.22900575, 0.17599828, 0.10733496, 0.17287488,\n",
       "       0.14248432, 0.22887791, 0.2179409 , 0.24508249, 0.17617175])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(T, Y):\n",
    "    E = 0\n",
    "    for i in range(len(T)):\n",
    "        if T[i] == 1:\n",
    "            E -= np.log(Y[i])\n",
    "        else:\n",
    "            E -= np.log(1 - Y[i])\n",
    "    return E"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107.846819039413\n"
     ]
    }
   ],
   "source": [
    "print(cross_entropy(T, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = np.array([0, 4, 4])\n",
    "z = Xb.dot(w)\n",
    "Y = sigmoid(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02354590500196028\n"
     ]
    }
   ],
   "source": [
    "print(cross_entropy(T, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}