Logistic Regression. Is 8.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_submission.csv  test.csv  train.csv\r\n"
     ]
    }
   ],
   "source": [
    "%ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.00000</td>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.000000</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>42000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.456643</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.219286</td>\n",
       "      <td>0.117095</td>\n",
       "      <td>0.059024</td>\n",
       "      <td>0.02019</td>\n",
       "      <td>0.017238</td>\n",
       "      <td>0.002857</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.887730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6.312890</td>\n",
       "      <td>4.633819</td>\n",
       "      <td>3.274488</td>\n",
       "      <td>1.75987</td>\n",
       "      <td>1.894498</td>\n",
       "      <td>0.414264</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>253.000000</td>\n",
       "      <td>253.00000</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              label   pixel0   pixel1   pixel2   pixel3   pixel4   pixel5  \\\n",
       "count  42000.000000  42000.0  42000.0  42000.0  42000.0  42000.0  42000.0   \n",
       "mean       4.456643      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "std        2.887730      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "min        0.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "25%        2.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "50%        4.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "75%        7.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "max        9.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "\n",
       "        pixel6   pixel7   pixel8    ...         pixel774      pixel775  \\\n",
       "count  42000.0  42000.0  42000.0    ...     42000.000000  42000.000000   \n",
       "mean       0.0      0.0      0.0    ...         0.219286      0.117095   \n",
       "std        0.0      0.0      0.0    ...         6.312890      4.633819   \n",
       "min        0.0      0.0      0.0    ...         0.000000      0.000000   \n",
       "25%        0.0      0.0      0.0    ...         0.000000      0.000000   \n",
       "50%        0.0      0.0      0.0    ...         0.000000      0.000000   \n",
       "75%        0.0      0.0      0.0    ...         0.000000      0.000000   \n",
       "max        0.0      0.0      0.0    ...       254.000000    254.000000   \n",
       "\n",
       "           pixel776     pixel777      pixel778      pixel779  pixel780  \\\n",
       "count  42000.000000  42000.00000  42000.000000  42000.000000   42000.0   \n",
       "mean       0.059024      0.02019      0.017238      0.002857       0.0   \n",
       "std        3.274488      1.75987      1.894498      0.414264       0.0   \n",
       "min        0.000000      0.00000      0.000000      0.000000       0.0   \n",
       "25%        0.000000      0.00000      0.000000      0.000000       0.0   \n",
       "50%        0.000000      0.00000      0.000000      0.000000       0.0   \n",
       "75%        0.000000      0.00000      0.000000      0.000000       0.0   \n",
       "max      253.000000    253.00000    254.000000     62.000000       0.0   \n",
       "\n",
       "       pixel781  pixel782  pixel783  \n",
       "count   42000.0   42000.0   42000.0  \n",
       "mean        0.0       0.0       0.0  \n",
       "std         0.0       0.0       0.0  \n",
       "min         0.0       0.0       0.0  \n",
       "25%         0.0       0.0       0.0  \n",
       "50%         0.0       0.0       0.0  \n",
       "75%         0.0       0.0       0.0  \n",
       "max         0.0       0.0       0.0  \n",
       "\n",
       "[8 rows x 785 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First try - use logistic regression to classify digit 8, not digit 8\n",
    "is it neccesary to reduce dimensions? Let's try without it.\n",
    "I'm going to do following steps:\n",
    "- add new column \"is_8\" to data with value \"1\" if it's 8 and \"0\" if it's not;\n",
    "- shuffle data\n",
    "- split data into train and test sets\n",
    "- with train dataset:\n",
    " - prepare feaures, labels, initial weights, learning rate, number of iterations\n",
    " - run training\n",
    "- check predictions on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create target column\n",
    "df['pixel784'] = 1\n",
    "df['is_8'] = df['label'] == 8\n",
    "df = df.astype({'is_8': int})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle data\n",
    "df = df.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 70% of 42000 rows is 29400\n",
    "train = df[:29400]\n",
    "test = df[29400:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29400, 785)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare feaures\n",
    "features = train.loc[:, 'pixel0':'pixel784']\n",
    "# normalization, because we know that max value is 254\n",
    "features = features\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29400,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare labels\n",
    "labels = train.loc[:, 'is_8']\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(785,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#inital weights\n",
    "weights = 10*np.random.rand(785,1).reshape(-1)\n",
    "weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning rate and number of iterations\n",
    "learning_rate = 0.01\n",
    "# for 29400 rows and 784 features, one iteration takes 0.36 seconds\n",
    "iterations = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1. / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(features, weights):\n",
    "    '''\n",
    "    Returns 1D array of probabilities\n",
    "    that the class label == 1\n",
    "    '''\n",
    "    z = np.dot(features, weights)\n",
    "    return sigmoid(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function(features, labels, weights):\n",
    "    '''\n",
    "    Returns 1D matrix of predictions\n",
    "    Cost = ( log(predictions) + (1-labels)*log(1-predictions) ) / len(labels)\n",
    "    '''\n",
    "    observations = len(labels)\n",
    "    \n",
    "    predictions = predict(features, weights)\n",
    "    \n",
    "    #Take the error when label=1\n",
    "    class1_cost = -labels*np.log(predictions)\n",
    "\n",
    "    #Take the error when label=0\n",
    "    class2_cost = (1 - labels)*np.log(1 - predictions)\n",
    "\n",
    "    #Take the sum of both costs\n",
    "    cost = class1_cost - class2_cost\n",
    "    #Take the average cost\n",
    "    cost = cost.sum()/observations\n",
    "\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_weights(features, labels, weights, lr):\n",
    "    '''\n",
    "    Vectorized Gradient Descent\n",
    "    '''\n",
    "    N = len(features)\n",
    "\n",
    "    predictions = predict(features, weights)\n",
    "\n",
    "    gradient = np.dot(features.T,  predictions - labels)\n",
    "\n",
    "    # take the average cost derivative for each feature\n",
    "    gradient /= N\n",
    "\n",
    "    # multiply the gradient by our learning rate\n",
    "    gradient *= lr\n",
    "\n",
    "    # subtract from our weights to minimize cost\n",
    "    weights -= gradient\n",
    "\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(features, labels, weights, lr, iters):\n",
    "    cost_history = []\n",
    "\n",
    "    for i in range(1, iters + 1):\n",
    "        weights = update_weights(features, labels, weights, lr)\n",
    "\n",
    "        #Calculate error for auditing purposes\n",
    "        cost = cost_function(features, labels, weights)\n",
    "        cost_history.append(cost)\n",
    "\n",
    "        # Log Progress\n",
    "        if i % 10 == 0:\n",
    "            print(\"iter: \" + str(i) + \" cost: \"+str(cost))\n",
    "\n",
    "    return weights, cost_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in exp\n",
      "  \n",
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:11: RuntimeWarning: divide by zero encountered in log\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:14: RuntimeWarning: divide by zero encountered in log\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter: 10 cost: inf\n",
      "iter: 20 cost: inf\n",
      "iter: 30 cost: inf\n",
      "iter: 40 cost: inf\n",
      "iter: 50 cost: inf\n"
     ]
    }
   ],
   "source": [
    "trained_weights, cost_history = train(features, labels, weights, learning_rate, iterations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in exp\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "predictions = predict(test.loc[:, 'pixel0':'pixel784'], trained_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decision_boundary(prob):\n",
    "    return 1 if prob >= .5 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(preds):\n",
    "    '''\n",
    "    input  - N element array of predictions between 0 and 1\n",
    "    output - N element array of 0s (False) and 1s (True)\n",
    "    '''\n",
    "    decision_boundary_v = np.vectorize(decision_boundary)\n",
    "    return decision_boundary_v(predictions).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(predicted_labels, actual_labels):\n",
    "    diff = predicted_labels - actual_labels\n",
    "    return 1.0 - (float(np.count_nonzero(diff)) / len(diff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_labels = classify(predictions)\n",
    "actual_labels = test.loc[:, 'is_8']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9394444444444444"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(predicted_labels, actual_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12600, 1220, 979, 718)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predicted_labels), np.count_nonzero(actual_labels), np.count_nonzero(predicted_labels), np.count_nonzero(predicted_labels*actual_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5885245901639344"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.count_nonzero(predicted_labels*actual_labels) / np.count_nonzero(actual_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision = np.count_nonzero(predicted_labels*actual_labels) / np.count_nonzero(predicted_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall = np.count_nonzero(predicted_labels*actual_labels) / np.count_nonzero(actual_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = 2*precision*recall/(precision + recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6530241018644838"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(785,)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trained_weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f5390e39240>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFWtJREFUeJzt3X+MnXWVx/H3odBf08pIf4xQuk6BWrfSteIEiOBSFVBZ1uqqaM0adiMWjWY10USX3Uiz7ho0oIuJUat0hfgDXBGom0ZUgrJdlDiQLj+KhYqztLZMf1HsL1raOfvH3Jq2zHPOdJ6Ze6/5fl5J0zv33O/zfOe598xz7z3P9/s1d0dEynNCqzsgIq2h5BcplJJfpFBKfpFCKflFCqXkFymUkl+kUEp+kUIp+UUKdWIzd9YxfZJ3dp884vZGnasRs7ZWo32dtjlP/kYbA2E023osaz+W6l592r59z57TSPR8P9u3iz3b9g3rF6+V/Gb2FuBGYBzwTXe/Lnp8Z/fJfLj3b0e8v3EcqtH2YBg/lByKqH2dtsNxgAlhfDz7K2N1+5a1H0t1j1s79z17TiPR8/3lntuGvZ0R//kxs3HAV4C3AvOBJWY2f6TbE5HmqvOZ/1xgvbs/5e4HgFuBxaPTLREZa3WSfxaw4YifNzbuO4qZLTWzXjPr3bN1b43dichoqpP8Q32p8KJvOdx9ubv3uHtPx4zJNXYnIqOpTvJvBGYf8fPpwKZ63RGRZqmT/L8G5prZHDMbD7wXWDk63RKRsTbiWoi7HzSzjwJ3M1jqW+Huj0VtjIGwTJFpZekmcohxteJZCTOL1zkuWcmpTnm1rrEuodZRd991jmt8XIZ/bUOtbHL3VcCqOtsQkdbQ5b0ihVLyixRKyS9SKCW/SKGU/CKFUvKLFKqtCudjWXPOasZZLT6S1+HjbR9gfK3t7w/aZ8c02/f+mu0jk6g31mMCB8L4+CA+Odl39pxlv/VYXoMQ9c2Po86vM79IoZT8IoVS8osUSskvUiglv0ihlPwihWqrUl+mzgy6dUp5mTrlLsjLaTvpDOO7mFoZ28LMWtt++qj5Wl5s+/bpYfyF9S+pDibVsBPO2hPG53WtC+MLWVMZ66I/bNvJzjBe9/VU57UcDYuPp3E/ms78IoVS8osUSskvUiglv0ihlPwihVLyixRKyS9SqKbW+Z0Taq1OGhnLOj7Etfx+usK2mzg1jG/mtDCebb//uepa/vO7k1WSnk+ejxOTYvzGk+L4nUFsfdx0oLMjjD/efU4cPzuIn/982PbcWQ+E8egaAoCZyXUEk9lXGau3crKG9IpIQskvUiglv0ihlPwihVLyixRKyS9SKCW/SKFq1fnNrA/YBRwCDrp7T/h4PJyGus50x9n01tmY+2hMPMS19j66w7aP7F8Qxp/re1kYz+rhkZf/1W/C+Lv4QRh/M3eH8Yunrg7j9lwQfE/YlB/2vDWMX88nw/gvH3tjdfCZiWHbtZ3zw/j4jnja8MxsNtRqPxpG4yKfN7j7tlHYjog0kd72ixSqbvI78BMze9DMlo5Gh0SkOeq+7b/A3TeZ2Uzgp2b2G3e/78gHNP4oLAXo/LP4c7WINE+tM7+7b2r8vwW4Azh3iMcsd/ced+/pmDGpzu5EZBSNOPnNrMPMph6+DVwKPDpaHRORsVXnbX8XcIeZHd7Od939x6PSKxEZcyNOfnd/Cnj1cbYKa/nZWP9ovvK6S01ndf5o/vsNydz2hw7Gcw1MOX1rGO+e1xfGz6N67Pl81oZto2WsAbYxLYw/cEryErisOpQd859xcRjP2p/xqscqY9k1JVkdfh5PhPFMtKx6vvR4dR4Mfgc/PCr1iRRKyS9SKCW/SKGU/CKFUvKLFErJL1KotlqiOxuWG01ZnJXy9hFfXRiVXiDu2zzipaIXdDwSxqexPYyfxqYw3h+UIdcSD01dx7wwvvr3rw/j6dTf64OppOPRxiQjpaEnnn77yln/URnLhir/BfFzlr1WewlHt7OOV1TG4lLe6NGZX6RQSn6RQin5RQql5BcplJJfpFBKfpFCKflFCtXkOr8lywvHsmG7kawuOz2ptc+hrzKW1eEX8fMwPveBjWF8z8L4b/TPJlQPfd3O9LDt2kPxdQDcGk9xze44HBw2yOZ8jkf0cs7i3jDew4OVsSVP3hVv/I44zCvj8N63xUujZ8u2N4PO/CKFUvKLFErJL1IoJb9IoZT8IoVS8osUSskvUqi2Gs+fiWr1h4inx57E3jCe1fmjKbAXsiZsO+NbSTH8V3G4428GwviCSx+ujN3Nm8O2O/5rVrzz6+Iw255MHjC3OvSupOnl8TTUr+P+MP73+6vH8/O5ZN/xy4Fk+oj02o9oeu5sbonRojO/SKGU/CKFUvKLFErJL1IoJb9IoZT8IoVS8osUKq3zm9kK4HJgi7uf3bjvFOA2BmdW7wOucPdn63Ymq9VH8Wy8frbscSc7w3hU55+xJqnjb4nDnJzE98ThM/7nmcrY1Rd8LWz79fOvDuMDBzvinfNQEg/q/PHU9lxx5i1h/EN8PYx3rAquj3g83jenJfE5cTh7Lc+kvzK2ha4RbztYJeFFhnPm/xbwlmPu+zRwj7vPBe5p/Cwif0LS5Hf3+4Adx9y9GLi5cftm4O2j3C8RGWMj/czf5e6bARr/V68XJSJtacy/8DOzpWbWa2a9e7buG+vdicgwjTT5+83sVIDG/5Vfabn7cnfvcfeejhnxYpki0jwjTf6VwJWN21cCyVSoItJu0uQ3s+8BvwTmmdlGM/sAgwM9LzGzJ4FLyAd+ikibSev87r6kIvSm49+dM46DldGsNlpHNkY6u05gL9XzsD8/99hiyDGCUjfAxOw6gOeS+H9Wh179+Xi8/a9Wnh/Gz10Wr1PPre+J40ER+J2Lvx02/Qz/EsZfteqpeN/RNAsL46bZq3vH2+L1DH5HdxjfF7yeMgeDPIlnQDiarvATKZSSX6RQSn6RQin5RQql5BcplJJfpFBtNXV3Vm6L4lnbrIy4gdlh/LecWRnb3xHP4/xzFoXx8XPi4cZv4N4wfu09X6iMfflHYVP+4VOPhvFLPx9fv/XwxxaE8bdR3YEe4iW2s+eUeNdwRnVoz5z4vPeDCfG84jvpDOPj2R/GdzG1MnZi8nsfCOYN9+M4n+vML1IoJb9IoZT8IoVS8osUSskvUiglv0ihlPwihWpynd84NEa7rDMkF+B+XhfG793+hsrYC3e+JGzLz+IwfXH4FxcfO3ny0a767DcrYzs+mQw3jsvRvIJ1Yfw93BbGxwdTpq9JxtXeSjxc+NDs+LU0L+j7WuaHbbPrPrIluF+bXMNwGpsrY1EdfzTpzC9SKCW/SKGU/CKFUvKLFErJL1IoJb9IoZT8IoVqq/H82Zj7qJaftd3FlDC+jnlh/IXrg1p+umpBPEU1rI/Dv1oWhu/9bPU1CMveeHu87Q/F4WhpcsiPe18whfUNN/1zvPNlcThb4vsXZwfXR0xPtn16HN60KF7De/a0DWG8K1i3ve7cFMOlM79IoZT8IoVS8osUSskvUiglv0ihlPwihVLyixQqrfOb2QrgcmCLu5/duG8Z8EFga+Nh17j7qrqdSedpD2S1z2wegU3b47ot4fT28TLYMCmJXxGHr4/DXfRXB9+f7Pr5JJ6IlosGeIDzqoMfTza+e3Uc3zgrjvfOqY5dnOw7XoGbQwez11scj5aMn8y+sG20JoAxELY90nDO/N8Chrpa4kvuvrDxr3bii0hzpcnv7vcByXQwIvKnps5n/o+a2cNmtsLMXjpqPRKRphhp8n8VOBNYCGwGbqh6oJktNbNeM+vdszX+LCMizTOi5Hf3fnc/5O4DwDeAc4PHLnf3Hnfv6ZiRffElIs0youQ3s1OP+PEdJN+Fi0j7GU6p73vAImC6mW0ErgUWmdlCwBmcePrqMeyjiIyBNPndfckQd980Bn1JjeNgEMvG81evhw7wwu7kI0k0vnvh3LhtTxyfeH1cTFl58uvD+CXXBvXwpFTOpXE4m99+OtvD+P2HgvUQLo/3za0XJg9IdAax7qRtMt6/u6svjEfrFUB83Um98fo27EfqCj+RQin5RQql5BcplJJfpFBKfpFCKflFCtXUqbsdG7Vph48VDZEE2JcMqz1hYlyaGTjrpOrge8Om/ONFnwnjn/v2Z8P43mR67dV7qmPndMRt7zptcRh/MJkf+3dJzWzntqDelky9nU6vncx4HpZnz4+bntTzhzAeDqMGJrM3jEd5UGcK++OhM79IoZT8IoVS8osUSskvUiglv0ihlPwihVLyixSqqXV+YyCcdvgAE8L2Uf1zH5PDtv10hfHxE6v7BfB8T3XB/KKLfhy2/dyquI5/ezK99m/iMO8LYgefC65PAN73v3fGG/9asvOr4vCUV26tjO0OjimQ1/nPTuJR+3jF9nSJ7WzI7v4ar+Vs29HQ9sEpNoZHZ36RQin5RQql5BcplJJfpFBKfpFCKflFCqXkFylUU+v8YOGUxdk45ag2ujep8x9Ixvunul+oDM1nba1Nv/ONyQMuisOrP3NOZeyMxx6MGy9M9p0tP94dT0s+fmF1zfqEVwYTEQADU5LrAHbH4XDq7unx2uSdPBvGp7IrjMe1+Pi1no/nj7c9XDrzixRKyS9SKCW/SKGU/CKFUvKLFErJL1IoJb9IodI6v5nNBm4BXgYMAMvd/UYzOwW4jcHFjvuAK9w9Lo4msvplVP/MluB+Niz6woRk3v6pL99cGetkZ9j25suuCONcFofvJ1jmGlh+98eqg2vibdcWl8vZsXFm0DaeayA5rOl4/ylnVc8l0N3RF7Y9i9+G8UnJvPwT0iW6q1/L2fUu0bwXfhzn8+E88iDwCXf/cwaXOviImc0HPg3c4+5zgXsaP4vIn4g0+d19s7s/1Li9C3gcmAUsBm5uPOxm4O1j1UkRGX3H9ZnfzLqB1wAPAF3uvhkG/0AAwfs7EWk3w05+M5sC3A583N3jhcyObrfUzHrNrHfP1n0j6aOIjIFhJb+ZncRg4n/H3X/YuLvfzE5txE8FtgzV1t2Xu3uPu/d0zIgXyxSR5kmT38wMuAl43N2/eERoJXBl4/aVwF2j3z0RGSvDGdJ7AfB+4BEzO1w4uga4Dvi+mX0AeBp4d93OZFN313FiUj6ZNCEu3XQN/cYGgCd4Rdj2R/x1GN+ZlCE3/n52GJ9yYVDSenNf2HbdVfPC+Atr4iG72RTYEzurh75OP3l72Hb8vLhcNiGYBh5gAY9UxrJhs9mQ3YNJ+2iK+sF4/LtFolKgHcfU3Wnyu/tqwCrCbxr2nkSkregKP5FCKflFCqXkFymUkl+kUEp+kUIp+UUK1eSpuz0ctpsP6a3ubhf9YduZQZ0e8qm/o7rwWuaHbddvPyuMTw1q4QBLZn03jP8r/1QZO+OGZ8K2nByHt14VF/Kz4cZ9dFfGuukL224gvr5hLyO/YjQbAp4t+Z7JhuVG1yjkU3fH2x4unflFCqXkFymUkl+kUEp+kUIp+UUKpeQXKZSSX6RQTa3zOyeEY/br1C+z8dfZdQBZTXnd9njce7jvafG+pxGPa4/GpQOc8bugln9j2JSfb4jji74Sr4O9+LKfxBuIpgOYEzfdcdHEMP4E8XMSPafbmBa2zer86VTwyXj9aLx/dD3LaNKZX6RQSn6RQin5RQql5BcplJJfpFBKfpFCKflFCtXUOr/hYS1/LMcxz0zq/PNZG8YnTKuuy+5P1hs4jU214ps4NYx/cc6HK2OvffrBsO2rkt/7meQ56U+WaIyuv8jmQchq7dmY/Cietc3H44+8jg/Nq+VHdOYXKZSSX6RQSn6RQin5RQql5BcplJJfpFBKfpFCpcVGM5sN3AK8DBgAlrv7jWa2DPggcHhx+GvcfVWdzmS11eg6gGy98052hvF5rAvjUS0+mpse4GEWhPHVD1wSxlkTh8Nf7fmk7cI4fNKFfwjjC6Y9HManB3MVZM9JFs/mcIjiB5PrF7I6fqZOHb/O+hXHYzhbOQh8wt0fMrOpwINm9tNG7Evufv2o9EREmipNfnffDGxu3N5lZo8Ds8a6YyIyto7rM7+ZdQOvAR5o3PVRM3vYzFaY2Usr2iw1s14z692zdV+tzorI6Bl28pvZFOB24OPu/gfgq8CZDH5q3AzcMFQ7d1/u7j3u3tMxY+Rrq4nI6BpW8pvZSQwm/nfc/YcA7t7v7ofcfQD4BnDu2HVTREZbmvxmZsBNwOPu/sUj7j9yqNk7gEdHv3siMlaG823/BcD7gUfM7HDR6RpgiZktBBzoA67ONuRYWK7LSn11hgNnpZsTk31PYm9lLJt6e2FSq9t7XvxxaOd5Q36d8kf7gqWqs+HGmcnB7w15iTVqnw17nUz8HVHWvs4Q8LpDcptVrqtjON/2rwZsiFCtmr6ItJau8BMplJJfpFBKfpFCKflFCqXkFymUkl+kUE2eunsgrJ9Gy3dDtqxxXOevW7cdH7att+/OZN9dbAnjdWR9ryuqtdepw0O945691sa6Dj/Wx304dOYXKZSSX6RQSn6RQin5RQql5BcplJJfpFBKfpFCmbs3b2dmW4H/O+Ku6cC2pnXg+LRr39q1X6C+jdRo9u3l7j5jOA9savK/aOdmve7e07IOBNq1b+3aL1DfRqpVfdPbfpFCKflFCtXq5F/e4v1H2rVv7dovUN9GqiV9a+lnfhFpnVaf+UWkRVqS/Gb2FjNbZ2brzezTrehDFTPrM7NHzGyNmfW2uC8rzGyLmT16xH2nmNlPzezJxv/xvN7N7dsyM/t949itMbPLWtS32WZ2r5k9bmaPmdnHGve39NgF/WrJcWv6234zGwc8AVwCbAR+DSxx97VN7UgFM+sDety95TVhM/tLYDdwi7uf3bjvC8AOd7+u8Yfzpe7+qTbp2zJgd6tXbm4sKHPqkStLA28H/o4WHrugX1fQguPWijP/ucB6d3/K3Q8AtwKLW9CPtufu9wE7jrl7MXBz4/bNDL54mq6ib23B3Te7+0ON27uAwytLt/TYBf1qiVYk/yxgwxE/b6S9lvx24Cdm9qCZLW11Z4bQ1Vg2/fDy6TNb3J9jpSs3N9MxK0u3zbEbyYrXo60VyT/U6j/tVHK4wN3PAd4KfKTx9laGZ1grNzfLECtLt4WRrng92lqR/BuB2Uf8fDqwqQX9GJK7b2r8vwW4g/Zbfbj/8CKpjf/HboK/49ROKzcPtbI0bXDs2mnF61Yk/6+BuWY2x8zGA+8FVragHy9iZh2NL2Iwsw7gUtpv9eGVwJWN21cCd7WwL0dpl5Wbq1aWpsXHrt1WvG7JRT6NUsa/A+OAFe7+b03vxBDM7AwGz/YwOLPxd1vZNzP7HrCIwVFf/cC1wJ3A94E/A54G3u3uTf/iraJvixh86/rHlZsPf8Zuct8uBP4beAQYaNx9DYOfr1t27IJ+LaEFx01X+IkUSlf4iRRKyS9SKCW/SKGU/CKFUvKLFErJL1IoJb9IoZT8IoX6f4A63ggC1FGFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "weights_pic = trained_weights[0:784].reshape(28, 28)\n",
    "plt.imshow(weights_pic, cmap='jet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf,\n",
       " inf]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}