Chapter3_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/utils/deprecation.py:77: DeprecationWarning: Function fetch_mldata is deprecated; fetch_mldata was deprecated in version 0.20 and will be removed in version 0.22\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/utils/deprecation.py:77: DeprecationWarning: Function mldata_filename is deprecated; mldata_filename was deprecated in version 0.20 and will be removed in version 0.22\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'COL_NAMES': ['label', 'data'],\n",
       " 'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),\n",
       " 'target': array([0., 0., 0., ..., 9., 9., 9.])}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Import MNIST dataset using sklearn (can do this for popular datasets)\n",
    "\n",
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original')\n",
    "\n",
    "#You will get error because this data is corrupted. \n",
    "# See https://github.com/ageron/handson-ml/issues/143 for details about this error. \n",
    "\n",
    "#You will have to manually download the data from \n",
    "#https://github.com/amplab/datascience-sp14/raw/master/lab7/mldata/mnist-original.mat\n",
    "#and place it into your scikit data home dir (/home/priyaroy/scikit_learn_data for me)\n",
    "\n",
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sklearn.utils.Bunch"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70000, 784) (70000,)\n"
     ]
    }
   ],
   "source": [
    "featuresPixels=mnist[\"data\"]\n",
    "labels=mnist[\"target\"]\n",
    "\n",
    "print(featuresPixels.shape,labels.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corresponding label is: 0\n"
     ]
    }
   ],
   "source": [
    "#Select one row and view it. The 784 features correpond to each of 28X28 pixels. \n",
    "#Each pixel carries some color between white and black. \n",
    "#View the image in binary format (ie. only 2 colors: white or black)\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#Select one row & arrange pixels in 28X28 format by reshaping featuresPixels\n",
    "some_digit=featuresPixels[200,:]\n",
    "some_image=featuresPixels[200,:].reshape(28,28)\n",
    "\n",
    "#View image. Use the imshow function\n",
    "\n",
    "plt.imshow(some_image, cmap=matplotlib.cm.binary)\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "\n",
    "#It looks like a 0. Let's check what the corresponding label is\n",
    "\n",
    "print(\"Corresponding label is:\", int(labels[200]))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Divide mnist data into training and test sets\n",
    "#sklearn already divides it for us. First 60,000 are training and\n",
    "#last 10,000 are test sets. \n",
    "\n",
    "x_train=featuresPixels[:60000]\n",
    "y_train=labels[:60000]\n",
    "\n",
    "x_test=featuresPixels[-10000:]\n",
    "y_test=labels[-10000:]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[38227 29298 34072 34699 36468 42647 28111  9038 11126 56136]\n"
     ]
    }
   ],
   "source": [
    "#Shuffle the training dataset because giving similar labels in a row can lead to\n",
    "#poor training.\n",
    "\n",
    "import numpy as np\n",
    "shuffle_index=np.random.permutation(60000)\n",
    "print(shuffle_index[:10])\n",
    "\n",
    "x_train, y_train=x_train[shuffle_index], y_train[shuffle_index]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is 200th image of the test sample predicted to be 5:  [False]\n",
      "Is the 200th image's label = 5?: False\n"
     ]
    }
   ],
   "source": [
    "#Train a binary classifier with Stochastic Gradient Descent (SGD) classifier\n",
    "\n",
    "#Convert labels of training and test sets to boolean\n",
    "\n",
    "y_train_5 = (y_train==5) #True if y_train = 5, False otherwise\n",
    "y_test_5 = (y_test==5)\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf=SGDClassifier(random_state=42)\n",
    "sgd_clf.fit(x_train, y_train_5)\n",
    "\n",
    "#Check on a test instance, say the 200th entry of the test sample\n",
    "print(\"Is 200th image of the test sample predicted to be 5: \",sgd_clf.predict([x_test[200]]))\n",
    "\n",
    "print(\"Is the 200th image's label = 5?:\", y_test_5[200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "score=cross_val_score(sgd_clf, x_train, y_train_5, scoring=\"accuracy\", cv=5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96425298, 0.96458333, 0.95575   , 0.96308333, 0.9588299 ])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[53483,  1096],\n",
       "       [ 1226,  4195]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Confusion matrix to evaluate your classifier\n",
    "# matrix is 2X2: [TN, FP]\n",
    "#                [FN, TP]\n",
    "#Diagonal elements are true negative and true positive, off-diagonal \n",
    "# are for false\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf, x_train, y_train_5, cv=5)\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_train_5, y_train_pred)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision score of sgd classifier is: 0.7928557928557929\n",
      "Recall score is: 0.7738424644899465\n"
     ]
    }
   ],
   "source": [
    "#Precision and Recall\n",
    "\n",
    "#Precision = TP/(TP+FP): Maximizing precision means minimize misidentification of background as signal\n",
    "#Recall = TP/(TP+FN): Maximizing recall means minimize loss of signal, at the cost of identifying \n",
    "#                     background as signal\n",
    "\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "print(\"Precision score of sgd classifier is:\", precision_score(y_train_5, y_train_pred))\n",
    "print(\"Recall score is:\", recall_score(y_train_5, y_train_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Precision and recall tradeoff\n",
    "#Threshold, decision function\n",
    "\n",
    "y_scores=sgd_clf.decision_function([featuresPixels[200,:]])\n",
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(59560,)\n",
      "(59560,)\n",
      "(59559,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8TFf/wPHPyR5CFkkIWURksYTEHmvsUUoVRVtKq9pHy9Pq6tfnaXV5npauqLa0j6ouVHWjKKWNfQm1L0nsYqmtReyR8/vjDEaKREwyM/F9v17zcufeM3e+cyfud+45556jtNYIIYS4vbnYOwAhhBD2J8lACCGEJAMhhBCSDIQQQiDJQAghBJIMhBBCUIBkoJSaoJQ6pJTaeJ3tSik1Wim1TSm1XilVx/ZhCiGEKEoFuTKYCKTcYHsHINryGAh8eOthCSGEKE75JgOt9ULg2A2KdAEmaWM54KeUCrFVgEIIIYqemw32UQnYa/U8y7LuQN6CSqmBmKsHvLy86oaHh9vg7YtWbm4uLi6O37QicdqOM8QIEqetOUucGRkZR7TWQbbery2SQYFprccD4wFiY2N1enp6cb59oaSmppKcnGzvMPIlcdqOM8QIEqetOUucSqndRbFfW6TBfUCY1fNQyzohhBBOwhbJYDrQ19KrqBFwXGv9tyoiIYQQjivfaiKl1GQgGQhUSmUBLwHuAFrrj4BZwB3ANuA00L+oghVCCFE08k0GWuve+WzXwGO2CObChQtkZWVx9uxZW+zOJnx9fdmyZYu9w8iXr68vO3fuJDQ0FHd3d3uHI4RwMsXagJyfrKwsypQpQ+XKlVFK2TscAE6ePEmZMmXsHUa+Tpw4wfnz58nKyiIyMtLe4QghnIxD9aM6e/Ys5cqVc5hE4EyUUpQrV86hrqqEEM7DoZIBIIngFsixE0IUlsMlAyGEEMVPkkEerq6uJCQkULNmTXr06MHp06dveZ+rVq1iyJAh192+f/9+unfvfsvvI4QQhSXJIA9vb2/Wrl3Lxo0b8fDwYMKECVdt11qTm5t7U/usV68eo0ePvu72ihUrMm3atELFK4QQtiDJ4AaaNWvG9u3b2bVrF7GxsfTt25eaNWuyd+9e5s6dS1JSEnXq1KFHjx5kZ2cDkJaWRuPGjalduzYNGjTg5MmTpKam0qlTJwAWLFhAQkICCQkJJCYmcvLkSXbt2kXNmjUB04jev39/4uPjSUxM5LfffgNg4sSJ3H333aSkpBAdHc2zzz5rn4MihCiRHKpraV7XGibknntg0CA4fRruuOPv2/v1M48jRyBvzUtqasHfOycnh9mzZ18eqyQzM5PPPvuMRo0aceTIEV577TXmzZtH6dKlGTFiBO+88w7PP/88PXv25Ouvv6Z+/fqcOHECb2/vq/b71ltvMXbsWJo0aUJ2djZeXl5XbR87dixKKTZs2MDWrVtp164dGRkZAKxdu5Y1a9bg6elJbGwsgwcPJiwsDCGEuFUOnQzs4cyZMyQkJADmyqBv376cPHmSiIgIGjVqBMDy5cvZvHkzTZo0AeD8+fMkJSWRnp5OSEgI9evXB6Bs2bJ/23+TJk0YOnQo9913H3fffTehoaFXbV+8eDGDBw8GIC4ujoiIiMvJoHXr1vj6+gJQvXp1du/eLclACGETDp0MbvRLvlSpG28PDLy5K4FLLrUZXHLy5EkASpcufXmd1pq2bdsyefLkq167YcOGfPf//PPP07FjR2bNmkWTJk2YM2fO364OrsfT0/PysqurKzk5OQV6nRBC5EfaDAqhUaNGLFmyhG3btgFw6tQpMjIyiI2N5cCBA6SlpQEmkeQ9YW/fvp34+Hiee+456tevz9atW6/a3qxZM7788ksAMjIy2LNnD7GxscXwqYQQtzNJBoUQFBTExIkT6d27N7Vq1SIpKYmtW7fi4eHB119/zeDBg6lduzZt27b92x3B7733HjVr1qRWrVq4u7vToUOHq7YPGjSI3Nxc4uPj6dmzJxMnTrzqikAIIYqCMuPMFb9rTW6zZcsWqlWrZpd4rsdZxia6FKcjHkNrzjCBiDPECBKnrTlLnEqp1Vrrerber1wZCCGEkGQghBBCkoEQQggkGQghhECSgRBCCCQZCCGEQJLB31gPYX3nnXfy119/2XT/EydO5PHHHwdg+PDhvPXWWzbdvxBCFIYkgzysh7AOCAjg448/tndIQghR5CQZ3EBSUhL79++//PzNN9+kfv361KpVi5deeuny+kmTJlGrVi1q165Nnz59AJgxYwYNGzYkMTGRNm3a8McffxR7/EIIUVAOO1DdEz8/wdqDa/MveBMSKiTwXsp7BSp78eJF5s+fT+/evQGYO3cumZmZrFy5Eq01nTt3ZuHChZQrV47XXnuNpUuXEhgYyLFjxwBo2rQpy5cvRynFJ598wsiRI3n77bdt+nmEEMJWHDYZ2MulIaz37dtHtWrVaNWqFWCSwdy5c0lMTAQgOzubzMxM1q1bR48ePQgMDAQgICAAgKysLHr27MmBAwc4f/48kZGR9vlAQghRAA6bDAr6C97WLrUZnD59mvbt2zN+/HieffZZtNYMGzaMRx555KryY8aMueZ+Bg8ezNChQ+ncuTOpqakMHz68GKIXQojCkTaD6yhVqhSjR4/m/fffJycnh/bt2zNhwoTL01vu27ePQ4cO0apVK7755huOHj0KcLma6Pjx41SqVAmAzz77zD4fQgghCshhrwwcQWJiIjVq1GDy5Mn06dOHLVu2kJSUBICPjw9ffPEFNWrU4IUXXqBFixa4urqSmJjIxIkTGT58OD169MDf359WrVqxc+dOO38aIYSzOXwY9u6FnTvhq6/gl1+K8M201vk+gBQgHdgGPH+N7RHAfGA9kAqE5rfPmJgYndfmzZv/ts7eTpw4Ye8QCuRSnI54DK399ttv9g4hX84Qo9YSp63ZI87cXK1379b600+1fucdrd98U+sBA7SePdtsnzVLazCPgACtH3lEa2CVLsB5+2Yf+V4ZKKVcgbFAWyALSFNKTddab7Yq9hYwSWv9mVKqFfA60MdG+UoIIZyW1rBwIezbB7m5kJEBNWvCPfeYbVWqwMWLV8orBZYKCNq2hR9+AC8vaNbMTPc7blzRxFmQaqIGwDat9Q4TqJoCdAGsk0F1YKhl+TfgB1sGKYQQji4725zIvb3h4EHw8wMPD0hMhI0br5RzcYE6daBHD7P8wQfm5B8WBq6u4OlpXgfg5gZduhRP/AVpQK4E7LV6nmVZZ20dcLdluStQRilVrjABaTvNvFYSyLETougdPQoTJ8Ly5eb5779Dixbg6ws+PuaEXqkSHDhgTuZTp5pf85s2wZYtcOYMpKWZxAEwcCDEx5vkUabMlURQ3GzVgPw08L5Sqh+wENgHXMxbSCk1EBgIZh7h1NTUq7b7+PiQlZWFr68v6tKRsrOLFy9y8uRJe4eRr5ycHLKysjh16tTfjqsjyc7Oduj4wDliBInT1m4U54cfRnHggBebN5fl6FEzJ/ljj23j7Nkstmwpw9GjVbnrrpMEBp7jzBlX/P3Ps379YfbuvQBATAwcOmT2dfBgcXyam5fvHMhKqSRguNa6veX5MACt9evXKe8DbNVah95ov9eaA/nChQtkZWX9bRJ5ezp79ixeXl72DiNfZ8+exc/Pj9DQUNzd3e0dznU5wzyzzhAjSJy29ttvqSiVzIYNsGyZ+bX+/PPmF3zFiqbePikJatSA1q2hXj1zFVDcimoO5IJcGaQB0UqpSMwv/l7AvXmCCwSOaa1zgWHAhMIE4+7u7nB36qampl6+69iROUucQhQHrU23THd38PeHs2dh1izYuhW2b4f9++HUKRgyBLp3hyVLoFOnppw+fWUfs2ZB587m5G81RFmJlW8y0FrnKKUeB+YArsAErfUmpdQrmC5O04Fk4HWllMZUEz1WhDELIW5TWsOOHXD6NBw/bnrhtGhhts2cafrhr1oFWVmweze0bw8//wwnTkC3blf2ExMD5cpdqbf384Pk5MN06RJChw5mW6lSxf/57KlAbQZa61nArDzrXrRangZMs21oQojbUW4unDxpGlovXDC9cwIDYe5cc3K35ucHf/5plidMMCf+oCCIjoZ//tP8CxAcDDNmmMRRpszf37NGDXjmmXSSk0OK9sM5MLkDWQhRbC5ehHPnYPNmU12zdi00bAh33WV66TRtaqp3LKO7ADB+PDz8MPz1F7RqZertExJMHb71if3DD02V0PWazDp1KtrP5uwkGQghbEZrUz2Tnm66Uu7bB8eORXCp/bhJE1ix4urX3H23SQZublCrljnJx8ebf11doWVLU+6ee8zjeoKDi+Qj3TYkGQghCiw3F/bsgUWL4MgRc6PVoUPwyCPmrtrnn4eRI6+UVwqCg69UvTz9tEkUvr7QoAEEBEBUlNnm6wtff13MH0hcJslACHGVc+dg1y5zM9WWLeaRlARDh5q6/Lwd/vz84I47TDJ44glzgq9e3TwqV4ZFi5Zj+piYnjvCMUkyEOI2s3EjTJ9uRsK81FhbvjxcmpojOtqMlAlmuISoKNOnHsxQCZ9+Cjk50K6deZ2n55V9h4TAc88V7+cRtiHJQIgSJjsbFiww3S9PnTKNtUrBffeZOvm33zbDKZQvb8q7u0Pjxqa+Xyn4739Nv/yEBIiNvbqR1ssL+vWzx6cSRU2SgRBOTmtzM9Xx49CokWm0te454+Jixrtp0MAkg5Ejza/3uLhr7+/++4snbuFYJBkI4WTOnjXdLb/6Kp4jR0w/+2PHTCJYtgwiIsy/ZcqYX/qRkaav/iVBQeYhhDVJBkI4oDNnYP16c5JfvBhWrjRdNjdtgj/+MIOd7d1bihYtoGxZMyRySop5rZeXSQxC3AxJBkLY2Z9/mgbb1avNr/nu3WHpUmjT5upyoaGmO2dEBPz739Cu3QqnGABOOAdJBkIUE60hM9Oc1EuVgr59zXg6x45dKdOjh0kGrVrBTz+Zap7YWPMa61471tU+QtiCJAMhisipU2aKw19+MX32Fyww1TsHDphkUK2aSQ4tW5runGFhVwZdUwo6drRv/OL2IslACBs4etQMg7xxoxlyoUULmDfPDLMAZuarVq3M+sBAs27YMPMQwhFIMhDiJh0/boZO0No02q5adXVVT7t25o7dxo1h8mTTpbNKFfvFK0RBSDIQogA2b4aPPoJffzVj6xw9anrx9Otnum5GRprnzZubE7+Hh+m+2auXvSMXomAkGYjb3htvmBEz3d3NSTw9vSKlSplf9IcOmaqdrVtN2UaN4JVXzFDMAL17m4cQzk6SgbhtZGebLpxTppi7dD/5xIzN8/d6+xguXDDJwMXF/NLv1QvuvNP05xeiJJJkIEqknBzz693TEz7/HJ55xvTRv/SLPiXlylg8Z8+aGbXOnzf/LliwjLvvTgJMY+/MmXb8IEIUE0kGosTYsQPmzDF3606ZYqZB7N3b9OQpWxbuvRdq1zZDLHfqdGX+W0/Pq/vwBwefw03+Z4jbjPzJC6d1/ryp4wdzkl+/3iz7+0PXrmYdmC6dGRn2iVEIZyHJQDiFnBxT35+WBqNHm5u4GjSA1FSzvW5d6NIFHnjA1PFf+tUvhCgYSQbCIeXmmv77kZGmi+a778Kzz5ptZcuacXvq179S7z9hgn3jFcLZSTIQDkFr01A7cyasW2eqfE6dghkzTP1+SooZj6dSJWjbFnx87B2xECWLJANR7HbsMD181q834/G8955Zf+ed5t+mTeGhh6BhQ3MXL0B8vHkIIYqGJANRbKZMgeeeq8+ePeZ5RISZMP2Sr74yN3XlnXBdCFH0JBmIInHypBmlMy0NHn0UKlQwVwTe3hd55x0zTHNY2JXySsmdvELYkyQDccsuNeIeOABPPWWmXNy168r2ypWhf38YMgSSkn6nZctkO0UqhLgeSQbippw+bX7x//ijOfmvX2/G9MnIgOBgM1tXbCzcdx8kJJheP35+5rU+PtLlUwhHVaBkoJRKAUYBrsAnWus38mwPBz4D/Cxlntdaz7JxrMIOzp2DLVsgJsbMrdu/P0ydaqZnjIiAevWuzLfr6mpG9BRCOJ98k4FSyhUYC7QFsoA0pdR0rfVmq2L/AqZqrT9USlUHZgGViyBeUcSOHoX33ze/+JcsMZOvA8yda7p0vvaaqe/v3PnqIRyEEM6tIFcGDYBtWusdAEqpKUAXwDoZaKCsZdkX2G/LIIXtnT4NixebKRnnzDH9+EeMML/uX3vN3PFbq5ap6unTBxITzeuio81DCFGyKK31jQso1R1I0VoPsDzvAzTUWj9uVSYEmAv4A6WBNlrr1dfY10BgIIBLiEvdu1+5myaBTYgrE4eXq5etPpNNZWdn4+MEdzjlF2d2ths+PjkAjBwZy7x55blwwQU3t1wiI08RE3OSp57KQCk4dcqV0qUv2iVOR+AMMYLEaWvOEmfLli1Xa63r2Xq/tmpA7g1M1Fq/rZRKAj5XStXUWudaF9JajwfGA/iE++jpB6czbd80/Lz8qBlck8QKiTSPaE7bKm3x9fK1UWi3JjU1leTkZHuHka+8cWoNu3ebX/3Dh8OZM6bB19vbjOdTrZqp9mnWzIXSpcsAZYCKxR6nI3KGGEHitDVnibOoFCQZ7AOseoQTalln7SEgBUBrvUwp5QUEAoeut9NK3pVY9tQyFuxawIyMGazct5KPf/+YMSvHoFDUDK5Jk7Am1AmpQ0rVFELKhODmIp2fCuLnn80QDpfG7o+NNVU958+bZDB8uF3DE0I4oIKcXdOAaKVUJCYJ9ALuzVNmD9AamKiUqgZ4AYfz23GAdwBdq3Wla7WuAJzLOUfqrlSWZy1n0Z5FfLbuMz5a/REA/l7+NA5rTJsqbWhZuSU1g2vi6uJa0M9Zoq1cCW+9FUNqqjnRJydDt25mVM+WLc1Qzq5yqIQQN5BvMtBa5yilHgfmYLqNTtBab1JKvQKs0lpPB54CPlZKPYlpTO6n82uMuAZPN0/aV21P+6rtAbiYe5G52+eSfjSdRXsW8d2W75iZaaadqlimIq0jW1M3pC4hZUKoGlCVxAqJqNukI/vRozB5shnjZ+VK8PAoT3i42eblBV9/bd/4hBDOpUD1LpZ7BmblWfei1fJmoIltQwNXF1c6RHegQ3QHnmj0BDm5OaQfSef3A7/z/dbv+Xnbz3y+/vPL5ct6liWlagqdYzrTpkobyvuUt3VIxe7UKXPCX7QIjh+Hl182Y/ePHQsvvWT6+r/0EiQkLOeuu2z+FQghbhNOVQnv5uJGjeAa1AiuQZ/afdBas+f4Hk6cO8HKfStZuGchP2/7mambpgIQFxhHs/BmNI9oToRvBNHloqngU8HOn6Jgjh839f5r1piE4OkJ1aub+XoB/vEPuOsu0/0TIDX1gv2CFUI4PadKBnkppYjwiwAgvnw8D9V5iFydS9q+NGZmzmTxnsV8teErPv7948uvKeddDn9vfxqFNqJeSD1Ke5SmrGdZIv0iqRNSp9jbIfbsMSf8KVNg7VozS9fMmeDra6Z07N4d+vY17QAuLldeFxRkHkIIYQtOnQyuxUW50DC0IQ1DGwJw/uJ5lmct588zf7Luj3VsPLSRHX/u4Pst3/PF+i+ueq23mzflfcrzUouXeKD2A0Xa/nDmjGnknT37yrp69aCJVU3P/PlF9vZCCHGVEpcM8vJw9aB5RHMAusR1ubz+XM45jp87zvmL5zly+ggbD23ki/VfsGLfCvr/2J+hc4bSLqodZU+XRe1SNI9ofsvJYd06eOEFaN8eBg82bQHjx5sEEB9vJnIXQgh7KPHJ4Ho83TwJdgsGILRsKAkVEri/1v2cv3ieMSvGsGD3AhbvWcy+k/v4eOfHhJYNpX1Ue+6udjd3RN9R4Pc5e9YM+/Dyy+Zfd/crM3qVKwfDhhXFpxNCiJtz2yaD6/Fw9eCpxk/xVOOn0Fozc/5MDpc7zJcbvuTbLd/yvzX/447oO0iOSCa+fDytI1vj7up++fVaQ2YmREWZvv0PPGBG+axQAd54AwYMMElACCEciSSDG1BK4ePmQ6fETvRP7M+FixcYsWQEY1aOYVam6Wlb2r00dSvWpYpHQ/avjef36Q05sj2cHRleREbCP/8Jd99trgZKlbLzBxJCiOuQZHAT3F3d+Vfzf/FCsxfIOpHFoj2L+HTeUpasXs1C//eg9AUzShNwx0/VaVmlBR2jO9KtUXsZSkMI4dDkDFUAWsOmTbBhg5nO8ehRRWRkGI89di9tK9zLqFHgW+4UDVN2sP30Kvae2EvqrlQmrZvEh6s+JKZcDAMSBxAVEIWvpy/NI5pfVbUkhBD2JskgHzk5irg4M63jJW5uMHCgWQ4KMuP/m5G742lOPAAvtniR7PPZzM6czX8W/Ydn5z17+fWl3UsTFxjHnTF30jGmI3VC6uCirG4iEEKIYibJwIrWkJZmunyuX2+6fSoF2dlm4peOHc3k7q6uZvyf/Ph4+NCjRg961OjBrr92cfzscTKOZrBg9wJWH1jNywteZviC4QAMSBzAvfH3klAhAX9v6WMqhChekgwwJ/tPPoExY2DHDtP9s0MHOHQIXF01+/IO2F0Ilf0qA1C7Qm161OgBwB/ZfzB542RmZc7i8/Wf88maTwCo4l+FmHIxhJcNp1NMJzpEd5A2ByFEkbqtzzCnTkHp0mYKyOeeM0M9//vfZswfPz9TJjW16N6/vE95nmj0BE80eoLjZ4+zdO9S1h5cy5K9S9h9fDfLs5Yz/vfxVPCpQJfYLpfnd4gLjJPhu4UQNnVbJYMzZ2DGDFi6FH79FU6cMA3CwcGQnm6qgOzF18v38gitl1y4eIFZmbOYsHYCk9ZNYtzqcYCpfroz5k6qBlSlakBV2lZpa6+whRAlxG2TDD75BIYMMQnB2xvq1IHevSE31wwAZ89EcD3uru50ietCl7gul4fvXn1gNb/u/JU52+cweePky2WjfaJpc6oN8cHx9KjRg8BSgXaMXAjhbEpkMtAaNm6EL7+ElBQz4md8PPTqBT17QuvWpkeQM7Eevrtv7b4AnL5wmvQj6czMnMmkNNONFWDQrEE0rNSQJxo9Qde4rni6edozdCGEE3CyU+KNaQ3z5pmpH5cuNSf8wECTDBo2NI+SpJR7KRJDEkkMSaRpblOat2jOmgNrmJ4+nWlbptH7294EeAdwX/x9PN34acJ9w+0dshDCQZWozu1dukC7drBzJ4weDfv3w9NP2zuq4uOiXKhbsS4vt3yZ9Y+uZ1qPaSRUSOCDtA+IeC+CTl91YnbmbHJ1rr1DFUI4GKe+Mrh4ESZMgDvugEqV4L77TBXQI48U7D6AkszVxZVu1bvRrXo3dvy5g/Grx/O/Nf9jZuZMKpapSK3ytYjyj6JbtW7UrlCbAO8Ae4cshLAjp7wyOHYMxo2DuDhzJ/DChWZ9z55mYLjbPRHkVcW/Cm+0eYOsJ7OY0m0KjcMaM3/HfMamjaXVpFYEvRlE60mtGbdqHIdPHbZ3uEIIO3C6K4PHHjN3BufkQEwMTJ5skoDIn6ebJz1r9qRnTXPAdvy5g/Qj6Szes5jXF7/Orzt/5dGZj9IiogWNQhuRXDmZpNAkfL187Ry5EKKoOXQy2LYNvv/eTP84fbqZE9jLC1q1MpPFNGxohosQhVPFvwpV/KvQIboDw5OHs2TvEr7d/C0r969kxJIRjFgyAoCeNXqSFJpEmyptiA2MlbuhhSiBHO5/9eHD8J//wMqVsGyZWRcXB3v3mglj3n7bvvGVVO6u7iRXTia5cjIAe4/vZd6OeczbOY8Fuxbw9aavAfD19KVlZEtiAmJoF9VORmAVooRwmGRw6WYwNzf46COoXt0MEdGnD8TGOt99Ac4uzDeM/on96Z/YH4A9x/cwf8d8Fu9ZzJK9S5iVOYuRS0cCUDWgKu2j2nNv/L00Cm0kI7AK4YTsfor99VczBPSpU+ZKwN8fDhyQyeEdTbhv+FXJ4cyFM8zMnMmGPzaw5uAaxq8ez9i0sXi5eZFQIYHWka1JrpxM/Yr1pc1BCCdgt2SQk6O4/35zl7CnJzz5pGkU9vCQROAMvN296V69O92rdwfgxLkTfLflO9YcWMPCPQt5ffHr/GfRf3BVrqRUTSGlagrVAqtxJueMnSMXQlyL3ZLBjh0+7N0Lw4aZG8MCpJu7UyvrWZZ+Cf3ol9APgD/P/Ena/jSmbZ7Gj+k/MjNz5uWyIetCSKmaQsvKLWka3pRI/0g7RS2EuKRAyUAplQKMAlyBT7TWb+TZ/i7Q0vK0FBCstfa70T4DA8/x229Qs+bNBy0cn7+3P+2i2tEuqh0fdfqIg9kHmbdjHj+t+omj7keZumkqn679FICm4U3pEtuFvrX7Elw62M6RC3F7yjcZKKVcgbFAWyALSFNKTddab75URmv9pFX5wUBifvsNCDgvieA24aJcqFimIn1r9yX8z3CSk5O5mHuRzYc3892W7/hm8zc888szPPPLMzyV9BT317qfWuVrSUO0EMWoIP/bGgDbtNY7tNbngSlAlxuU7w1MvsF2IXB1cSW+fDwvJb/ExkEbWdR/ET1r9OSdZe+QOC6RJhOa8OmaT8k6kWXvUIW4LSit9Y0LKNUdSNFaD7A87wM01Fo/fo2yEcByIFRrffEa2wcCAwGCgoLqTp069dY/QRHLzs7Gx8fH3mHkq6TEeeDMAZYfW87EXRM5kXMChaKuf12iSkfRtVJXynuVt3uMjkLitC1nibNly5artdb1bL1fWyeD5zCJYHB+bxwbG6vT09MLF3UxSk1NJTk52d5h5KukxXku5xyZxzKZtG4S32z+hl1/7cLNxY2O0R2pE1KHxAqJtKjcgrKeZe0Wo71JnLblLHEqpYokGRSkAXkfEGb1PNSy7lp6AY/dalBCeLp5UjO4JiPbjmRk25FsOrSJt5e9zbKsZUxPn47G/IiJD44nNjCWSL9IHqj9ANWDqqNkjBIhblpBkkEaEK2UisQkgV7AvXkLKaXiAH9gmU0jFAKoEVyDCV0mAJB9Pptle5exZO8SlmctZ9HuRUzbPI03l75JWc+y1AmpwxMNn6BRaCPK+xR9tZIQJUG+yUBrnaOUehyYg+laOkFrvUkp9QqwSms93VK0FzBF51emoGTnAAAYtklEQVTvJMQt8vHwoW1UW9pGtb28bs/xPfyw9Qe+2/Idi/csJnVXKgBR/lE0DmtM75q9SamaIlcNQlxHge4z0FrPAmblWfdinufDbReWEDcn3DecIQ2HMKThEP46+xer9q9i3cF1LNm7hNnbZvP5+s+p4FOBOiF1aB/V3ozAWi4WVxdXe4cuhEOw+9hEQtian5cfbaq0oU2VNjzFU5y/eJ4v1n/Bgt0LWLLHDLIH5qqhXVQ7nkp6iqiAKDtHLYR9yV09osTzcPXgwcQH+eyuz9g2ZBtrH1nL661fx8PVgw9XfUitj2rx7C/Psv/kfnuHKoTdyJWBuO3UrlCb2hVq83zT59l6ZCuvLHiFt5e9zbvL36VBpQbEucYR/mc4kX6R0sYgbhtyZSBua3GBcXzV7SsyHs/g6aSnydW5TNg1gajRUfi87kPTCU15d9m7bDq0iVyda+9whSgycmUgBBAVEMXrbV4HYNKsSZwOPs2Ww1v4Mf1Hhs4dCkCITwidYzvTKaYT9SvWl26rokSRZCBEHuGlwkmulwzAqA6j2HN8D7MyZzF/53y+3PAl41aPA6BBpQYkhSaRFJpEy8iWMuKqcGqSDITIR7hvOI/We5RH6z3K2ZyzzM6czcp9K1mydwnjVo9j1IpRuCgXkkKTaBXZim7VulG7Qm17hy3ETZFkIMRN8HLzomu1rnSt1hWACxcvsObgGmakz2D2ttn8Z9F/eHXhq1QPqs6QBkPoEN2BsLJh0hAtHJ4kAyFugburOw0qNaBBpQa82upVDp86zOfrP+f9le/z6MxHASjtXpq4wDiiy0UT4RtBTLkY/Lz8aFm5Jf7eMsercAySDISwoaDSQQxNGsqTjZ5k7cG1LM9aztYjW9l0eBOpu1I5cvoIObk5AHi7eVM9qDrNI5rTObYzjUIb4eXmZedPIG5XkgyEKAJKKRJDEkkMuXrSvwsXL7D3xF52/bWLH7f+yKbDmxi9YjTvLn8XD1cPEiok0DmmMwPqDJDeSqJYSTIQohi5u7pTxb8KVfyr0CqyFQDHzx5n8Z7FLNi9gEV7FvGv3/7F8AXD6VWzF13jupJSNYVS7qXsHLko6SQZCGFnvl6+dIzpSMeYjgCs2r+KiWsn8tm6z/hi/RcA3BlzJ/fUuIfOsZ2LZEIfISQZCOFg6lWsR72K9Xin/TvMzpzNhLUTmJ4+nRkZM/Bw9eDe+Ht5KPGhy20PQtiCJAMhHJSHqwdd4rrQJa4Lf539i/k75vND+g98uf5LJq6dSGnX0gw4P4DOsZ1pGt4UD1cPe4csnJiMTSSEE/Dz8qNb9W583vVzDj9zmM/u+oxqZasxasUoWk9qjd8bftQdX5cXf3uRjKMZyBxT4mbJlYEQTsbXy5e+tfsS/mc40XWiWZa1jB/Tf+TXnb/y6sJXeXXhq4SWDaVpeFMG1RtE0/CmctObyJckAyGcWKWylehevTvdq3dHa03msUzm7ZjHoj2L+HHrj0zZOIVIv0iea/IcDyQ8IPcxiOuSaiIhSgilFDHlYhhUfxCTu00ma2gW4zuNJ6h0EI/OfJTIUZE8Necpth/bbu9QhQOSZCBECRXgHcDDdR9m+UPLmd93PuG+4byz/B2qjqlK5KhI7v32XsasGMO2Y9vsHapwAFJNJEQJp5SiVWQrVgxYwaZDm/h5288sy1rGgt0LmLxxMi5zXGgW3oyO0eZeh2qB1aSN4TYkyUCI20iN4BrUCK5x+fnuv3bzQdoHfL/1e56d9yzPznuWmsE16V2zN+2i2pFQIQE3FzlN3A6kmkiI21iEXwQj2o4gY3AG2wZvY+wdYynjUYYXfn2B+h/Xx3+EP50nd2bMijGsPbhWpv4swSTlCyEAM/XnoIBBDKo/iAMnD7Bg9wIW7FrArG2zmJExA4BKZSrRrVo3ulXvRuOwxnLVUILINymE+JuQMiH0qtmLXjV7AbD58GZW7V/Fd1u+Y9zqcYxeORp3F3cahzWmT60+3Bt/L97u3naOWtwKSQZCiHxVD6pO9aDq9K3dl5PnTvJTxk8s3buUGRkzGDBjAEPnDqVTTCeahDWhSVgTmfbTCUkyEELclDKeZegd35ve8b0Z3WE0v+z4hSkbp/D91u/5asNXl8t1jO5Iy8otqXCmArk6FxclTZSOrEDJQCmVAowCXIFPtNZvXKPMPcBwQAPrtNb32jBOIYQDUkrRLqod7aLa8UnnT1j/x3p+3fkrmUczmb1tNjMzZwIwdNNQHkp8iGbhzUgKS8LPy8/OkYu88k0GSilXYCzQFsgC0pRS07XWm63KRAPDgCZa6z+VUsFFFbAQwjG5KBcSKiSQUCHh8rrdf+3mnZnv8Pv53xmxZASvL34dV+VKs4hmNAtvRnxwPPHl44n0i8TTzdOO0YuCXBk0ALZprXcAKKWmAF2AzVZlHgbGaq3/BNBaH7J1oEII5xPhF0HXSl0ZlTyK7PPZpO1L45cdvzB722xeXfjq5XLBpYPpGN2RlKoptKnShgDvADtGfXtS+Q11q5TqDqRorQdYnvcBGmqtH7cq8wOQATTBVCUN11r/fI19DQQGAgQFBdWdOnWqrT5HkcnOzsbHx8feYeRL4rQdZ4gRnD/OMxfP8Pufv3P8wnHS/kxj1Z+ryM7JxgUXksol0TusNzV8a1xjj8Ubp6Np2bLlaq11PVvv11YNyG5ANJAMhAILlVLxWuu/rAtprccD4wFiY2N1cnKyjd6+6KSmpiJx2o4zxOkMMULJiLMDHS4v5+TmkLYvjW+3fMuYlWNYcnQJ9SrWI8o/iir+VegY3ZGksKQia4h2luNZVAqSDPYBYVbPQy3rrGUBK7TWF4CdSqkMTHJIs0mUQogSz83FjaSwJJLCkni2ybOMWzWOX3b8wrKsZXy75VteX/w6waWDaRzWmFaVW9GzZk+CS0vzpK0UJBmkAdFKqUhMEugF5O0p9APQG/hUKRUIxAA7bBmoEOL2EVw6mH+3+Df/bvFvAI6fPc6szFl8v/V7luxdwg9bf2Do3KF0qNqBtlXa0jyiObXK15IB9m5BvslAa52jlHocmINpD5igtd6klHoFWKW1nm7Z1k4ptRm4CDyjtT5alIELIW4fvl6+l+9tAHNH9MS1E/lqw1eXh8rw8fAhLjCO2HKxxAXG0S6qHQ0qNbBn2E6lQG0GWutZwKw86160WtbAUMtDCCGKVPWg6oxsO5IRbUaw7+Q+5m6fy7qD69h6dCuL9iziyw1f8u/f/k2bKm24o+odPN7gcdxd3e0dtkOTO5CFEE5LKUVo2VAeTHzwqvVHTx9l1IpRvLrwVebtmMcbS97giYZP8HDdhwksFWinaB2b3B8uhChxypUqxystX+Hiixf57p7vCPcN5/9+/T+C3gyi0juVSPpfEo/NfIzp6dM5l3PO3uE6BLkyEEKUWC7Kha7VutK1WleW7l3KL9t/4eftP3P87HE+XPUhH6z6gCr+VXipxUtU0pXsHa5dSTIQQtwWGoc1pnFYY15KfgmAMxfO8FPGT/x38X954IcHqOhVkYfVwzxS9xFCyoTYOdriJ9VEQojbkre7Nz1q9OD3gb/zQ88fCPQM5OUFLxP6bijtv2jPot2LyG+EhpJErgyEELc1pRRd4rrge9CXkJohfLH+C8asHEPzic2JLRdLv4R+dI3rSky5mBJ9H4NcGQghhEVsYCyvtnqVPU/u4X+d/0dw6WCGzR9G3Ng4osdE0/f7vkxYM4FV+1dx4eIFe4drU3JlIIQQeZT1LMuDiQ/yYOKDbD+2nTnb5/BTxk/8sPUHPl//OQBBpYIYmjSUQfUHUdazrJ0jvnWSDIQQ4gaiAqIYFDCIQfUHceHiBTKPZbLhjw18uvZThs0fxoglI+hTqw9NwprQoFIDKvtVdsrqJEkGQghRQO6u7pfng+5Zsydp+9J4Y8kbfPz7x4xZOQYwVwwNKjWgYaWGtIpsReOwxk6RHCQZCCFEIdWvVJ9v7/mWCxcvsOHQBlZkrWDl/pWs3LeSmZkzeTH1RRpWasgHHT+gTkgde4d7Q5IMhBDiFrm7ulMnpA51QurwD/4BwJHTR5i0bhIvL3iZuuPr0r16d4Y1HUZChYQim5PhVkgyEEKIIhBYKpChSUN5MPFB3l76Nm8te4tpm6dRxqMMtSvUpnb52sSWi6VOSB0ahjbEzcW+p2NJBkIIUYT8vPx4tdWrPFrvUWZlzmLdH+tYe3Atk9ZN4uT5kwB4uHoQUy6G5uHNaVG5hUkUgbHFGqckAyGEKAaVylbi4boPX36eq3M5dOoQC3cvZNX+Vaw9uJZxq8fxwaoPAEiokMDTSU/TJa4LPh5FPzezJAMhhLADF+VCBZ8K3FPjHu6pcQ8AZ3POsvXIVqanT+fz9Z9z//f3A2b+hrti7yKlakqRxSPJQAghHISXmxcJFRJIqJDAv5r/i/k75l/umTRiyQj+u/i/RfbejtekLYQQAhflQtuotrzQ/AWWPrSUw88c5rt7viu69yuyPQshhLAZf29/ulbrWmT7l2QghBBCkoEQQghJBkIIIZBkIIQQAkkGQgghkGQghBACSQZCCCGQZCCEEIICJgOlVIpSKl0ptU0p9fw1tvdTSh1WSq21PAbYPlQhhBBFJd+xiZRSrsBYoC2QBaQppaZrrTfnKfq11vrxIohRCCFEESvIlUEDYJvWeofW+jwwBehStGEJIYQoTgVJBpWAvVbPsyzr8uqmlFqvlJqmlAqzSXRCCCGKhdJa37iAUt2BFK31AMvzPkBD6yohpVQ5IFtrfU4p9QjQU2vd6hr7GggMBAgKCqo7depU232SIpKdnY2PT9FPLHGrJE7bcYYYQeK0NWeJs2XLlqu11vVsvmOt9Q0fQBIwx+r5MGDYDcq7Asfz229MTIx2Br/99pu9QygQidN2nCFGrSVOW3OWOIFVOp/za2EeBakmSgOilVKRSikPoBcw3bqAUirE6mlnYMutJikhhBDFJ9/eRFrrHKXU48AczK/+CVrrTUqpVzAZajowRCnVGcgBjgH9ijBmIYQQNlagaS+11rOAWXnWvWi1PAxTfSSEEMIJyR3IQgghJBkIIYSQZCCEEAJJBkIIIZBkIIQQAkkGQgghkGQghBACSQZCCCGQZCCEEAJJBkIIIZBkIIQQAkkGQgghkGQghBACSQZCCCGQZCCEEAJJBkIIIZBkIIQQAkkGQgghkGQghBACSQZCCCGQZCCEEAJJBkIIIZBkIIQQAkkGQgghkGQghBACSQZCCCGQZCCEEAJJBkIIIShgMlBKpSil0pVS25RSz9+gXDellFZK1bNdiEIIIYpavslAKeUKjAU6ANWB3kqp6tcoVwb4J7DC1kEKIYQoWgW5MmgAbNNa79BanwemAF2uUe5VYARw1obxCSGEKAZuBShTCdhr9TwLaGhdQClVBwjTWs9USj1zvR0ppQYCAy1PzymlNt5kvPYQCByxdxAFIHHajjPECBKnrTlLnLFFsdOCJIMbUkq5AO8A/fIrq7UeD4y3vG6V1trh2xYkTttyhjidIUaQOG3NmeIsiv0WpJpoHxBm9TzUsu6SMkBNIFUptQtoBEyXRmQhhHAeBUkGaUC0UipSKeUB9AKmX9qotT6utQ7UWlfWWlcGlgOdtdZFkr2EEELYXr7JQGudAzwOzAG2AFO11puUUq8opTrfwnuPv4XXFieJ07acIU5niBEkTlu7reNUWuui2K8QQggnIncgCyGEkGQghBDChslAKZWglFqulFqrlFqllGpgWa+UUqMtQ1mst9yTcOk1DyilMi2PB6zW11VKbbC8ZrRSSlnWByilfrGU/0Up5V/IWAcrpbYqpTYppUZarR9mec90pVR7q/XXHI7D0qi+wrL+a0sDO0opT8vzbZbtlQsTp2VfT1mG+Ai0PHeo46mUetNyLNcrpb5XSvlZbXO441mAz1OgoVds+H5hSqnflFKbLX+P/7Ssv+Z3Y8vvv5Dxuiql1iilfrI8v+nv7Gb/LgoRo59Saprl73KLUirJEY+nUupJy3e+USk1WSnlZdfjqbW2yQOYC3SwLN8BpFotzwYUptvpCsv6AGCH5V9/y7K/ZdtKS1llee2l/Y4EnrcsPw+MKEScLYF5gKflebDl3+rAOsATiAS2A66Wx3agCuBhKVPd8pqpQC/L8kfAPyzLg4CPLMu9gK8LeUzDMA33u4FABz2e7QA3y/KIS/twxONZgM9y3diK6gGEAHUsy2WADMuxu+Z3Y8vvv5DxDgW+An4qzHdWmL+LQsT4GTDAsuwB+Dna8cTczLsT8LY6jv3seTxt+Uc9B+hpWe4NfGVZHgf0tiqXjvkP0BsYZ7V+nGVdCLDVav3lcpdea/WfKL0QcU4F2lxj/TBgWJ7Pk2R5zMlbzvKHcIQrJ8LL5S691rLsZimnChHrNKA2sIsrycChjmeeeLsCXzrq8SxA/NeMzdbvk08MPwJtr/fd2PL7L0RsocB8oBXwU2G+s5v9uyhEjL6Yk6zKs96hjidXRnYIsByfn4D29jyetmwzeAJ4Uym1F3jL8ubWH/qSLMu6G63PusZ6gPJa6wOW5YNA+ULEGQM0s1xqLVBK1S9knOWAv7Tpeps3zsuvsWw/bilfYEqpLsA+rfW6PJsc7XhaexDzS6kwcRbp8Syg68VWLCyX/omYwR6v993Y8vu/We8BzwK5lueF+c5uNv6bFQkcBj61VGd9opQqjYMdT631Psx5cg9wAHN8VmPH43lTw1EopeYBFa6x6QWgNfCk1vpbpdQ9wP+ANjez/5uhtdZKqWv2i80nTjdMNm4E1AemKqWqFFWcN5JPnP+HqYIpFoU9nlrrHy1lXgBygC+LLsqSSynlA3wLPKG1PmFdDX2j76a4KKU6AYe01quVUsn2jCUfbkAdYLDWeoVSahSmWugyBzme/pgBPyOBv4BvgBR7xnRTyUBrfd2Tu1JqEmYIazAf7BPL8vWGs9gHJOdZn2pZH3qN8gB/KKVCtNYHlFIhwKFCxPkP4Dttrp1WKqVyMQNU3WjYjWutPwr4KaXcLJnauvylfWUppdwwl65HCxqnUioe80eyznJSCAV+V6ZR3qGOpyXefkAnoLXluHKDOLnO+ls+njaQ39ArRUIp5Y5JBF9qrb+zrL7ed2PL7/9mNAE6K6XuALyAssAobv47u9m/i5uVBWRprS8NpT8Nkwwc7Xi2AXZqrQ8DKKW+wxxj+x3PwtQdXqcObAuQbFluDay2LHfk6gaalZb1AZi6PX/LYycQYNmWt4HmDsv6N7m6EWhkIeJ8FHjFshyDuZRSQA2ubojZgWmEcbMsR3KlIaaG5fXfcHVjzyDL8mNc3dgz9RaP7S6utBk42vFMATYDQXnWO+zxvMFnuW5sRfWwfCeTgPfyrL/md2PL7/8WYk7mSgPyTX1nhfm7KER8i4BYy/Jwy7F0qOOJGfl5E1DKsp/PgMH2PJ62/KNuiqnzWoep86xr9cc+FtOyvQGoZ/WaB4Ftlkd/q/X1gI2W17zPlTuly2EasDIxPYICChGnB/CFZf+/A62str1gec90rHoIYHocZFi2vWC1vorlD2Ob5Uu81EPJy/J8m2V7lVs8tru4kgwc7XhuwyTUtZbHR45+PPP5PNeMrQjfrymggfVWx/CO6303tvz+byHmZK4kg5v+zm7276IQ8SUAqyzH9AfMydzhjifwMrDVsq/PMSd0ux1PGY5CCCGE3IEshBBCkoEQQggkGQghhECSgRBCCCQZCCGEQJKBcEJKqXLKjI67Vil1UCm1z7L8l1JqcxG8X7KyjNJ5E69JVdeYB1wp1U8p9b7tohPCNiQZCKejtT6qtU7QWidgbsx517KcwJVxc67LcgenEMKKJANR0rgqpT62jBM/VynlDZd/qb+nlFoF/FMpFaSU+lYplWZ5NLGUa2F11bFGKVXGsl8fdWWM/C8vjWGvlGptKbdBKTVBKeWZNyClVH+lVIZSaiVmyAEhHI4kA1HSRANjtdY1MAOAdbPa5qG1rqe1fhszrs67Wuv6ljKXxtJ6GnjMcqXRDDhjWZ+IGZm3OuYu0SZKKS9gImbo9njMEAD/sA7GMg7Oy5gk0NTyeiEcjiQDUdLs1FqvtSyvBipbbfvaarkN8L5Sai0wHShrGTl0CfCOUmoI4KevDCe8UmudpbXOxQwZURmItbxfhqXMZ0DzPPE0xEz0dFhrfT5PDEI4DKk7FSXNOavli4C31fNTVssuQCOt9dk8r39DKTUTM67LEqtpBPPuV/7viBJFrgzE7WouZpRIwMzhbfk3Smu9QWs9AkgD4m6wj3SgslKqquV5H2BBnjIrgBaWHlDuQA9bfQAhbEmSgbhdDQHqKTMJ+mbM0OYATygzQfl64AJXZm77G8tVRX/gG6XUBkxPpo/ylDmAGUZ5GaYKaoutP4gQtiCjlgohhJArAyGEEJIMhBBCIMlACCEEkgyEEEIgyUAIIQSSDIQQQiDJQAghBPD/fcQaYmfHuUwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Threshold to choose precision and recall\n",
    "\n",
    "#First get decision scores of all instances (rows) in the training set\n",
    "\n",
    "y_scores=cross_val_predict(sgd_clf, x_train, y_train_5, cv=3, method=\"decision_function\")\n",
    "\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)\n",
    "\n",
    "print(precisions.shape)\n",
    "print(recalls.shape)\n",
    "print(thresholds.shape)\n",
    "\n",
    "#Thresholds has one less element, so read all elements of precisions\n",
    "#and recalls except the last entry\n",
    "\n",
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"Precision\")\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\")\n",
    "    plt.xlabel(\"Threshold\")\n",
    "    plt.legend(loc=2)\n",
    "    plt.ylim(0.4,1)\n",
    "    plt.xlim(-80000, 80000)\n",
    "    plt.grid()\n",
    "    \n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision when threshold is set to 60,000 is: \n",
      "0.891\n",
      "Corresponding recall is: \n",
      "0.574\n"
     ]
    }
   ],
   "source": [
    "#Setting the threshold\n",
    "\n",
    "#Suppose you want 95% precision, then from the graph we\n",
    "#see that the corresponding threshold is 60,000\n",
    "\n",
    "y_train_pred_prec95 = (y_scores >60000)\n",
    "\n",
    "print(\"Precision when threshold is set to 60,000 is: \") \n",
    "print(np.around(precision_score(y_train_5, y_train_pred_prec95), decimals=3))\n",
    "\n",
    "print(\"Corresponding recall is: \")\n",
    "print(np.around(recall_score(y_train_5, y_train_pred_prec95), decimals=3))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd4VGX2wPHvSYGAICgIiCBNEAgJHWVdJQg2ir2Di1hYVETsnUVXrNhFBQugP9vaUXGxLAHZRQUEkaKI1ABCpAQihLTz++OdSQYImUnInclkzud55pmZO3dmzp3APfd933vPK6qKMcYYAxAX6QCMMcZUHpYUjDHGFLGkYIwxpoglBWOMMUUsKRhjjCliScEYY0wRSwrGGGOKWFIwxhhTxJKCMcaYIgmRDqCs6tevr82bNy/Xe//8808OOeSQig2okrNtjg22zbHhYLZ5/vz5f6jqEcHWi7qk0Lx5c+bNm1eu96anp5OWllaxAVVyts2xwbY5NhzMNovImlDWs+4jY4wxRSwpGGOMKWJJwRhjTJGoG1MwxoQmLy+PjIwMcnJyIh2KJ+rUqcOyZcsiHUZYhbLNSUlJNGnShMTExHJ9hyUFY6qojIwMateuTfPmzRGRSIdT4Xbu3Ent2rUjHUZYBdtmVWXLli1kZGTQokWLcn2HZ91HIvKqiGwWkcUHeF1E5BkRWSEii0Ski1exGBOLcnJyqFevXpVMCKZkIkK9evUOqnXo5ZjCZOD0Ul4/A2jtuw0DXvAwFmNikiWE2HOwf3PPuo9UdZaINC9llbOA19TNB/qtiNQVkSNVdaNXMRljTEVRdbfCQigo2P9x4DqB6+bng4i7z82F+Hj32q5dkJCw9+f770ULiNd84pKq4XWPWSTHFI4C1gU8z/At2y8piMgwXGuChg0bkp6eXq4vzM7OLvd7o5Vtc2woaZvr1KnDzp07IxOQz2OPPca7775LfHw8cXFxPPXUU3Tv3p38/HzGjh3Lhx9+WHSF7tlnn82tt94KQN26dUlOTiYvL4+EhAQuueQSrrvuOkTiinawBQWFbNuWza+//srdd9/Bb7/9Rq1atWjWrBX//Ofj/PrrL0yY8BSTJn3oW18oLIS4uL13uCAUFEB+fhxxcVq0zL/O7t3xJCYWMmbMMAYPvpFmzZL58sv3mDBhNPXqNWLkyMeYNu01brnlGU9+w9rs4GjWUEA86+SYkP6mOTk55f4/EBUDzao6EZgI0K1bNy3vFX12BWRssG12li1bdtADsf6j28CjXf+R8L633Fx3BLx7tzv6nTt3Dp999iWffrqQgoLq7Nz5B3l5uWRm1uaRR+4gM/MP3nxzCXXqJJGdvZNJkx5n7dra7N4N1avX4LXXFqEKW7Zs5u67L2X16j0MG3bfXvHt2ZPDJZdcyKhRT3DSSQMBmD8/nZ9/ziYrqzp79sSzaVPSQf0GALm58dx11ytFv8nHH7/C3Xe/RKdOfyUuDtq370b16i7h+H8D/08vUnxzn5VPtWoJVK/uPisuDhIT3X1hIVSr5taLK8wn6Y8Mqu34g4LE6hQ2akr9+LyQ/qZJSUl07ty5XNsayaSwHmga8LyJb5kxMcm/w83NhT17YPNm9zg3F/LyYMcO2LnT7Tzy8lz3Q14eZGfDt9+24Pvv3XP/7dRToUYNtzPy77j93xN4v2ePexwf73ZK/m6M/PyD256ff95IzZr12batum9JfRISYMOGXbzzzkt8/PFq4uOTyM4GqM3QoWPYvbv4/f7vr1u3AXfeOZHLL+/O1VePKeozj48v5Msv36Rjx5707TuwaKfbvXsaCQmweHE6iYlQvz78+OP33HffDezZk0NSUg2efHISrVsfy/LlSxg1aih5ebkUFhYyadL7NGnSmCuuuJANGzIoKCjgttvu5bzzLqJ//zQefngc06dP46efZvPYY1dy5pln0r9/f8aNG8enn37Kn3/+yfXXX8/ixYvJy8tjzJgxnHXWWUyePJkPPviA7OxsCgoKmDlzZvB/DEt+hpwcaNSI+MaNqRkXR0EYWn6RTApTgREi8jZwHJBl4wkm3AoLITOzeCfruhGKH2/f7naSy5dD9erFO+lffoFatdz79+xxzxMS3E5438/5/Xd3O+qo4uX7vg7FO+/yabbfkhNPhD/+cI+7dy/v55Zu6dLio2B/4qlRw/2eZ599KlOm3M9FF7XhxBP7MmDARaSl9WLp0hUcffTRpKbWLtqRJyTsfUQdFwcdOxY/79KlJVBA06abadSoIQA7d/7Jjh2L6du3K6mp+8e2dSvUrAnNm8Phh7elf/9vSEhI4KuvvuLpp+/i/fff58EHX+SWW25g0KBB5ObmUlBQwLRp0zj66MZMn/4ZAFlZWdSu7WKqVg3uu280M2f+h3HjxtGtW7e9umnGjh3LySefzKuvvsr27dvp0aMHffv2BeCHH35g0aJFHH744Qf+QfPzXXYWcf9gqlWDMBf98ywpiMhbQBpQX0QygH8AiQCq+iIwDegHrAB2AUO9isVUDf7BuG3bYNMm10T3H1WvX+/+0+bkwPTprZk+3R1Bf/cdNGnils+aBccc49bfswdWrQpv/FlZpb+u6vYF1asX7wvq1XOPExPd/dq1bhsOO8ztSBMT3f2aNZto2LAhzZsXL69bFxo2hKSD7z05oHbtSnu1FosWzeebb75hxowZjBhxEQ8//DBdunQhPt5tA8CkSZN4+umn2bJlC//73/9o2tR1IJR07VV5T6zJyspiyJAh/Prrr4gIeXl5APTs2ZOxY8eSkZHBueeeS+vWrUlJSeHmm2/m9ttvZ8CAAZx44okhf88XX3zB1KlTGTduHOD69teuXQvAKaeccuCEoOqy2Lp1LhkccUTxDxRmXp59dEmQ1xW4zqvvN5VPfr7bqW3ZsncXyYoVsHCh2xnu2gXp6dCiBSxYULxD9HehhOYopk4tfjZ3bvHjH38s+R1JSdCggduhxse7e//j1auhZ08XQ/v2LqZq1WDjRkhJce/1x37kke65//3+W2Gh20lXq1b8+YHfU62a64OOjy/fb5uevoy0tIZ7LVu2DHz714NogRyc+Ph40tLSSEtLIyUlhSlTpnDhhReydu3aoguxhg4dytChQ+nQoQMFBQUlfs7KlSuJj4+nQYMGey1PTk4O3hUD3HvvvfTu3ZsPP/yQ1atXF42/XHrppRx33HF89tln9OvXjwkTJnDyySfzww8/MG3aNO655x769OnD6NGjQ9peVeX999/n2GOP3Wv5d999d+CS17m5sGaNO2o45BDXBI2gqBhoNpWXqvv3vG2b23nm5LiulKQkmD/f3c+d65aVxYIF7t7fXePn7+tu08Ylk1693A61enV35P+Xv8CWLeuoUaMpXbsWD9o1a1a8s65f361fvbp77OWRdCz75ZdfiIuLo3Xr1gAsXLiQZs2aUbNmTa688kpGjBjBhAkTSEpKoqCggNzAP3SAzMxMhg8fzogRI/Y7B//SSy/loYce4rPPPqN///4AzJo1a78j8qysLI466igAJk+eXLR85cqVtGzZkpEjR7J27VoWLVpE27ZtOfzwwxk8eDB169bl5ZdfDnmbTzvtNJ599lmeffZZRIQFCxaUPuC7ZYv7DwQugzdoUP7mUAWxpGD24++mycx0R5tLl7qumAUL3I5f1XXd5OS41m559OxZvGPeuhVat3Z9v127uoOlpCTX9VGrFtSpU3x0Hsr/l/T030hLaxp8ReOp7Oxsrr/+erZv305CQgLHHHMMEydOBFzf+7333kuHDh2oXbs2NWrUYMiQITRu3BiA3bt306lTp6JTUi+77DJuuumm/b6jRo0afPrpp4waNYpRo0aRmJhIamoqTz/9NH/4B1SA2267jSFDhvDAAw8UJQ+Af/3rX7z++uskJibSqFEj7rrrLubOncutt95KXFwciYmJvPBC6NfV3nvvvYwaNYrU1FQKCwtp0aIFn3766YHfkJDg/pE3a+b+M1QCopFqV5ZTt27d1CbZCV1p2/z77/C//8Hnn8OiRa47JCvLneVSVoce6o7ed+2C1FR3NJ+S4rqIevaEVq2gZcvw/Lu3v7OzbNky2pXe6R/VorL2kaobEFN1fY3+ZSG2DkLd5pL+9iIyX1W7BXuvtRSqOFV3NP/llzBnDsye7QZlVfGdCnhgDRoUH8V36eJ29gkJbnCxZUt3NF+jhlvPGBPErl2uqb1rlxtE9ieDSlaKxJJCFbF1K7z/vuuinDHD9fG7Ada0Ut/Xo4e7HXqo29knJ7tB3rp1wxG1MTGgsNA1w3//3Z1J0LKlSwqVLBn4WVKIQnl57gydr792p1yuXQsrV5b+npYti4/4u3Z1R/0tW5b/bBdjTIj27HEJ4fDD3WByQuXe7Vbu6EyR3bvdaZsPPQSffLL/6/6de+vWcOGF7qya44+HzMxv6Ncv9POsjTEVwH/lY716ro+1Q4dKM5AcjCWFSio72/X/P/EEfPONO9MnUL167kh/wAB35WrnziV3+aSnl3zetzHGIzt2uLGD3Fx3SXWNGlGTEMCSQqWyZQu88AL88597n5vv17SpO01z7Fg45ZRK2yVpTGzKz4eMDFdbJCkJjj3WJYQo4+UkOyaI+fPh2mvdUb6I6/K59969E0Jysusyysx0Ywdz57pCZ5YQTDSIj4+nU6dOdOjQgYEDB7J9+/YK+dzVq1dz3HHHVchn7eu1116jQ4cOpKSk0Llz56KSFZdffjnvvfdeyW9ShZ9/dgmhUSN36Xspp45u2LCB888/v+j5JZdcQmpqKk8++SSjR4/mq6++qtBtKgtrKYRZfj5MmgQPPuhamPtq3BjOOQduvNGd229MNKtRowYLFy4EYMiQIYwfP5677747wlEd2Oeff85TTz3FF198QePGjdmzZw+vvfbagd+Ql1dcza8MBewaN25clGB+//135s6dy4oVK8oVc35+PgkVOHhtLYUwyc2Fu+92/2aGDStOCKmpMGoUTJ3qyiKvXw/PPWcJwVQ9PXv2ZP16Vx0/OzubPn360KVLF1JSUvj4448B1wJo164dV199NcnJyZx66qns9tXTnj9/Ph07dqRjx46MHz++6HNzcnIYOnRo0ZH9jBkzAFfO4uyzz+aUU06hefPmPPfcczzxxBN07tyZ448/nq1bt+4X40MPPcS4ceOKrqyuXr06V1999X7r3X/ffXTv3JkO7doxbMgQVBUOO4xnXnmF9u3bk5qaysUXXwzAzJkz6dSpE506daJz587s3LmT1atX06FDBwBOPfVU1q9fT6dOnfjmm2/2apHMnz+fXr160bVrV0477TR+95XUTUtLY9SoUXTr1o2nn3764P84gVQ1qm5du3bV8poxY0a531teBQWq48erNmiw95Qk11yjum6d998fiW2ONNtmZ+nSpXsv6NVr/9v48e61P/8s+fVJk9zrmZn7vxaCQw45RFVV8/Pz9fzzz9fPP/9cVVXz8vI0KyvL99GZ2qpVKy0sLNRVq1ZpfHy8LliwQFVVL7jgAn399ddVVTUlJUVnzpypqqq33HKLtmvXTlVVx40bp0OHDlVV1WXLlmnTpk119+7dOmnSJG3VqpXu2LFDN2/erIceeqi+8MILqqo6atQoffLJJ/eL97DDDtPt27eXuC1DhgzRd999VzUnR7d8953q3Lmqy5bp4Esu0alTp6qq6pFHHqk5OTmqqrpt2zZVVR0wYIDOnj1bVVV37typeXl5umrVKk1OTlZV3etx4Pfk5uZqz549dfPmzaqq+vbbb+vgwYNVVbVXr156zTXXHPB33+9vr6rAPA1hH2vdRx6ZPRtee81dUOY/IGneHK66CkaOLLW70Zgqw1/DaP369bRr145TTjkFcAejd911F7NmzSIuLo7169ezadMmAFq0aEGnTp0A6Nq1K6tXr2b79u1s376dk046CYDLLruMzz5z8x3Mnj2b66+/HoC2bdvSrFkzli9fDkDv3r2pXbs2tWvXpk6dOgwc6GZnS0lJYdGiRWXfoJ07YckSZsycyaNvvcWuvDy2bt1KcmoqAwcOJDU1lUGDBnH22Wdz9tlnA3DCCSdw0003MWjQIM4991yaNGkS0lf98ssvLF68uOg3Kygo4Igjjih6/aKLLip7/CGwpFDBli+HQYMgsDxT48Zw221w/fWu5r8xEVHanL01a5b+ev36pb9+AP4xhV27dnHaaacxfvx4Ro4cyRtvvEFmZibz588nMTGR5s2bk+M777p6wOmb8fHxRd1H5RH4WXFxcUXP4+LiyC9harnk5GTmz5/PySefXPIHxseTk5jItePGMW/ePJo2bcqYMWOKYv/ss8+YNWsWn3zyCWPHjuWnn37ijjvuoH///kybNo0TTjiB6dOnkxRCaV5VJTk5mTlz5hQtC5yf+YCluA+S7aIq0KefurPQ/AnhyCNhyhR31tANN1hCMLGrZs2aPPPMMzz++OPk5+eTlZVFgwYNSExMZMaMGazxl48+gLp161K3bl1mz54NwBtvvFH02oknnlj0fPny5axdu3a/+QxCdeedd3LrrbcW9d3n5uTw8rhxsGGDf0PI8R3p169fn+zs7KL+/8LCQtatW0fv3r155JFHyMrKIjs7m99++42UlBRuv/12unfvzs8//xxSLMceeyyZmZlFSSEvL49ly5aVa7vKwloKFWDnTjj/fPjiC/e8Vi3XfdSxY2TjMqYy6dy5M6mpqbz11lsMGjSIgQMHkpKSQrdu3Wjbtm3Q90+aNIkrrrgCEeHUU08tWn7ttddyzTXXkJKSQkJCApMnT96rhVAW/fr1Y9OmTfTt2xctLETy8rhiwIC9rh6te9hhXH311XTo0IFGjRrR3TfXaUFBAYMHDyYrKwtVZeTIkdStW5d7772XGTNmEBcXR3JyMmeccQYbNwafebhatWq89957jBw5kqysLPLz8xk+fDg9evQo17aFLJSBh8p0q2wDzevXqx5zTPEA8umnq+bmVvjXlJsNusaGkAaaq5gdO3Z488EFBe4skLlzVRcuVN261ZvvKYdQt/lgBpqtQ+MgfPGFKzC3YoWbCOatt9zcBCXNLWuMiRJ79rg5D+rXd1ePRmiu5Eix7qNy+ugjd5EZuHGEL76Ao4+ObEzGmHKK4gJ2Fc2SQjm88w74rkuhY0c3e1nNmpGNyZiSqOp+8xqbfWRluXmSo7SA3b70IGfTtO6jMtq4sTgh1Krl5jOwhGAqo6SkJLZs2XLQO4kqKz8fVq2CX391pwa2bRuVBewCqSpbtmwJ6ZTXA7GWQhmsXw9nnOEeJye7+Q0q+XwZJoY1adKEjIwMMjMzIx2KJ3Jycsq/81N1R3h5eW5AsGZNN29tJRfKNiclJYV8gVxJbJcWotWr3aDytm1u3Ontty0hmMotMTGRFi1aRDoMz6Snp9O5c+eyvSkz040bxMW5FkKzZuC7ejoalGuby8i6j0KQmwv9+7uE0KED/PCDuzfGRAlVV564TRt46SW37KyzoiohhIslhSBUYehQWLrUXaE8e7arYWSMiRKrV8Npp8EVV0BKCvTuHemIKjVLCkFMmABvvumuPXjvPdf9aIyJEq+/7pr1c+bA88+7+k1t2kQ6qkrNesVLsW0b3HWXe/z88/CXv0Q2HmNMGTVsCCedBC++aBcShSjkpCAihwJHArtVda13IVUejzziEkP79q7laYyp5PLy4NFH3cVoo0e7uWsD6iSZ4EpNCiJSG7gGuBSoBfwBJIlIPWA28LyqfuN5lBGQkeH+bYG7twqnxlRyP/zgjt5+/BEuvdQNCNqFe2UWbFf3IZAJ9FHVY1T1eFXtBDQDngIuEpEDHkOLyOki8ouIrBCRO0p4/WgRmSEiC0RkkYj0O5iNqSg5OdCnj/s3lZbmzjwyxlRSu3fDHXdAjx6uZtGHH8Ibb1hCKKdSWwqq2vcAywuB73y3EolIPDAeOAXIAOaKyFRVXRqw2j3Av1T1BRFpD0wDmpdpCzwwfrybLCcx0c2eZoypxFauhCeegMsvh8cei7kCdhUtWPdRammvq2pp89n1AFao6krfZ70NnAUEJgUFDvU9rgNsCBaw17ZuhbFj3eNnn4WmTSMbjzGmBDt20Ojf/3ZN+eTk4gvRzEELNtA8vpTXFDiplNePAgKvG88AjttnnTHAFyJyPXAIUGLLJJwee8wNLteuDYMHRzoaY8x+pk2D4cM5dv16GDIE2rWzhFCBxKtiWSJyPnC6ql7le34ZcJyqjghY5yZfDI+LSE/gFaCDr3sq8LOGAcMAGjZs2PXtt98uV0zZ2dnUqlXrgK9nZSVywQU9ycuL45FHfqRHj23l+p7KJNg2V0W2zVVTYlYWrcaPp9GXX/Jns2YsGDGC/G7dIh1WWB3M37l3797zVTX4D1baDDzAmaXdgry3JzA94PmdwJ37rLMEaBrwfCXQoLTP9XLmtRtvdLOnnXxyub+i0rFZyGJDld/m/HzVNm1UExJUR49Wzcmp+ttcgoPZZkKceS1Y99EFpeUTYGopr88FWotIC2A9cDHu1NZAa4E+wGQRaQck4c52Crs9e2DKFPf4zDMjEYExZj+bNsERR0B8PIwb57qJUksd6jQHKdjZR5eV94NVNV9ERgDTgXjgVVVdIiL34zLWVOBm4CURuRGXZC73ZbSwmzjRDTIfeyxcd10kIjDGFFGFV1+Fm2+Ghx+G4cNh4MBIRxUTynJF82lAMu5oHgBVfbC096jqNNxppoHLRgc8XgqcEGoMXlF1A8wAN91kJbGNiaiVK+Hqq+E//4FevaBvxM8/iSkh7f5E5HmgLu5so0nAecC3HsYVVkuXuvk1qld3pzobYyJkyhS49lrXXfTiiy45WDmBsAr11/6rql4KbFHVe3Gnlh7jXVjhNWeOu+/XD6pVi2wsxsS0xo3h5JPdkdrf/24JIQJC7SjZ7bvPEZFGwBagsTchhd8HH7h7K7NuTJjl5roxg8JCGDMGTjnF3UzEhJqGPxeRusA4YCGwGnjXq6DCaedO+OIL99jGsYwJo7lzoWtX+Mc/3DhCZM4xMfsIqaWgqmN8D98VkU+BGqq61bOowuibb1yV3S5dbEY1Y8Ji1y5X1vrJJ910hlOn2hFZJRJSS0FEhvtaCqjqbkB9VxlHPf94wgkRPwfKmBixapUrLHb11bBkiSWESibU7qPhqrrd/0RVt+HmWYh6M2e6+xNPjGwcxlRpWVkwaZJ7nJwMK1a4s4tsfttKJ9SkEB/4RETigMSKDye8Cgth4UL32JKCMR757DOXCK66Cn7+2S2z8sOVVqhJ4UsReUtEeolIL+AN4CsP4wqLNWvcQPMRR0CjRpGOxpgqJjMTBg2CAQPcHAdz5kDbtpGOygQR6imptwLXAjf6nn8JTPAkojBavtzdt2sX2TiMqXIKCuCvf3XjB/fd52ZGs4uAokKoZx8ViMgE4HNVXeFxTGGzyDdFUHJyZOMwpsr4/Xdo0MBdkfz44+6Uvg4dIh2VKYNQzz4aAPyEayEgIp1E5EMvAwsHf1Lo1CmycRgT9QoLYcIEaNPG3YPrNrKEEHVCHVO4D1faYjuAqi6kCpS5+M43w3TnzpGNw5iotmIF9OnjKpl27w6nnRbpiMxBCDUp5AWekuoT1Zcf7trlpnVNTLSWgjHlNmkSpKTADz/ASy/BV19By5aRjsochFAHmpeJyIVAnG/SnJFEeZXUDRvc/ZFHusRgjCmHo492LYPx4+GooyIdjakAobYURgBdgULgAyAXGOVVUOHgP13aSlsYUwZ79rjCdaN906L06QMffWQJoQoJ9eyjP4HbfTcAROQoYJdHcXnuo4/cvY0nGBOi776DK690pSmGDHEF7EQiHZWpYEFbCiLSXUTOFpH6vufJIvIa8J3n0Xloly+d1asX2TiMqfT+/NNNSdizpytX8emnMHmyJYQqqtSkICIP4a5eHgT8W0TGADOAH4E2nkfnoVWr3P1f/xrZOIyp9Nasgeefd2cXLVkC/ftHOiLjoWDdR2cBHVV1t4gcDqwDUlR1pfeheSsry903aBDZOIyplLZvh/fec/WK2rd3p502aRLpqEwYBOs+yvGVysY3f8LyqpAQCgvdnMxgNY+M2c/HH7tEMHx48RkZlhBiRrCWQksR8U1WiQAtAp6jqud6FpmHNm+G7GxXo8vGFIzx2bQJRo6Ef/0LUlPhk0+sgF0MCpYUztvn+XNeBRJO/vIWdo2NMT7+AnZr18IDD8Btt9kFPDGq1KSgql+HK5BwWr/e3dvJEybmbdjg+lDj4+Hpp92FO+3bRzoqE0HBzj76SETOEJH9koeINBOR0SJyhXfhecN/NXOfPpGNw5iIKSyEF15w3UMvvuiW9etnCcEE7T66DrgZGC8im4BMIAloCawFxqvq+96GWPH8SaFx48jGYUxELF/u5keeNQv69oUzzoh0RKYSCdZ9tB64CbhJRI4BjgR2A7+o6s4wxOeJwLpHxsSUV16BESMgKQlefRUuv9z6Uc1eQi2Ih29ynSoxwY5/TMHKtZiY07y5axmMH29HRaZEISeFqmTuXHd/xBGRjcMYz+3ZA//8p3v8wANuIM0G00wpQq2SWqXUquXuDzsssnEY46n//c9NFjJ2LGzc6ArYGRNEyElBRKr5xhWiWm6uu3AN4PDDIxuLMZ7IzoYbbnDXHezaBf/+txtLsLEDE4JQ52juTznmaBaR00XkFxFZISJ3HGCdC0VkqYgsEZE3yxJ8eWzb5u7r14e4mGwnmSpv7Vo3T/J118HixTY9pimTUMcU7sfN0TwD3BzNwVoNIhIPjAdOATKAuSIyVVWXBqzTGrgTOEFVt4mI5+XpMjPd/e7dXn+TMeGTsHMnTJwIw4a5aw1WrrRzrk25hJoU8lR1u+zd/AzWQdkDWOEvoCcib+Oqri4NWOdq3LUO2wBUdXOI8ZTbdt9M0w0bev1NxoTJhx/S/aqrXOnfXr3g2GMtIZhy83KO5qNwpbb9MnCtjUBtAETkv0A8MEZV/73vB4nIMGAYQMOGDUlPTw8x7L1lZ2ezcOFPQAoNG/5Bevricn1ONMnOzi737xWtYmWbq23dyjHPPEODmTPJadmSnx56iOyNG92gcgyIlb9zoHBsc6hJYQQwmuI5mqcDd1XQ97cG0oAmwCwPsae1AAAdHElEQVQRSVHV7YErqepEYCJAt27dNC0trVxflp6eTuPGKQC0alWf8n5ONElPT4+J7QwUE9tcUOBKVKxbBw8+yMLu3enVt2+kowqrmPg77yMc2xxqUjhNVfedo/lcXII4kPVA04DnTXzLAmUA36lqHrBKRJbjksTcEOMqM3/3kZXMNlEpI8N1DcXHwzPPQIsW0LYtGmNHzMY7oZ5/c08Jy+4O8p65QGsRaSEi1YCLgan7rPMRrpWAbw7oNoCnk/j4T0etXdvLbzGmghUWwrPPutbBCy+4ZWecYfMdmApXaktBRE4DTgeOEpEnAl46FNeVdECqmi8iI3BdTfHAq6q6RETuB+ap6lTfa6eKyFKgALhVVbeUf3OC27HD3VtSMFHj55/dtJj//a87vXTAgEhHZKqwYN1Hm4HFQA6wJGD5TqDE6w4Cqeo0YNo+y0YHPFZ8BfdCjPeg7fSV8Tv00HB9ozEH4eWXXQG7mjVhyhS47DK7CM14KliV1AXAAhF5Q1VzwhSTp7Zudfd160Y2DmNC0qoVDBwIzz1n51GbsAh1oPkoERkLtMfNpwCAqrbxJCoP+c/Ws/9fplLKyYH773ePH3wQevd2N2PCJNSB5snAJECAM4B/Ae94FJOntvhGLBp4fu20MWU0e7YrYPfQQ+7SeytgZyIg1KRQU1WnA6jqb6p6Dy45RB3/KanWfWQqjZ073bjBiSe6lsIXX8BLL9nYgYmIULuP9ohIHPCbiAzHXW8Qlefv/P67u7ekYCqNjAxXxXTkSFfm2l/b3ZgICDUp3AgcgitvMRaoA1zhVVBeyc0tbhjVrBnBQIzZsgX+9S+45hpo184VsLOZ0EwlEFJSUNXvfA93ApcBiEjUTWaZne02NyHBWuYmQlTh/fddWeutW+Hkk10BO0sIppIIOqYgIt1F5GzfFceISLKIvAZ8F+StlU52djwA+fkRDsTEpo0b4bzz4IILoGlTmDfPJQRjKpFSk4KIPAS8AQwC/i0iY3BzKvyIr8JpNNmzxyWFdu0iHIiJPQUFbiD588/h0Ufh22+hY8dIR2XMfoJ1H50FdFTV3SJyOK4Udop/joRok5WVGOkQTKxZtw6OOsoVsBs/3hWwaxN1x1MmhgTrPspR1d0AqroVWB6tCQFAxJ33bbOuGc8VFLgqpoEF7E47zRKCqfSCtRRaioi/PLYALQKeo6rnehaZB/LzXQ60blzjqWXL4MorYc4cV8l04MBIR2RMyIIlhfP2ef6cV4GEQ36+O+Uo0XqRjFcmToTrr3dleF9/HQYNslPdTFQJVhDv63AFEg47drhsUK1ahAMxVVfr1nDOOa7ryGqpmCgU6sVrVYK/lEyMTGFrwmH3bhgzxrUGHn7YCtiZqBdq7aMqobDQNeNbtIhwIKZqmDXLnVb66KOQlWUF7EyVUKakICLVvQokHHJy3ObWrx/hQEx027EDrr0WevVyZxl9/bU7w8jGDkwVEFJSEJEeIvIT8KvveUcRedbTyDywe7frLTvkkAgHYqLbhg0weTLcdBMsWuRKVRhTRYTaUngGGABsAVDVH4Go6zhdu9ZVwbP5mU2Z/fEHPP+8e9y2LaxaBY8/bkcYpsoJNSnEqeqafZYVVHQwXktIKAQgOzvCgZjooQrvvAPt28OoUbB8uVtuU/eZKirUpLBORHoAKiLxIjIKWO5hXJ7wd/k2aRLZOEyU2LABzj4bLr4YmjWD+fPtimRT5YV6Suo1uC6ko4FNwFe+ZVElL8/lQJvDxARVUAAnnQTr18O4cXDDDa7mujFVXKj/yvNV9WJPIwmDvDzXVKge1edQGU+tWeOakvHxbgyhZUs45phIR2VM2ITafTRXRKaJyBARidph2pwcVzq7Ro0IB2Iqn4ICeOIJV1fdX8Du1FMtIZiYE1JSUNVWwANAV+AnEflIRKKu5eCvfZSUFOFATOWyeDGccALcfDP06ePGEYyJUSFfvKaq/1PVkUAXYAdu8p2o4h9TsNpHpsiLL0KXLvDbb/DmmzB1qp2JYGJaqBev1RKRQSLyCfA9kAn8xdPIPOBvKVhSMEUlKdq1c9NjLl0Kl1xiVyWbmBfqQPNi4BPgUVX9xsN4POWfjtOSQgzbtQtGj3YDyY884kpV9OoV6aiMqTRCTQotVbXQ00jCwH/2kQ00x6j0dLjqKtdVdO21rrVgLQNj9lJqUhCRx1X1ZuB98c9lGSBaZ16zU1JjTFYW3HabmwCnVSv4z3+svLUxBxCspfCO775cM66JyOnA00A88LKqPnyA9c4D3gO6q+q88nxXKPwtBes+ijEbN8L//R/ccgvcdx/UrBnpiIyptILNvPa972E7Vd0rMYjICOCAM7OJSDwwHjgFyMBd6zBVVZfus15t4Abgu7KHXzaZme5cVEsKMSAzE95+202N2bYtrF4NRxwR6aiMqfRCPSX1ihKWXRnkPT2AFaq6UlVzgbeBs0pY75/AI0BOiLGU2yGH5AM2R3OVpkqDr75yZxXdfHNxATtLCMaEpNSkICIXiciHQAsR+SDg9iWwPchnHwWsC3ie4VsW+PldgKaq+lk5Yi8zOyW1ilu3DgYOpP3Yse5K5AULrICdMWUUbEzhe9wcCk1wXUF+O4EFB/PFIhIHPAFcHsK6w4BhAA0bNiQ9Pb1c35mffxIAc+bMJDExNqZOzM7OLvfvFU2koIAef/sb1bZu5eerriLz4otdF1IMbDvEzt85kG2zN0Q9mldWRHoCY1T1NN/zOwFU9SHf8zrAb4B/doNGwFbgzNIGm7t166bz5pV9LFoV4nztosLC2DkTMT09nbS0tEiH4Z3Vq6FpU3fdwVdfQcuWpK9dW7W3uQRV/u9cAtvmshGR+araLdh6wbqPZvrut4nI1oDbNhHZGuSz5wKtRaSFiFQDLgam+l9U1SxVra+qzVW1OfAtQRLCwch3wwnEx8dOQqjS8vNdSet27YpnROvb11U1NcaUW7DuI//J3GWe6l5V831nKE3HnZL6qqouEZH7gXmqOrX0T6hYe/a4+4Komy/O7GfRIrjySpg3D846C847L9IRGVNlBDsl1X8Vc1Ngg6rmishfgVTg/3CF8Up7/zRg2j7LRh9g3bQQYy4Xf0vBRLnnn3cT3hx2mJsm84ILrOlnTAUK9ZTUj3BTcbYCJgGtgTc9i8oD/qRQr15k4zDl5B/76tDBTY+5dClceKElBGMqWKi1jwpVNU9EzgWeVdVnROSgzj4KN39SsBkVo8yff8I997g/3GOPuSkyTzop0lEZU2WF2lLIF5ELgMuAT33LouoSsLw8d28XrkWRr7+GlBR46ik3KOTRmXLGmGJluaK5N6509koRaQG85V1YFc9aClFk+3ZXzbRvX/cHmzULnnnGuoqMCYNQp+NcDIwE5olIW2Cdqo71NLIK5j/7yFoKUWDTJle36Pbb4ccf4cQTIx2RMTEjpONmETkReB1YDwjQSEQuU9X/ehlcRfInBZufuZLyJ4IbboBjj3UXpdUv85nQxpiDFGpnypNAP3+FUxFph0sSQa+OqyxsTKGSUnVlrUeNguxs6NcPWre2hGBMhIQ6plAtsOS1qi4DoqqsnCWFSmjtWujfH/72N9c6WLjQJQRjTMSE2lL4QURexF2wBjCIgyyIF265ue7ekkIlkZ8PaWmu2+ipp2DECFeDxBgTUaEmheG4gebbfM+/AZ71JCKPbNvm7v1jCyZCVq6EZs3cWUUvveRqFbVoEemojDE+QbuPRCQFOB34UFXP9N0eU1XPJ8WpSP4B5i1bIhtHzMrPh0cegfbtYbyvCnufPpYQjKlkglVJvQtX4mIQ8KWIlDQDW1TwF8Jr2zayccSkhQvhuOPgjjvcQPIFF0Q6ImPMAQRrKQwCUlX1AqA7cI33IXkjsHS2CaPnnoPu3WH9enjvPfjgAzjyyEhHZYw5gGBJYY+q/gmgqpkhrF9p+VsKdkVzmPhLUqSmwqBBroCdlbg2ptILtotsKSIf+B4L0CrgOap6rmeRVTBrKYRJdjbcfbc7zWvcOCtgZ0yUCZYU9j20e86rQLxmLYUw+OILGDbMXX9w/fWutWD1ioyJKsEm2fk6XIF4batv8lBrKXhg2za46SaYPNldhDZrFvz1r5GOyhhTDlE7RlBe69dHOoIqaPNmN4h8553uTCNLCMZErZjpTKle3d03bBjZOKqM33+Ht96CG28sLmBn09oZE/XK1FIQkepeBeK1Qt9s04ceGtk4op4qTJniLkK780749Ve33BKCMVVCSElBRHqIyE/Ar77nHUUkqspc+Aea42Kuw6wCrV4Np58Ol1/ukoIVsDOmygm1++gZYADu6mZU9UcR6e1ZVB7wtxRsoLmc8vOhd2/44w9XpmL4cMuwxlRBoSaFOFVdI3ufXljgQTye8ScF24+V0YoVrj5RQgK8+qorYNesWaSjMsZ4JNRd5DoR6QGoiMSLyChguYdxVThLCmWUlwcPPgjJycUF7Hr3toRgTBUXakvhGlwX0tHAJuAroqwOko0plMEPP8CVV7oxgwsugIsuinRExpgwCSkpqOpm4GKPY/GUjSmE6Jln3IVoRxzhitedc06kIzLGhFFISUFEXgJ03+WqOqzCI/KIdR8F4S9J0bmzmx7z8cfhsMMiHZUxJsxC7T76KuBxEnAOsK7iw/GOJYUD2LnTXW9QvbpLBCee6G7GmJgUavfRO4HPReR1YLYnEXnEX97CkkKAzz+Hv/8dMjJg1CgrYGeMKXftoxZAVBWMqFXL3W/YENk4KoUtW1wXUb9+7of573/hiScsIRhjQh5T2EbxmEIcsBW4w6ugvOCvfWSTfuGSwkcfwT33uFv1qK1eYoypYEGTgrgr1joC/vqihaq636DzAd57OvA0EA+8rKoP7/P6TcBVQD6QCVyhqmtCDz90/oiTkrz49CiwcSO88QbcfDO0aQNr1thAsjFmP0G7j3wJYJqqFvhuoSaEeGA8cAbQHrhERNrvs9oCoJuqpgLvAY+WKfoyiNmBZlV3JXK7dnDvve4KZbCEYIwpUai7yIUi0rmMn90DWKGqK1U1F3gbOCtwBVWdoaq7fE+/BZqU8TtCFpNJYdUqUm+91V2I1rEj/PijFbAzxpSq1O4jEUlQ1XygMzBXRH4D/sTN16yq2qWUtx/F3qetZgDHlbL+lcDnB4hjGDAMoGHDhqSnp5cWdolWr24BNGP16pWkp68t8/ujjRQU0GPwYGpnZbH8xhvZMGCAG2WPgZH27Ozscv0biWa2zbEhHNscbEzhe6ALcKaXQYjIYKAb0Kuk11V1IjARoFu3bpqWllbm7/jcl25atWpJWlrLckYaBX791RWti4+Ht95izubN9LzwQtpEOq4wSk9Ppzz/RqKZbXNsCMc2B+tMEQBV/a2kW5D3rgeaBjxvQvFgdfEXiPQF7gbOVNU9ZYi9TPwjIVW2+ygvDx54ADp0gOeec8vS0tjToEFk4zLGRJVgLYUjfGcIlUhVnyjlvXOB1iLSApcMLgYuDVzBN04xATjdV1/JM1V6TGHePDdusGgRXHwxXHJJpCMyxkSpYEkhHqiFr8VQFqqaLyIjgOm+z3lVVZeIyP3APFWdCjzm+/x3fXM1rFVVT7qqqmxSePppV8CuUSP4+GM409OePmNMFRcsKWxU1fvL++GqOg2Yts+y0QGP+5b3s8uqyiUFf0mKbt1cK+HRR6Fu3UhHZYyJcsGSQpWpe+BPClFfyWHHDrj9dncV3pNPwgknuJsxxlSAYMfNfcISRRhUiYHmadPcTGgTJ7rpMUO7jtAYY0JW6i5SVbeGKxCvRXX30R9/wODB0L8/1KkD//sfPPZYFWj2GGMqm2jcRZbLGl9FpahMCtu2wSefwD/+4abKPK60awCNMab8Qp1kJ+rVru3u//gjsnGEbP16V8Du1ltdaYo1a2wg2RjjuWg8bi6XQw5x9/XqRTaOoFThpZegfXsYMwZ+810jaAnBGBMGMZMU/GOy1apFNo5S/fYb9OkDw4ZBly7uYrRjjol0VMaYGBIz3Uf+pFBpx2bz811C2LoVJkyAq66K0gEQY0w0s6QQab/8Aq1auVNMp0xxj5t4VkHcGGNKFTOHopUuKeTmwn33QUoKjB/vlvXqZQnBGBNR1lKIhLlz4YorYPFiuPRSGDQo0hEZYwwQQy0Fv4gnhaeeguOPL7724I03oH79CAdljDFOzCSFiFeE8AfQowdcfTUsWQIDBkQ2JmOM2Yd1H3ktKwtuuw1q1HCthL/8xd2MMaYSirmWQliTwiefuIvQXn4ZqlevBM0VY4wpnSUFL2RmugHkM890l1B/+y088kglGNAwxpjSWVLwQlaWK3N9331uqszu3cPwpcYYc/BsTKGirFsH//d/cMcdrjTFmjWuzLUxxkQRaykcrMJCePFFN/nNAw8UF7CzhGCMiUKWFA7Gr7/CySfDNde4U01/+skK2Bljopp1H5VXfj6ccgps3w6vvAJDh9pAsjEm6llSKKtly9ykNwkJ8PrrroBd48YHHZ8xxlQG1n0Uqj173HSYqanw3HNu2YknWkIwxlQpMdNS8CtXUvj2W7jySli6FC67zN2MMaYKirmWQpk9/rgrS7Fzp7v24LXXomBOT2OMKZ+YSwohtxQKC919z54wfLgrc33GGZ7EZowxlUXMdB+FnBS2b4ebb4aaNeHZZ62AnTEmplhLIdBHH7kCdlOmQO3aVsDOGBNzLCkAbN4MF14I55wDDRvC99/Dgw/adQfGmJhjSQFgxw748ksYO9YlhC5dwhqbMcZUFp4mBRE5XUR+EZEVInJHCa9XF5F3fK9/JyLNvYplv6Swdq1LAqquNMXatXDXXZCY6FUIxhhT6XmWFEQkHhgPnAG0By4Rkfb7rHYlsE1VjwGeBB7xKp6ipKCF8PzzroDdgw8WF7CrXdurrzbGmKjhZUuhB7BCVVeqai7wNnDWPuucBUzxPX4P6CPiTUe+KrThF/5ydxpcd5071XTJEitgZ4wxAbxMCkcB6wKeZ/iWlbiOquYDWYAnV4ZJQT7TOY1DV/8EkybB9OnQvLkXX2WMMVErKq5TEJFhwDCAhg0bkp6eXubP2J3Xjr/XfJWzbyqkXfMEmDmzgqOsnLKzs8v1e0Uz2+bYYNvsDS+TwnqgacDzJr5lJa2TISIJQB1gy74fpKoTgYkA3bp107S0tDIHk5YG6enLSEs7uczvjWbp6emU5/eKZrbNscG22Rtedh/NBVqLSAsRqQZcDEzdZ52pwBDf4/OB/6jaFWPGGBMpnrUUVDVfREYA04F44FVVXSIi9wPzVHUq8ArwuoisALbiEocxxpgI8XRMQVWnAdP2WTY64HEOcIGXMRhjjAldzFzRbIwxJjhLCsYYY4pYUjDGGFPEkoIxxpgilhSMMcYUkWi7LEBEMoE15Xx7feCPCgwnGtg2xwbb5thwMNvcTFWPCLZS1CWFgyEi81S1W6TjCCfb5thg2xwbwrHN1n1kjDGmiCUFY4wxRWItKUyMdAARYNscG2ybY4Pn2xxTYwrGGGNKF2stBWOMMaWokklBRE4XkV9EZIWI3FHC69VF5B3f69+JSPPwR1mxQtjmm0RkqYgsEpGvRaRZJOKsSMG2OWC980RERSTqz1QJZZtF5ELf33qJiLwZ7hgrWgj/to8WkRkissD377tfJOKsKCLyqohsFpHFB3hdROQZ3++xSES6VGgAqlqlbrgy3b8BLYFqwI9A+33WuRZ40ff4YuCdSMcdhm3uDdT0Pb4mFrbZt15tYBbwLdAt0nGH4e/cGlgAHOZ73iDScYdhmycC1/getwdWRzrug9zmk4AuwOIDvN4P+BwQ4Hjgu4r8/qrYUugBrFDVlaqaC7wNnLXPOmcBU3yP3wP6iIiEMcaKFnSbVXWGqu7yPf0WNxNeNAvl7wzwT+ARICecwXkklG2+GhivqtsAVHVzmGOsaKFsswKH+h7XATaEMb4Kp6qzcPPLHMhZwGvqfAvUFZEjK+r7q2JSOApYF/A8w7esxHVUNR/IAuqFJTpvhLLNga7EHWlEs6Db7GtWN1XVz8IZmIdC+Tu3AdqIyH9F5FsROT1s0XkjlG0eAwwWkQzc/C3Xhye0iCnr//cy8XSSHVP5iMhgoBvQK9KxeElE4oAngMsjHEq4JeC6kNJwrcFZIpKiqtsjGpW3LgEmq+rjItITN5tjB1UtjHRg0agqthTWA00DnjfxLStxHRFJwDU5t4QlOm+Ess2ISF/gbuBMVd0Tpti8EmybawMdgHQRWY3re50a5YPNofydM4CpqpqnqquA5bgkEa1C2eYrgX8BqOocIAlXI6iqCun/e3lVxaQwF2gtIi1EpBpuIHnqPutMBYb4Hp8P/Ed9IzhRKug2i0hnYAIuIUR7PzME2WZVzVLV+qraXFWb48ZRzlTVeZEJt0KE8m/7I1wrARGpj+tOWhnOICtYKNu8FugDICLtcEkhM6xRhtdU4G++s5COB7JUdWNFfXiV6z5S1XwRGQFMx5258KqqLhGR+4F5qjoVeAXXxFyBG9C5OHIRH7wQt/kxoBbwrm9Mfa2qnhmxoA9SiNtcpYS4zdOBU0VkKVAA3KqqUdsKDnGbbwZeEpEbcYPOl0fzQZ6IvIVL7PV94yT/ABIBVPVF3LhJP2AFsAsYWqHfH8W/nTHGmApWFbuPjDHGlJMlBWOMMUUsKRhjjCliScEYY0wRSwrGGGOKWFIwlY6IFIjIwoBb81LWbX6gapJl/M50XyXOH30lIo4tx2cMF5G/+R5fLiKNA157WUTaV3Ccc0WkUwjvGSUiNQ/2u01ssKRgKqPdqtop4LY6TN87SFU74oolPlbWN6vqi6r6mu/p5UDjgNeuUtWlFRJlcZzPE1qcowBLCiYklhRMVPC1CL4RkR98t7+UsE6yiHzva10sEpHWvuWDA5ZPEJH4IF83CzjG994+vjr9P/nq3Ff3LX9YiuenGOdbNkZEbhGR83H1pd7wfWcN3xF+N19romhH7mtRPFfOOOcQUAhNRF4QkXni5lG4z7dsJC45zRCRGb5lp4rIHN/v+K6I1AryPSaGWFIwlVGNgK6jD33LNgOnqGoX4CLgmRLeNxx4WlU74XbKGb6yBxcBJ/iWFwCDgnz/QOAnEUkCJgMXqWoKrgLANSJSDzgHSFbVVOCBwDer6nvAPNwRfSdV3R3w8vu+9/pdBLxdzjhPx5W18LtbVbsBqUAvEUlV1WdwpaR7q2pvX+mLe4C+vt9yHnBTkO8xMaTKlbkwVcJu344xUCLwnK8PvQBX02dfc4C7RaQJ8IGq/ioifYCuwFxfeY8auARTkjdEZDewGld++Vhglaou970+BbgOeA43P8MrIvIp8GmoG6aqmSKy0lez5legLfBf3+eWJc5quLIlgb/ThSIyDPf/+kjchDOL9nnv8b7l//V9TzXc72YMYEnBRI8bgU1AR1wLd79Jc1T1TRH5DugPTBORv+Nmp5qiqneG8B2DAgvmicjhJa3kq8fTA1eE7XxgBHByGbblbeBC4GfgQ1VVcXvokOME5uPGE54FzhWRFsAtQHdV3SYik3GF4fYlwJeqekkZ4jUxxLqPTLSoA2z01ci/DFccbS8i0hJY6esy+RjXjfI1cL6INPCtc7iEPj/1L0BzETnG9/wyYKavD76Oqk7DJauOJbx3J658d0k+xM2edQkuQVDWOH0F3+4FjheRtriZx/4EskSkIXDGAWL5FjjBv00icoiIlNTqMjHKkoKJFs8DQ0TkR1yXy58lrHMhsFhEFuLmUnjNd8bPPcAXIrII+BLXtRKUqubgKlC+KyI/AYXAi7gd7Ke+z5tNyX3yk4EX/QPN+3zuNmAZ0ExVv/ctK3OcvrGKx3GVUH/Ezc38M/AmrkvKbyLwbxGZoaqZuDOj3vJ9zxzc72kMYFVSjTHGBLCWgjHGmCKWFIwxxhSxpGCMMaaIJQVjjDFFLCkYY4wpYknBGGNMEUsKxhhjilhSMMYYU+T/AeBTWmd3T7p5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under the ROC curve, AUC =  0.96\n"
     ]
    }
   ],
   "source": [
    "#The Receiver Operating Charateristic (ROC) curve\n",
    "\n",
    "#It is a plot of Recall (or sensitivity) verus False Positive Rate\n",
    "\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds =roc_curve(y_train_5, y_scores)\n",
    "#Note: the order matters. The first returned array is the FPR, the 2nd is TPR.\n",
    "\n",
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    #TPR and FPR for a random classifier\n",
    "    x=np.linspace(0,1,10)\n",
    "    y=x\n",
    "    plt.plot(fpr, tpr, 'b-', linewidth=2, label=label)\n",
    "    plt.plot(x,y,'r--', label=\"Random Classifier\")\n",
    "    plt.legend(loc=1)\n",
    "    plt.xlabel(\"False Positive Rate\")\n",
    "    plt.ylabel(\"True Positive Rate (Recall)\")\n",
    "    plt.grid()\n",
    "    \n",
    "plot_roc_curve(fpr,tpr, \"SGD Classifier\")\n",
    "plt.show()\n",
    "\n",
    "#Find area under the curve (AUC) (A way to compare classifiers)\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "print(\"Area under the ROC curve, AUC = \", np.around(roc_auc_score(y_train_5, y_scores), decimals=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#Compare classifiers with ROC curves and AUC\n",
    "\n",
    "#First train a RandomForest Classifier on your data\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf=RandomForestClassifier(random_state=42)\n",
    "\n",
    "#RandomForest Clf does not have decision function. Use predict_proba instead. It \n",
    "#returns an np array\n",
    "y_prob_forest = cross_val_predict(forest_clf, x_train, y_train_5, cv=3, method=\"predict_proba\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.  0. ]\n",
      " [1.  0. ]\n",
      " [0.3 0.7]\n",
      " ...\n",
      " [1.  0. ]\n",
      " [0.9 0.1]\n",
      " [1.  0. ]]\n"
     ]
    }
   ],
   "source": [
    "#y_prob_forest.shape\n",
    "print(y_prob_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd4VGX2wPHvSQiEJghIE6QJ0kIH2yJRUBT72kXFgoiKLIqunUV/YMWGooANLGtZ1BUR+xKwIAKCVAsgJRTpgQAh7fz+eGeSISSZSTIlkzmf55lnZu7cmXtuypz7dlFVjDHGGIC4SAdgjDGm/LCkYIwxJo8lBWOMMXksKRhjjMljScEYY0weSwrGGGPyWFIwxhiTx5KCMcaYPJYUjDHG5KkU6QBKql69etq8efNSvXffvn1Ur149uAGVc3bOscHOOTaU5ZwXLly4XVWP8rdf1CWF5s2bs2DBglK9NyUlheTk5OAGVM7ZOccGO+fYUJZzFpF1gexn1UfGGGPyWFIwxhiTx5KCMcaYPFHXpmCMCUxWVhapqalkZGREOpSQqFWrFitXrox0GGEVyDknJibSpEkTEhISSnUMSwrGVFCpqanUrFmT5s2bIyKRDifo9u7dS82aNSMdRlj5O2dVZceOHaSmptKiRYtSHSNk1Uci8pqIbBWRZUW8LiIyXkRWicgSEekWqliMiUUZGRnUrVu3QiYEUzgRoW7dumUqHYayTWEKcGYxr58FtPbchgAvhTAWY2KSJYTYU9bfeciqj1R1jog0L2aX84E31K0H+qOI1BaRRqq6OVQxGRNsqofecnMj8/zPP6tRr96hr4vAvn2Hxxuq56H87MKeHzyYQHp65I7v+zw31928jwu+XqlS/rbMzPz9fPfJyYEqVdy+qpCdnb9vHDlUicumyhGVCXWNWSTbFI4GNvg8T/VsOywpiMgQXGmCBg0akJKSUqoDpqenl/q94ZaTI+zZU4m9exNIS3P3e/ZUYs+eBNLTK5GT464GcnMl7w9LVXz+IMXzB9iM557b6PNFInmPA3l/wfd4H7t93Pu9jwN5z6HHK/v7C9svJ6cXIhme/d32wo9X/DH9v788XYX3OmzLZ59BVlYEQvHx2mtj+fzzfxMfH49IHPfdN4mOHY8nOzubSZNG8c03/yEx0Y3Q7dfvEq6//n4Ajj8+nlatksjOzqJSpUoMGHANV155O3FxvpUbiQCsW/c7Tz89gg0b/qBatZo0bXosd975PGvXruStt8bxzDMzgnIuY8YM5sor76Bly/Z8/fV/mDRpFHXrNmT48CeZOfMN7rxzfJmPsX//4dtqsodmrCMnJ54NB49l7969fj8nIyOj1N91UdHQrKqTgckAPXr00NKO6IvECMjsbNi9G3buhB07Dr0vbtuePWEN05SRCMTFuXvvLZzPDxzYR40a1Q95vXJlqFbt0BgLxhzI84JXyL7b4uPz98vOxpOU3bbFi+fyww8zmDbtZxITq3DgwHaysjKpUwfGjHmAnTu38O67S6lePZH09L28+eZTxMdDzZqQmFiVr79ezL59kJq6lfvvv5J9+/Zw000P5cUQF5fLEUdkcsklZzNq1NMkJZ0LwMKFKezate2QeI88Mv9nsXdv0f9fcXFw9NH555+amn/l/8ADr+Tt9/HHrzJmzMv07/83AE45pQdpafnvy8qCxMT8n1H9+vk/p02bsomPr5RXehCBhIT8+6pV3eOcg9lU2Z5KtX3bya5UhYP1mlIvMSugxvXExES6du3qd7/CRDIpbASa+jxv4tlWLuXkQFpaYF/ovve7d5fueHFx7g+5Th2oW9fdex/Xrp3/R+Tvi2P16j9o06Z1oa9H8ksslM/nzfuRE088IWzHCxbvF2pmpvtC8V4Ur14N27a5L7K9e9327Gz3xdO4MSQnQ0rKfJKSkpk40W3PynJfRr5JoWHD/C+q7dvJ+xI7eNAdOz7efQFWrQrejiuZmbBkSdExt2kDRxzhPmPLFhen18aNm6lZsx65uVUA6N69HgB79+7nww9f5uOP11K5ciJZWVClSk0GDx5NTo77GxeBY46BjRvhyCPrc999k7n22p4MGTI6r868cmVlzpx/07v3iVx33bksX+7i79kzmUqVYNmyFBISoF49WLv2J+699x9kZGRQuXJVnnzydVq3Po7ff1/OiBHXkZWVSW5uLlOmfECNGo259NJLSU1NJSsrh3/+80Euuugyzj47mcceG8cXX8xk6dLvePjhG1i8+DzOPvtsxo0bx4wZM9i3bx+33XYby5YtIysri9GjR3P++eczZcoUPvzwQ9LT08nJyWH27Nn+/xjW/woZGdCwIZUaN6ZWXFxApYSyimRSmA4ME5F3geOBtPLWnrBvHzz7LLz4ImzeXPgVkz8i7kvc98u94H1h22rVyv9SKIuUlI0kJ7cu+wdFkQ0bMgi0N15urvsiy8pyX7Q5Oe7+iCOgUSO3z+bN8J//uPrezEx3++03qFHDvX/sWPDOUXbvvbB0qfsM723LFncbMQJGj3b7LVsGZ52V/zq4vxXv39jy5dC+vXt8993wwQeFx3/uuS4pAOzaBQ88kP9a797uy9+ruJ/JvffC3//uHr//Pjz0UNH7/vKL+xlUq5b/N1qliksm1au7+6wsuOCCM5g69WEuu6wNffr0Y9Cgy+jTpw9r1qzimGOOoVOnmnlJtVKlQ0s4Xo0auWTWrVtLIIemTbfSsGEDwCWX115bRvfu3QHo0OHQOHfudDE2bw516rTl22+/pVKlSnz99dc899x9fPDBBzzyyETuvPMfDBw4kMzMTHJycpg5cyaNGzfm008/BSAtLY2aNd25Vq4MDz00itmz/8e4cePo0aPHIdU0Y8eO5bTTTuO1115j9+7d9OrVi379+gHw888/s2TJEurUqVP0Dzc7O7/4dfTR7oBhnvQvZElBRN4BkoF6IpIK/AtIAFDVicBMYACwCtgPXBeqWEoqMxNeeQUefhj++it/e61agX2h+z6uXdv9jk3Zqbo611273O/lwAH3u6pfHzp2dPts2gTPPNOaL76A9HSYNw+aNHEXXF27wpgx+Vf3jRrlfyEXNGQITJrkHs+eDf/4R9Fx3Xtv/v/t3Llu/8J8/HF+UsjJcVUTBc9PxH3BZmfnb2/TBo4/3n0/rF8Pxx7rSpEJCdDNpyN3nTouloQEd6tdGxo0yC8dFOeoo9xx4uLgxx+L37dTp8K316/vbvlqsGTJQr799ltmzZrFZZddxmOPPUa3bt2Ij3fnAPD666/z3HPPsWPHDn744QeaNs2vQCh4YVTakllaWhqDBg3ijz/+QETI8jS2nHjiiYwdO5bU1FT+/ve/07p1a5KSkhg5ciR3330355xzDr179w74OF9++SXTp09n3LhxgKvbX79+PQCnn3560QlB1WWxDRtcMjjqqPwfUJiFsvfRFX5eV+DWUB2/NHJz4b333NXWmjVuW69e8OijcMop+T0ITOlkZ7vqD+/fekYG/Pe/sGoVLF7svgz374eUFHdV+8kn+fW7F1zgvlQLc801MHWqe7xxI0yffjTTp+e/Pn++u1+/HhYuhBkz3O+ySxf4/HP3WmKi+0KrVMkl8aN8Jhju2dNd1Wdmuqv3ypXdbfNmSEo69EJu7FiXtCpVOvSWm+uuWL3atXPxxMe71ytXdnXphV1APPJIYD/fOnUO3XflSvD5fg24pDtsmLsFQ3x8PMnJySQnJ5OUlMTUqVO59NJLWb9+fd5ArOuuu47rrruOjh07kpOTU+jnrFmzhvj4eOofmnXo0KGD/6oY4MEHH+TUU0/lo48+Yu3atXlti1deeSXHH388n376KQMGDGDSpEmcdtpp/Pzzz8ycOZMHHniAvn37MmrUqIDOV1X54IMPOO644w7ZPm/evKKnvM7MhHXrXH1e9equCBpB9jWH+2f54gt3lbV4sdvWtq37B7vgguDWG1c0qu7vuXZtd8vOhk8/dV9ICxe6L9v58111C0D//vlfxLt2wRVFXDosWuTaY7xJwftlWamSO0abNi6Z9OnjfldeRx8NF1+8gapVm9K9e35VRLNmLpZ27fKT+1tvuf9Bf1fSrVrBzJmB/TxOPjmw/SpXPvQLuyL67bffiIuLo3VrV325ePFimjVrRrVq1bjhhhsYNmwYkyZNIjExkZycHDK9/S8L2LZtG0OHDmXYsGEU7IN/5ZVX8uijj/Lpp59y9tlnAzBnzpzDrsjT0tI42vPHNGXKlLzta9asoWXLlgwfPpz169ezZMkS2rZtS506dbjqqquoXbs2r7zyCoHq378/zz//PM8//zwiwqJFi4pv8N2xw/0DgfuD8G2RjpCYTwo//gj33JNf5G/SxNWnXnNN7JYMvNU0O3a4xj6vG25wX/SqruomI8OVdgFefhkGD3Yl4OXL4f77C//sefPyH1ev7pLEzp3QurW7ku7e3W2vUgVatszf9733Du3pUpTGjeHWW1eTnOz/G7duXb+7mDJIT0/ntttuY/fu3VSqVIljjz2WyZMnA67u/cEHH6Rjx47UrFmTqlWrMmjQIBo3bgzAgQMH6NKlC1lZrkvq1VdfzR133HHYMapWrcqMGTMYMWIEI0aMICEhgU6dOvHcc8+x3adB5Z///CeDBg1izJgxeckD4P333+fNN98kISGBhg0bct999zF//nzuuusu4uLiSEhI4KWXAh9X++CDDzJixAg6depEbm4uLVq0YMaMYrrEVqrkSgbNmrk/+vJAVaPq1r17dy2tWbNmHfL8qafye6XXqaM6bpzq/v2l/vhyqeA5F7Rtm+rgwaq9eqk2bap6xBH5PxNQTU/P3/eYYw59zff2f/+Xv9/nn6tefLHqQw+p3nef6iefqK5YoZqREZpzLMjfOVdEhZ3zihUrwh9IGO3ZsyfSIZRcbq7q5s2qmzYdui1AgZ5zYb97YIEG8B0bo9fCsHUreKsJ77nH3WrVimxMoaDqrua/+so1gn73Hfztb+7KHlwdf1Gl4/r14euv4fzz3fNHHnENvO3auav4xETX06RANS/9+7ubMcbH/v2wdq27P/LI/J4F5ax+OmaTwmOPuS6nZ5/tGpKjnXdMRMuWrs79qqtclYvrAHaoX3/NTwoNGrguj/Hx7su+QwfXyFu79uHHGDgwlGdgTAWVm+t6JWzZ4v7RWrZ0SaGcJQOvmEwKqalu7AG4LorRJivL9dD55htXR79+vestde65MH26q6YcOxb+/BN++sm9p2VLV2/frZvr3uhVrZpLkMaYEPGO7KtTxzUml/PGyvIdXYiMGeN+T5dc4rolRpMXX4RbC+nIGx8Pv/+e/7xFC3jzTVi16lsGDAi8n7UxJghyclzRvW5dV8fasWP5aUj2I+aSwpo18OqrblBMcaM2Iy093dX/P/20+3t6+mm3vUcPd1+3rrv6P+ccN3K1a9dDq3zi4ly3zU2bCu/3bYwJkT17XNuBd9h31apRkxAgBpPC2LGuzv2aa1wdenmyYwe89BL83//lT5kLrpF49Gg39ULPnjBnjmssLqdVksbEpuxsVze9fbvrhXHccS4hRJlQLrJTLn37rbsfMSKycYDrAeU1f76buOvBBw9NCB06wOOPu4QALhH07m0JwUSH+Ph4unTpQseOHTn33HPZXdoZIgtYu3Ytx/s2jhWhefPmJCUl0aVLF7p06cIPP/wQlOMfRpUpjz3GppUr3WRN7dsT8oUPQiSmkkJOjivVgUvikZCd7Xr+tGjhev5450/v0cO1Q7nBV260rqqbOO2f/4xMrMaUVdWqVVm8eDHLli2jTp06TJgwIewxzJo1i8WLF7N48WJOOumkgN6T7Tv5VHGysvK6lk75/HM21ajhRsAGYzbLCIneyEth+/YqZGW5RO47pXA4ZGa6Ub6VK7vJ1rzJyXvhIuJKDhs3wgsvuKkVjKlITjzxRDZudLPjp6en07dvX7p160ZSUhIfeya2Wrt2Le3atePGG2+kQ4cOnHHGGRw4cACAhQsX0rlzZzp37lym5KKq3HXXXXTs2JGkpCTec323SUlJoXfv3px33nm090xR+9Zbb9GrVy+6dOnCTTfdRE5ODjk5OVx77bV0bNeOpPbteWbMGKZNm8aCxYsZOHgwXbp0yYs5KgUywq083coyovmZZ35WUD3ppFJ/RKm8+qpq/fqHjgC++WbVDRtCf2wb3Rsb/I1oLmokellv/lSvXl1VVbOzs/Xiiy/Wzz77TFVVs7KyNC0tTVVVt23bpq1atdLc3Fz9888/NT4+XhctWqSqqpdccom++eabqqqalJSks2fPVlXVO++8U9u1a+f3+M2aNdOOHTtq586dtVevXqqqOm3aNO3Xr59mZ2frli1btGnTprpp0yadNWuWVqtWTdesWZP38zvnnHM0MzNTVVVvvvlmnTp1qi744Qftd9JJqvPnq65cqbs8o5P79Omj8+fP9/9DKQMb0Rxkmze7Rp9A59ovi7Q0N0JaNX/xkebN3fxAw4dHbXWjMSXincNo48aNtGvXjtNPPx1wF6P33Xcfc+bMIS4ujo0bN/KXZ576Fi1a0MXTV7x79+6sXbuW3bt3s3v3bk455RQArr766rz1DvyZNWsW9erVy3v+3XffccUVVxAfH0+DBg3o06cP8+fP54gjjqBXr1608HxBfPPNNyxcuJCePXvmnUv9GjU495hjWLNuHbe99BJnX3wxZ1Sw4fsxVX20ebObDtN3orVg+/1310PoqKPcQEYRuO8+N/vq6tWuCskSggm3UJUV/PG2Kaxbtw5Vzav2efvtt9m2bRsLFy5k8eLFNGjQgIwMt652FZ/um/Hx8YHX7weB7/TWqsqgQYPy2iN+++03Rt9/P0c2bswvv/xC8oABTJw0icGDB4ctvnCIsaTgSgqhSgozZrgG7AULXPuTt6cTuIVJorjtyZgyqVatGuPHj+epp54iOzubtLQ06tevT0JCArNmzWKdd/roItSuXZvatWvz3XffAS6plFbv3r157733yMnJYdu2bcyZM4devXodtl/fvn2ZNm0aW5cuhU2b2LlzJ+t272b7kUeSm5DARRddxJgxY/j5558BqFmzZliWywy1GKs+Ck1JYe9euPhi+PJL97xGDTfwrHPn4B7HmGjWtWtXOnXqxDvvvMPAgQM599xzSUpKokePHrT1XRSjCK+//jrXX389IsIZZ5yRt33Tpk0MHjyYmQEuenHhhRcyd+5cOnfujIjwxBNP0LBhQ3799ddD9mvfvDljhg7ljIsuIjcujoTq1ZkwYQJVq1bluuuuIzc3F4BHPZOnXXvttQwdOpSqVasyd+5cqkbhGAUA0UDKgOVIjx49dMGCBaV6b8OGGfz1VyJr1gSvXWHLFjduYNUq9/zMM938QwkJwfn8skpJSclbZSpW2Dk7K1eupF15G6EZRN6V24IuN9et67pli/tHPuaYiC2NWVCg51zY715EFqpqD3/vjamSwoEDbvku70CwYPjrL7coTK1aMHEiXH558D7bGBMBBw+6f+x69dyYg3I+gV2wxdTZepNCUUullsTu3W6uoc6d3UykO3e68Q/GmCgUxRPYBVvMNH1mZUFWVhzx8WX/Xb/3nitNekfsV65sCcGYqJWW5taQ/fNPt84sxGxCgBhKCvv2ufsaNco2b9DmzflVRHffXfa4jDERkp3tEsEff7iugW3bRuUEdsEWM9VH6enuvixVRxs3wllnuccdOkAEpnExxgSDqluCMCMDGjVyN+szDsRQUvAtKZTG2rVu1bJdu1zV0bvvxlz7kzHRLyvL/eOKuEbkypXDPxFaORczqbEsJYXMTLeW865drv3p55/dvTGmeJGeOjuPqlvnYNkydw+up0ghCeGNN97Imyyva9eujBs3DnDjEKZNmxaM8Nm0aRMXX3xx3vMrrriCTp068cwzzzBq1Ci+/vrroBynNGLmWtebFEpTUli1yi2m1KiRG5RWq1ZwYzOmovJOcwEwaNAgJkyYwP333x/eIA4ehHXr3D9xjRrFzjPz2Wef8eyzz/Lll1/SuHFjDh48yBtvvBH0kBo3bpyXYLZs2cL8+fNZ5R3sVELZ2dlUCmK1RcyUFLzVR6UpKbRv7y4wvvrKEoIxpRWqqbMzMjK47rrr8q7sZ82aBcCUKVO4YMAATu/Th+bJybzw1Vc8/emndD3xRE444QR27tx5WIyPPvoo48aNo3HjxoCbh+nGG288bL+HH36Ynj170rFjR4YMGYJ3EPD48eNp3749nTp14nJPj5TZs2fnLfLTtWtX9u7dy9q1a+noqW4444wz2LhxI126dOHbb789pESycOFC+vTpQ/fu3enfvz9btmwBIDk5mREjRtCjRw+ee+65Mv5mDhUzSaE0JYWMDDe4EVwy6NAh+HEZEzbJyYffXnzRvbZ/f+GvT5niXt++/fDXSiAnJ4dvvvmG8847D4DExEQ++ugjfv75Z2bNmsXIkSPzvlj/+OMPbr31VpYvX07t2rX54IMPALjuuut4/vnn+eWXXw757AkTJiAiLF26lHfeeYdBgwblTa637Ndf+fCll5j/00/c/9hjVKtenUWLFnHiiScWWgJYtmwZ3bt393s+w4YNY/78+SxbtowDBw4wY8YMAB577DEWLVrEkiVLmDhxIgDjxo1jwoQJLF68mG+//faw6S+mT59Oq1atWLx4Mb17987bnpWVxW233ca0adNYuHAh119/PQ8//HDe65mZmSxYsICRI0f6jbckAk4KInKEiBwnIscENYIw8a5wVpI2pdGj4bzz3NrJxpiS806d3bBhQ/7666/Dps7u1KkT/fr1K9XU2V7fffcdV111FQBt27ShWaNG/O6ZOO/Uvn2p2bUrRzVpQq1atTj33HMBSEpKYq13patSmDVrFscffzxJSUn873//Y/ny5QB06tSJgQMH8tZbb+VV6Zx88snccccdjB8/nt27dwdc1fPbb7+xbNkyTj/9dLp06cKYMWPySloAl112WanjL06x0YlITeBm4EqgBrAdSBSRusB3wIuq+m0xH1FueGffDbTqLTUVnnjCtU/NmwcDBoQuNmPCIiWl6NeqVSv+9Xr1in+9CN42hf3799O/f38mTJjA8OHDD5k6OyEhgebNmxc5dXbAq5jt2+e6CWZmupvqIZ8VFxeX9zwuLq7QKbk7dOjAwoULOe2004o8TEZGBrfccgsLFiygadOmjB49Oi/2Tz/9lDlz5vDJJ58wduxYli5dyj333MPZZ5/NzJkzOfnkk/niiy9ITEz0ezqqSocOHZg7d27eNt9ZWKsHY2qGQvgrKXwEbAP6quqxqnqCqnYBmgHPApeJyPVFvVlEzhSR30RklYjcU8jrx4jILBFZJCJLRCRkX705Oe4+kKSQkQF9+7qEkJxsCcGYsgrl1Nm9//Y33p48GVau5PfVq1m/fTvH9e1bqlGq9957L3fddVde3X1mZiavvPLKIft4E0C9evVIT0/Pq//Pzc1lw4YNnHrqqTz++OOkpaWRnp7O6tWrSUpK4u6776Znz56HzcZalOOOO45t27blJYWsrCxWrlxZ4nMqqWK/IlW1XxHbc4F5nluhRCQemACcDqQC80Vkuqqu8NntAeB9VX1JRNoDM4HmJTqDAHkvCuLj/e87YYJbLCchAULQ8cCYmBSqqbNvuf56bp49m6SrrqJSYiJT3njjkBJCSQwYMIC//vqLfv36oaqICNdff+h1b+3atbnxxhvp2LEjDRs2zFuZLScnh6uuuoq0tDRUleHDh1O7dm0efPBBZs2aRVxcHB06dOCss85i8+bNfmOpXLky06ZNY/jw4aSlpZGdnc3QoUMLXfshqIpbqxPoVNzNz3tPBL7weX4vcG+BfSYBd/vs/4O/9UNLu0bzc8+5taJuu634/XbsUD3ySLfvxImlOlS5YusVxwZ/azRXONnZut93kfOMjMjFEkblYY3m4iZyUOCUYl4/Gtjg8zwVKDjaZDTwpYjcBlQHCi2ZBEOgJYUnn3SD1GrWBE/blTGmPElLg3XrSMzMzJ/VNIYnsAs2f9VHvYt7PQiuAKao6lMiciLwpoh0VFc9lUdEhgBDABo0aEBKKRq8fv+9KdCKTZs2kJKyusj9GjWqxYUXHsVJJ+1g/vxdJT5OeZOenl6qn1c0s3N2atWqVSGWh/SSnByqbN1Kwp495FSuzP4mTdzVXgU6R39ycnIC+p1mZGSU+n/AX++j84p7XVWnF/PyRqCpz/Mmnm2+bgDO9HzWXBFJBOoBWwscZzIwGdzKa6VZVcvbgN+iRVOSk5sWuV9yMgwf7g03+tkqZLGhqJXXatSogZRlWuDyQtVNb33wIDRqRHyjRrBvX2hWXivHAll5TVVJTEyka9eupTqGv+qjS4o7NlBcUpgPtBaRFrhkcDmua6uv9UBfYIqItAMScb2dgs7b+6io6qPcXNdZoSL8/xgDboDYjh07qFu3bvQmBpvArkRUlR07dgTU5bUo/qqPri7udT/vzRaRYcAXQDzwmqouF5GHcQ0e04GRwMsicjsuyVzraRAJOn/jFCZMgGeegbvugptvDkUExoRXkyZNSE1NZdu2kFxnhV56ulvS8MgjC52vKCMjo0xfftEokHNOTEykSZPS13QEPIuSiPQHOuCu5gFQ1UeKe4+qzsR1M/XdNsrn8Qrg5EBjKIviSgqqroF5w4bAuqwaEw0SEhJo0aJFpMMouTVr4MYb4X//gz594JVX4NhjD9stJSWl1FUk0Soc5xxQUhCRF4HauN5GrwMXAT+GMK6gK66ksGKFSwhVqsC114Y1LGOMr6lT4ZZb3NXZxIkuOdjiN2EV6E/7b6p6JbBDVR/EdS09PHWXY8WVFLyN0AMGuCpLY0yENG4Mp53mrtRuuskSQgQEWn3knXwkQ0QaAjuAxqEJKTSKm+biww/d/amnhi8eYwxujqLHHnM9PUaPhtNPdzcTMYGm4c9EpDYwDlgMrAX+E6qgQqGowWt798KXX7rHngkUjTHhMH8+dO8O//qXa0cITR8TU0IBlRRUdbTn4X9EZAZQVVUPX6GiHPOui1CwNPr9964U0a0bNG8e9rCMiT3798OoUa67X6NGMH26XZGVIwGVFERkqKekgKoeANQzyjhqeC9CCnbX7t8fNm8+rwpGAAAgAElEQVSG994Lf0zGxKQ//4Tnn3eNyMuXW0IoZwKtPhqqqnkrbqvqLtw6C1HDmxQKlhREoGHDQnu8GWOCJS0NXn/dPe7QwS18PnGirW9bDgWaFA6piReROCAh+OGEjrf6KFoHdhoTtT791CWCwYPBu5ZA06KnmjGRFWhS+EpE3hGRPiLSB3gb+DqEcQVdYdVHa9e6EsLAgREJyZiKbds29891zjluVPLcuRDAugkmsgLtknoXcAtwu+f5V7i1EKJGYUnht99g9Wo4+ujIxGRMhZWTA3/7m2s/eOghuOceGwQUJQLtfZQjIpOAz1R1VYhjConCksKSJe6+Q4fwx2NMhbRlC9Sv7/p+P/WU69LXsWOkozIlEGjvo3OApbgSAiLSRUQ+CmVgwVZYQ7M3KXTpEv54jKlQcnNh0iRo08bdg6s2soQQdQJtU3gIN7XFbgBVXUyUTXNRWEPzPM8K0zE2p5YxwbVqFfTtC0OHQs+erp+3iVqBJoUs3y6pHlE1/LBg9dH+/fDHH5CQYCUFY0rt9dchKQl+/hlefhm+/hpatox0VKYMAm1oXikilwJxnkVzhhNls6QWTAqbNrn7Ro1cYjDGlMIxx7iSwYQJ1mOjggg0KQwDRgG5wIfAl8D9oQoqFAq2KRxxBIwbZwnBmBI5eBAefdTVxz78sKs26ts30lGZIAq099E+4G7PDQARORrYH6K4gq5gm0L9+jByZOTiMSbqzJsHN9zgpqYYNMhdadlo0ArHb5uCiPQUkQtEpJ7neQcReQOYF/LogqiouY+MMX7s2wd33AEnnuimq5gxA6ZMsX+mCqrYpCAij+JGLw8EPheR0cAs4BegTcijC6KCSeGtt1zPue3bIxeTMVFh3Tp48UXXu2j5cjj77EhHZELIX/XR+UBnVT0gInWADUCSqq4JfWjBVbBN4ZFHYOVKN+iyXr3IxWVMubR7N0yb5uYrat/edTstw2LwJnr4qz7K8EyVjWf9hN+jMSHAoW0KubluTWZwM6QaY3x8/LFLBEOH5k9gZwkhZvgrKbQUEc9ilQjQwuc5qvr3kEUWZL7VR1u3Qnq6m6Orbt3IxmVMufHXXzB8OLz/PnTqBJ98YhPYxSB/SeGiAs9fCFUgoeabFLzTW9gYG2M8vBPYrV8PY8bAP/9p/bVjVLFJQVW/CVcgoeabFDZuzH9sTEzbtMnVocbHw3PPuQns2rePdFQmgvz1PvqviJwlIoclDxFpJiKjROT60IUXPL5JwTua2cbcmJiVmwsvveSqhyZOdNsGDLCEYPxWH90KjAQmiMhfwDYgEWgJrAcmqOoHoQ0xOHx7H+XkuLYEG5VvYtLvv7v1kefMgX794KyzIh2RKUf8VR9tBO4A7hCRY4FGwAHgN1XdG4b4QmLUKHczJua8+ioMGwaJifDaa3DttVaPag4R6NxHeBbXicoFdowxHs2bu5LBhAluNkhjCgh06uyopz4Tff/yy6HPjamwDh6EBx5wN3ANaR9+aAnBFClmkoKXiGtPi4tz07gYU2H98INbLGTsWNi82a6ETEACTgoiUtnTrhDVsrPzex/VrBnZWIwJifR0+Mc/3LiD/fvh889dW4K1HZgABLpG89mUYo1mETlTRH4TkVUick8R+1wqIitEZLmI/LskwZeE9yIpPd3d16t36HrNxlQY69e72R5vvRWWLbPlMU2JBNrQ/DBujeZZ4NZo9ldqEJF4YAJwOpAKzBeR6aq6wmef1sC9wMmquktE6pfiHEpkr6fP1IEDoT6SMeFTae9emDwZhgxxYw3WrIHGjSMdlolCgSaFLFXdLYcWP/1VUPYCVnkn0BORd3Gzrq7w2edG3FiHXQCqujXAeEpt3z5336BBqI9kTJh89BE9Bw92jWR9+sBxx1lCMKUWyjWaj8ZNte2Viitt+GoDICLfA/HAaFX9vOAHicgQYAhAgwYNSElJCTDsfNu2dQCOYtmydUAzGjTYTkrKshJ/TrRJT08v1c8rmsXKOVfeuZNjx4+n/uzZZLRsydJHHyV982bXqBwDYuX37Csc51zaNZq/AO4L0vFbA8lAE2COiCSp6m7fnVR1MjAZoEePHpqcnFziA3nXTKhbtxkArVrVozSfE21SUlJi4jx9xcQ55+S4KSo2bIBHHmFxz5706dcv0lGFVUz8ngsIxzkHmhT6q2rBNZr/jksQRdkINPV53sSzzVcqME9Vs4A/ReR3XJKYH2BcJdaqFYwbBx06hOoIxoRQaqqrGoqPh/HjoUULaNsWjbErZhM6gfa/eaCQbff7ec98oLWItBCRysDlwPQC+/wXV0rAswZ0GyCki/g0bw4jR8KZZ4byKMYEWW4uPP+8Kx289JLbdtZZtt6BCbpiSwoi0h84EzhaRJ72eekIXFVSkVQ1W0SG4aqa4oHXVHW5iDwMLFDV6Z7XzhCRFUAOcJeq7ij96RQXTyg+1Zgw+PVXtyzm99+77qXnnBPpiEwF5q/6aCuwDMgAlvts3wsUOu7Al6rOBGYW2DbK57HimXAvwHjLbMUKN57n5JNdqcGYcu2VV9wEdtWqwdSpcPXVNgjNhJS/WVIXAYtE5G1VzQhTTCH16acwdy68844lBRMFWrWCc8+FF16wftQmLAJtaD5aRMYC7XHrKQCgqm1CElUIeKuPdu1y9/b/ZcqljAx4+GH3+JFH4NRT3c2YMAm0oXkK8DogwFnA+8B7IYoppLwjmuuHfOy0MSX03XduArtHH4Vt26whzEREoEmhmqp+AaCqq1X1AVxyiDreEc21a0c2DmPy7N3r2g1693YlhS+/hJdftrYDExGBVh8dFJE4YLWIDMWNN4iqOUa9F127PcPiLCmYciM11c1iOny4m+a6Ro1IR2RiWKBJ4XagOm56i7FALeD6UAUVDtWqRToCE9N27ID334ebb4Z27dwEdrbwjSkHAkoKqjrP83AvcDWAiETlsvcibjColcxNRKjCBx+4aa137oTTTnMT2FlCMOWE3zYFEekpIhd4RhwjIh1E5A1gnp+3live6qOPPsqvQjImrDZvhosugksugaZNYcEClxCMKUeKTQoi8ijwNjAQ+FxERuPWVPgFzwyn0SYuDqpXj3QUJubk5LiG5M8+gyeegB9/hM6dIx2VMYfxV310PtBZVQ+ISB3cVNhJ3jUSjDF+bNgARx/t6iwnTHAT2LWJyuspEyP8VR9lqOoBAFXdCfwe7Qnhtttc254xIZWT42Yx9Z3Arn9/Swim3PNXUmgpIt7psQVo4fMcVf17yCILMm+bwrp18OefkY3FVHArV8INN7j5VM46y01TYUyU8JcULirw/IVQBRJOCQmRjsBUWJMnu+JozZrw5pswcKB1dTNRxd+EeN+EK5Bwqlw50hGYCqt1a7jwQld1ZHOpmCgU6OC1qOc7jUyMLGFrwuHAARg92pUGHnvMJrAzUS/QuY8qlBYtIh2BqRDmzHHdSp94AtLSbAI7UyGUKCmISJVQBRJO9epFOgIT1fbsgVtugT59XC+jb75xPYys7cBUAAElBRHpJSJLgT88zzuLyPMhjSxEzjnHjSEyptQ2bYIpU+COO2DJEjdVhTEVRKAlhfHAOcAOAFX9BYjKitOhQ+HiiyMdhYk627fDiy+6x23bun7NTz1lw+NNhRNoUohT1XUFtuUEOxhjyh1VeO89aN8eRoyA3393223pPlNBBZoUNohIL0BFJF5ERgC/hzCukFm+3C1qZYxfmzbBBRfA5ZdDs2awcKGNSDYVXqBJ4WbgDuAY4C/gBM+2qHP33fD555GOwpR7OTlwyiluFbRx49zo5KSkSEdlTMgFOk4hW1UvD2kkYVSlQvShMiGxbh00aeImsHvxRWjZEo49NtJRGRM2gZYU5ovITBEZJCJRtQxnYapWjXQEptzJyYGnn3aroHknsDvjDEsIJuYElBRUtRUwBugOLBWR/4pI1JYcEhMjHYEpV5Ytg5NPhpEjoW9f145gTIwKePCaqv6gqsOBbsAe3OI7UcnmPjJ5Jk6Ebt1g9Wr4979h+nRXfWRMjAp08FoNERkoIp8APwHbgJNCGlkIWVIweVNStGvnlsdcsQKuuMJGJZuYF2hD8zLgE+AJVf02hPGEhSWFGLZ/P4wa5RqSH3/cTVXRp0+kozKm3Ag0KbRU1dyQRhImL70EHTpEOgoTESkpMHiwqyq65RZXWrCSgTGHKDYpiMhTqjoS+EBEDpsCMppWXvNq2tQammNOWhr8859uAZxWreB//7PprY0pgr+Swnue+1KtuCYiZwLPAfHAK6r6WBH7XQRMA3qq6oLSHMuYIm3eDG+9BXfeCQ89BNWqRToiY8qtYhuaVfUnz8N2qvqN7w1oV9x7RSQemACcBbQHrhCR9oXsVxP4BzCvNCdQUuPGuZmPTQW3bRs875nIt21bWLsWnnzSEoIxfgTaJfX6Qrbd4Oc9vYBVqrpGVTOBd4HzC9nv/4DHgYwAYymTlBTIzg7HkUxEqFL/669dr6KRI/MnsDvqqMjGZUyUKDYpiMhlIvIR0EJEPvS5fQXs9vPZRwMbfJ6nerb5fn43oKmqflqK2EvNeh9VUBs2wLnn0n7sWDcSedEim8DOmBLy16bwE24NhSa4qiCvvcCishxYROKAp4FrA9h3CDAEoEGDBqSkpJT4eDt2JAF1AZg7dzYJCbGxdGJ6enqpfl7RRnJy6HXNNVTeuZNfBw9m2+WXuyqkGDh3iJ3fsy8759AoNimo6p/An8DXpfjsjUBTn+dNPNu8agIdgRRx3QIbAtNF5LyCjc2qOhmYDNCjRw9NTk4ucTB16+Y/7tevT8z0RExJSaE0P6+osXat61IWHw9Tp0LLlmxbv75in3MhKvzvuRB2zqHhr/potud+l4js9LntEpGdfj57PtBaRFqISGXgcmC690VVTVPVeqraXFWbAz8ChyWEYPEOYI2Ls67pFUJ2tus10K5d/opo/fq5WU2NMaXmr/rI25m7xEvdq2q2iAwDvsB1SX1NVZeLyMPAAlWdXvwnBFdu7qH3JootWQI33AALFsD558NFF0U6ImMqDH/VR96v0KbAJlXNFJG/AZ2At3AT4xX3/pnAzALbRhWxb3KAMZdJ27bhOIoJmRdfhH/8A4480i2TecklVvQzJogC7ZL6X9xSnK2A14HWwL9DFlUIxMe7+3HjIhuHKSVv/V/Hjm55zBUr4NJLLSEYE2SBzn2Uq6pZIvJ34HlVHS8iZep9ZExA9u2DBx6ASpXc4LNTTnE3Y0xIBFpSyBaRS4CrgRmebQmhCSk0vBeaGWEZImeC4ptv3LrIzz4LBw/m/xKNMSFTkhHNp+Kmzl4jIi2Ad0IXVvAdOODub7klsnGYAOze7WYz7dfPlRDmzIHx462qyJgwCHQ5zmXAcGCBiLQFNqjq2JBGFmTeXkeVAq0wM5Hz11/w7rtw993wyy/Qu3ekIzImZgT0FSkivYE3cYPPBGgoIler6vehDC6YvEkhIaoqvWKINxH84x9w3HFuUFq9EveENsaUUaDXzc8AA1R1BYCItMMliR6hCizYvNXRVlIoZ1TdtNYjRkB6OgwYAK1bW0IwJkICbVOo7E0IAKq6EoiqaeW8JQVv11RTDqxfD2efDddc40oHixe7hGCMiZhAr5t/FpGJuAFrAAMp44R44WYlhXImOxuSk1210bPPwrBhlrGNKQcC/Yocimto/qfn+bfA8yGJKESysg69NxGyZg00a+ay88svu7mKWrSIdFTGGA+/1UcikgScCXykqud5bk+qalT1+K9d292feWZk44hZ2dnw+OPQvj1M8MzC3revJQRjyhl/s6Teh5viYiDwlYgUtgJbVKhSxd337x/ZOGLS4sVw/PFwzz2uIfmSSyIdkTGmCP5KCgOBTqp6CdATuDn0IZkK5YUXoGdP2LgRpk2DDz+ERo0iHZUxpgj+ksJBVd0HoKrbAti/3EpPd/dz50Y2jpjhbdnv1AkGDnQT2NkU18aUe/4amluKyIeexwK08nmOqv49ZJEF2d697j7GVu8Lv/R0uP9+N0pw3DibwM6YKOMvKRS8tHshVIGEmvfC1Xo9htCXX8KQIW78wW23uR+6zVdkTFTxt8jON+EKJNS8XVHjorYCrBzbtQvuuAOmTHGD0ObMgb/9LdJRGWNKIea+InfsiHQEFdDWra4R+d57XU8jSwjGRK2YGd/rLSF4xyuYMtqyBd55B26/PX8Cu7p1Ix2VMaaMSlRSEJEqoQok1LxtCtWqRTaOqKcKU6e6QWj33gt//OG2W0IwpkIIKCmISC8RWQr84XneWUSiapoLb0khMTGycUS1tWvdkPBrr3VJwSawM6bCCbSkMB44B9gBoKq/4FZiixpNmrj7G2+MbBxRKzsbTj0VfvjBTVMxZw60bRvpqIwxQRZom0Kcqq6TQ7sX5oQgHlPerFrl5ieqVAlee81NYNesWaSjMsaESKAlhQ0i0gtQEYkXkRHA7yGMy0RaVhY88gh06JA/gd2pp1pCMKaCCzQp3AzcARwD/AWcQJTNg7Rhg7t/553IxhEVfv4ZevVyI5PPPx8uuyzSERljwiSg6iNV3QpcHuJYQso7eG3fvsjGUe6NH+8Goh11lJu87sILIx2RMSaMAkoKIvIyoAW3q+qQoEcUYjaiuQjeKSm6dnXLYz71FBx5ZKSjMsaEWaANzV/7PE4ELgQ2BD+c0PGOU7CkUMDevW68QZUqLhH07u1uxpiYFGj10Xu+z0XkTeC7kEQUIhmedeJsfjYfn30GN90EqakwYoRNYGeMKfXcRy2ABsEMJNQqedLfzp2RjaNc2LHDVRENGAA1asD338PTT1tCMMYE3Kawi/w2hThgJ3BPqIIKBW+1kVWT45LCf/8LDzzgblWidvYSY0yQ+U0K4kasdQY2ejblquphjc5FvPdM4DkgHnhFVR8r8PodwGAgG9gGXK+q6wIPP3DeZNClSyg+PQps3gxvvw0jR0KbNrBunWVIY8xh/FYfeRLATFXN8dwCTQjxwATgLKA9cIWItC+w2yKgh6p2AqYBT5Qo+hKoV8/dx1xSUHUjkdu1gwcfdCOUwRKCMaZQgbYpLBaRriX87F7AKlVdo6qZwLvA+b47qOosVd3vefoj0KSExzDF+fNPOt11F9xwA3TuDL/8YhPYGWOKVWz1kYhUUtVsoCswX0RWA/tw6zWrqnYr5u1Hc2i31VTg+GL2vwH4rIg4hgBDABo0aEBKKRZa3rixM3Aks2f/SvXqW0r8/mgjOTn0uuoqaqal8fvtt7PpnHNg0yZ3q+DS09NL9TcSzeycY0M4ztlfm8JPQDfgvFAGISJXAT2APoW9rqqTgckAPXr00OTk5BIfY+9ed79tW1uSkyvw7J5//OEmrYuPh3feYe7WrZx46aW0iXRcYZSSkkJp/kaimZ1zbAjHOfurPhIAVV1d2M3PezcCTX2eNyG/sTr/ACL9gPuB81T1YAliL5HAWkKiWFYWjBkDHTvCCy+4bcnJHKxfP7JxGWOiir+SwlGeHkKFUtWni3nvfKC1iLTAJYPLgSt9d/C0U0wCzvTMrxRyFXJE84IFrt1gyRK4/HK44opIR2SMiVL+kkI8UANPiaEkVDVbRIYBX3g+5zVVXS4iDwMLVHU68KTn8//jWathvaqGuKoqlJ8eAc895yawa9gQPv4Yzgvpj88YU8H5SwqbVfXh0n64qs4EZhbYNsrncb/SfnbJY3H3FSYpeKek6NHDlRKeeAJq1450VMaYKOcvKVSUr9CKMyHenj1w991uselnnoGTT3Y3Y4wJAn9fkX3DEkUYRXVJYeZMtxLa5MluMqcK33pujAm3YpOCqlaY6ePaePpk9o3GNLd9O1x1FZx9NtSqBT/8AE8+GeUZzhhTHkV7ZUrAEhLcfbVqkY2jVHbtgk8+gX/9yy2VeXxxYwCNMab0Al1kx4Tbxo1uAru77nJTU6xbZw3JxpiQi5mSwjrP3KuLFkU2Dr9U4eWXoX17GD0aVnvGCFpCMMaEQcwkhbQ0d781LEPkSmn1atfoMWQIdOvmBqMde2ykozLGxJCYqz4qt22z2dkuIezcCZMmweDBFaD/rDEm2sRcUih3fvsNWrVyXUynTnWPm9gM4saYyLBL0UjJzISHHoKkJJgwwW3r08cSgjEmomKmpFCuprmYPx+uvx6WLYMrr4SBAyMdkTHGADFYUoh4Unj2WTjhhPyxB2+/nb9WqDHGRFjMJIWaNd193boRCsBbVOnVC268EZYvh3POiVAwxhhTuJhJCi1buvuuJV1puqzS0uCmm+D2293zk06CiRPddBXGGFPOxExSiIhPPnGD0F55BapUsQnsjDHlXswkhexsd5+ZGYaDbdvmGpDPO8/VV/34Izz+eDlo0DDGmOLFTFJYtszdf/99GA6WluamuX7oIbdUZs+eYTioMcaUXcx0SfUK2cX6hg3w1ltwzz1uaop166zdwBgTdWKmpBCy6vzcXNdw3KEDjBmTP4GdJQRjTBSKmaTgFdSSwh9/wGmnwc03u66mS5faBHbGmKgWc9VHQZOdDaefDrt3w6uvwnXXWUOyMSbqxUxSCNo0FytXukVvKlWCN990E9g1blzm+Iwxpjyw6qNAHTzolsPs1AleeMFt693bEoIxpkKJmZJCy5awZQu0a1eKN//4I9xwA6xYAVdf7W7GGFMBxUxJwbuaZYnnnnvqKTc1xd69buzBG29EcAIlY4wJrZhJCiWWm+vuTzwRhg51o9/OOiuyMRljTIjFTPXR5s3ufsMGPzvu3g0jR0K1avD8866UcNJJIY/PGGPKg5gpKWza5O6LTQr//a+bwG7qVDfXtk1gZ4yJMTGTFIrtkrp1K1x6KVx4ITRoAD/9BI88YuMOjDExJ2aSgleh3/N79sBXX8HYsS4hdOsW9riMMaY8CGlSEJEzReQ3EVklIvcU8noVEXnP8/o8EWkeynjcMT0P1q93SUDVTU2xfj3cdx8kJIQ6BGOMKbdClhREJB6YAJwFtAeuEJH2BXa7AdilqscCzwCPhyqevOojzYUXX3QT2D3ySP4Edt71Oo0xJoaFsqTQC1ilqmtUNRN4Fzi/wD7nA1M9j6cBfUVCV5Hfht+47o1kuPVW19V0+XKbwM4YY3yEMikcDfj29Un1bCt0H1XNBtKAkIwMS6yUzRf0p8HWpfD66/DFF9C8eSgOZYwxUSsqximIyBBgCECDBg1ISUkp8We0bteOm9Je44KRubRrXglmzw5ylOVTenp6qX5e0czOOTbYOYdGKJPCRqCpz/Mmnm2F7ZMqIpWAWsCOgh+kqpOByQA9evTQ5OTkEgeTnAwpKStJTj6txO+NZikpKZTm5xXN7Jxjg51zaISy+mg+0FpEWohIZeByYHqBfaYDgzyPLwb+p2ojxowxJlJCVlJQ1WwRGQZ8AcQDr6nqchF5GFigqtOBV4E3RWQVsBOXOIwxxkRISNsUVHUmMLPAtlE+jzOAS0IZgzHGmMDF3IhmY4wxRbOkYIwxJo8lBWOMMXksKRhjjMljScEYY0weibZhASKyDVhXyrfXA7YHMZxoYOccG+ycY0NZzrmZqh7lb6eoSwplISILVLVHpOMIJzvn2GDnHBvCcc5WfWSMMSaPJQVjjDF5Yi0pTI50ABFg5xwb7JxjQ8jPOabaFIwxxhQv1koKxhhjilEhk4KInCkiv4nIKhG5p5DXq4jIe57X54lI8/BHGVwBnPMdIrJCRJaIyDci0iwScQaTv3P22e8iEVERifqeKoGcs4hc6vldLxeRf4c7xmAL4G/7GBGZJSKLPH/fAyIRZ7CIyGsislVElhXxuojIeM/PY4mIdAtqAKpaoW64abpXAy2BysAvQPsC+9wCTPQ8vhx4L9Jxh+GcTwWqeR7fHAvn7NmvJjAH+BHoEem4w/B7bg0sAo70PK8f6bjDcM6TgZs9j9sDayMddxnP+RSgG7CsiNcHAJ8BApwAzAvm8StiSaEXsEpV16hqJvAucH6Bfc4HpnoeTwP6ioiEMcZg83vOqjpLVfd7nv6IWwkvmgXyewb4P+BxICOcwYVIIOd8IzBBVXcBqOrWMMcYbIGcswJHeB7XAjaFMb6gU9U5uPVlinI+8IY6PwK1RaRRsI5fEZPC0cAGn+epnm2F7qOq2UAaUDcs0YVGIOfs6wbclUY083vOnmJ1U1X9NJyBhVAgv+c2QBsR+V5EfhSRM8MWXWgEcs6jgatEJBW3fstt4QktYkr6/14iIV1kx5Q/InIV0APoE+lYQklE4oCngWsjHEq4VcJVISXjSoNzRCRJVXdHNKrQugKYoqpPiciJuNUcO6pqbqQDi0YVsaSwEWjq87yJZ1uh+4hIJVyRc0dYoguNQM4ZEekH3A+cp6oHwxRbqPg755pARyBFRNbi6l6nR3ljcyC/51RguqpmqeqfwO+4JBGtAjnnG4D3AVR1LpCImyOoogro/720KmJSmA+0FpEWIlIZ15A8vcA+04FBnscXA/9TTwtOlPJ7ziLSFZiESwjRXs8Mfs5ZVdNUtZ6qNlfV5rh2lPNUdUFkwg2KQP62/4srJSAi9XDVSWvCGWSQBXLO64G+ACLSDpcUtoU1yvCaDlzj6YV0ApCmqpuD9eEVrvpIVbNFZBjwBa7nwmuqulxEHgYWqOp04FVcEXMVrkHn8shFXHYBnvOTQA3gP5429fWqel7Egi6jAM+5QgnwnL8AzhCRFUAOcJeqRm0pOMBzHgm8LCK34xqdr43mizwReQeX2Ot52kn+BSQAqOpEXLvJAGAVsB+4LqjHj+KfnTHGmCCriNVHxhhjSsmSgjHGmDyWFIwxxuSxpGCMMSaPJQVjjDF5LCmYckdEckRksc+teTH7Ni9qNskSHjPFMxPnL54pIo4rxWcMFZFrPI+vFZHGPq+9IiLtgxznfBHpEsB7RohItbIe28QGSwqmPDqgql18bmvDdNyBqtoZN1nikyV9s6pOVNU3PE+vBRr7vDZYVSRtnVMAAANTSURBVFcEJcr8OF8ksDhHAJYUTEAsKZio4CkRfCsiP3tuJxWyTwcR+clTulgiIq0926/y2T5JROL9HG4OcKznvX098/Qv9cxzX8Wz/THJX59inGfbaBG5U0Quxs0v9bbnmFU9V/g9PKWJvC9yT4nihVLGORefidBE5CURWSBuHYWHPNuG45LTLBGZ5dl2hojM9fwc/yMiNfwcx8QQSwqmPKrqU3X0kWfbVuB0Ve0GXAaML+R9Q4HnVLUL7ks51TPtwWXAyZ7tOcBAP8c/F1gqIonAFOAyVU3CzQBws4jUBS4EOqhqJ2CM75tVdRqwAHdF30VVD/i8/IHnvV6XAe+WMs4zcdNaeN2vqj2ATkAfEemkquNxU0mfqqqneqa+eADo5/lZLgDu8HMcE0Mq3DQXpkI44Pli9JUAvOCpQ8/BzelT0FzgfhFpAnyoqn+ISF+gOzDfM71HVVyCKczbInIAWIubfvk44E9V/d3z+lTgVuAF3PoMr4rIDGBGoCemqttEZI1nzpo/gLbA957PLUmclXHTlvj+nC4VkSG4/+tGuAVnlhR47wme7d97jlMZ93MzBrCkYKLH7cBfQGdcCfewRXNU9d8iMg84G5gpIjfhVqeaqqr3BnCMgb4T5olIncJ28szH0ws3CdvFwDDgtBKcy7vApcCvwEeqquK+oQOOE1iIa094Hvi7iLQA7gR6quouEZmCmxiuIAG+UtUrShCviSFWfWSiRS1gs2eO/Ktxk6MdQkRaAms8VSYf46pRvgEuFpH6nn3qSODrU/8GNBeRYz3PrwZme+rga6nqTFyy6lzIe/fipu8uzEe41bOuwCUIShqnZ8K3B4ETRKQtbuWxfUCaiDQAzioilh+Bk73nJCLVRaSwUpeJUZYUTLR4ERgkIr/gqlz2FbLPpcAyEVmMW0vhDU+PnweAL0VkCfAVrmrFL1XNwM1A+R8RWQrkAhNxX7AzPJ/3HYXXyU8BJnobmgt87i5gJdBMVX/ybCtxnJ62iqdwM6H+glub+Vfg37gqKa/JwOciMktVt+F6Rr3jOc5c3M/TGMBmSTXGGOPDSgrGGGPyWFIwxhiTx5KCMcaYPJYUjDHG5LGkYIwxJo8lBWOMMXksKRhjjMljScEYY0ye/wcT4oEUgtFTfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under the ROC curve, AUC =  0.99\n"
     ]
    }
   ],
   "source": [
    "y_scores_forest = y_prob_forest[:,1] #read prob that image is 5\n",
    "\n",
    "fpr_f, tpr_f, thresholds_f =roc_curve(y_train_5, y_scores_forest)\n",
    "\n",
    "plt.plot(fpr, tpr, 'b--', linewidth=2, label=\"SGD Classifier\")\n",
    "plot_roc_curve(fpr_f,tpr_f, \"Rand. Forest\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Area under the ROC curve, AUC = \", np.around(roc_auc_score(y_train_5, y_scores_forest), decimals=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "################################################\n",
    "#Multiclass classification\n",
    "################################################\n",
    "\n",
    "#There are binary (e.g. SVM classifiers, sgd, linear classifiers) \n",
    "#and multiclass classifiers (e.g. Random Forest, Bayes). \n",
    "\n",
    "#Example of binary classifier. They fit multiclass data by\n",
    "#performing one-versus-one or one-versus-all classification\n",
    "#using multiple binary classifiers.The class of the classifier\n",
    "#with highest score is chosen. \n",
    "\n",
    "sgd_clf.fit(x_train, y_train)\n",
    "some_digit_score=sgd_clf.decision_function([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The scores for 'some_digit' to be 0 or 1 or...9 is:  [[  300144.79041848  -679258.32292334  -276603.41076358  -420683.44862811\n",
      "   -430181.50994591  -215777.67347907 -1002951.60026811  -280569.69802376\n",
      "   -341468.05888627  -392722.19098836]]\n",
      "The index # corresponding to max. score is: 0\n",
      "Class (or label) corresponding to this index # is: 0.0\n"
     ]
    }
   ],
   "source": [
    "print(\"The scores for 'some_digit' to be 0 or 1 or...9 is: \",some_digit_score)\n",
    "\n",
    "print(\"The index # corresponding to max. score is:\", np.argmax(some_digit_score))\n",
    "\n",
    "print(\"Class (or label) corresponding to this index # is:\", sgd_clf.classes_[np.argmax(some_digit_score)])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The forest clf fit predicts that some_digit is: [0.]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Example of a multiclass classifier\n",
    "\n",
    "forest_clf.fit(x_train, y_train)\n",
    "print(\"The forest clf fit predicts that some_digit is:\", forest_clf.predict([some_digit]))\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean accuracy score of sgd clf over 3 sets of cross validation:  0.8606683719273477\n",
      "Std. dev. :  0.012874753279149024\n",
      "\n",
      "\n",
      "Mean accuracy score of forest clf over 3 sets of cross validation:  0.9410166537265879\n",
      "Std. dev. :  0.000332182071325712\n"
     ]
    }
   ],
   "source": [
    "#Evaluate accuracy of binary and multiclass classifiers using cross validation\n",
    "\n",
    "#first check for binary clf\n",
    "scores_sgd = cross_val_score(sgd_clf, x_train, y_train, cv=3, scoring=\"accuracy\")\n",
    "print(\"Mean accuracy score of sgd clf over 3 sets of cross validation: \", scores_sgd.mean())\n",
    "print(\"Std. dev. : \", scores_sgd.std())\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "scores_forest = cross_val_score(forest_clf, x_train, y_train, cv=3, scoring=\"accuracy\")\n",
    "print(\"Mean accuracy score of forest clf over 3 sets of cross validation: \", scores_forest.mean())\n",
    "print(\"Std. dev. : \", scores_forest.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n",
      "/home/priyaroy/machine_learning/env/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean accuracy score of sgd clf over 3 sets of cross validation, after standardisation:  0.9103666238681747\n",
      "Std. dev. :  0.0022968738112583557\n"
     ]
    }
   ],
   "source": [
    "#Improve fit quality by performing transformations\n",
    "#Use standard scaler transformation to scale the intensity of each pixel\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler=StandardScaler()\n",
    "x_train_scaled = scaler.fit_transform(x_train.astype(np.float64))\n",
    "\n",
    "scores_sgd_trans = cross_val_score(sgd_clf, x_train_scaled, y_train, cv=3, scoring=\"accuracy\")\n",
    "print(\"Mean accuracy score of sgd clf over 3 sets of cross validation, after standardisation: \", scores_sgd_trans.mean())\n",
    "print(\"Std. dev. : \", scores_sgd_trans.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1, 5],\n",
       "       [2, 6],\n",
       "       [3, 7]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([1,2,3])\n",
    "\n",
    "print(x.shape)\n",
    "\n",
    "y=[5,6,7]\n",
    "z = np.c_[x,y]\n",
    "\n",
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Multilabel classification\n",
    "\n",
    "y_train_greaterthan7 = (y_train>7)\n",
    "y_train_odd = (y_train%2==1)\n",
    "y_multilabel = np.c_[y_train_greaterthan7, y_train_odd]\n",
    "\n",
    "#Fit a multilabel classifier, like the nearest neighbor clf\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "nn_clf = KNeighborsClassifier()\n",
    "nn_clf.fit(x_train, y_multilabel)\n",
    "nn_clf.predict([some_digit])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}