Pytorch From Basic To Advance Level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient of w1 w.r.t to L: -36.0\n",
      "Gradient of w2 w.r.t to L: -28.0\n",
      "Gradient of w3 w.r.t to L: -8.0\n",
      "Gradient of w4 w.r.t to L: -20.0\n"
     ]
    }
   ],
   "source": [
    "from torch import FloatTensor\n",
    "from torch.autograd import Variable  \n",
    "# Define the leaf nodes\n",
    "a = Variable(FloatTensor([4])) \n",
    "weights = [Variable(FloatTensor([i]), requires_grad=True) for i in (2, 5, 9, 7)] \n",
    "# unpack the weights for nicer assignment\n",
    "w1, w2, w3, w4 = weights \n",
    "b = w1 * a\n",
    "c = w2 * a\n",
    "d = w3 * b + w4 * c\n",
    "L = (10 - d) \n",
    "L.backward() \n",
    "for index, weight in enumerate(weights, start=1):    \n",
    "    gradient, *_ = weight.grad.data    \n",
    "    print(f\"Gradient of w{index} w.r.t to L: {gradient}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages\n",
    "import torch\n",
    "import numpy as np\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.4.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Numpy example\n",
    "'''Creating numpy variable'''\n",
    "num1 = np.array([4, 5, 10])\n",
    "num2 = np.array([12,10, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 8, 10, 20])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sum\n",
    "num = num1 + num1\n",
    "num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Torch example\n",
    "'''Creating a torch tensors'''\n",
    "num1 = torch.tensor([4, 5, 10])\n",
    "num2 = torch.tensor([12,10, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 8, 10, 20])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sum\n",
    "num = num1 + num1\n",
    "num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 4, 10, 21,  2])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Creating a torch tensor\n",
    "val = torch.tensor([4,10,21,2])\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 4, 10, 21,  2], dtype=int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#converting into numpy array\n",
    "val.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10, 12, 15],\n",
       "       [10, 20, 14]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create numpy variable\n",
    "val= np.array([[10,12,15],[10,20,14]])\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[10, 12, 15],\n",
       "        [10, 20, 14]], dtype=torch.int32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Conversion into torch\n",
    "val=torch.from_numpy(val)\n",
    "val"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Note:  The NumPy and PyTorch store data in memory in the same way."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensor Introduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[3, 4, 5, 6],\n",
       "        [4, 6, 7, 8]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create a tensor\n",
    "val=torch.tensor([[3,4,5,6], [4,6,7,8]])\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#data type\n",
    "val.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6., 12., 14., 10.],\n",
       "        [10., 11., 12., 16.]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create a float tensor\n",
    "val=torch.FloatTensor([[6,12,14,10], [10,11,12,16]])\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Data Types\n",
    "val.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[True, True, True, True],\n",
       "        [True, True, True, True]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Creating tensor for boolean value\n",
    "val=torch.tensor([[6,12,14,10], [10,11,12,16]], dtype=torch.bool)\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1148, 0.2817, 0.0763, 0.5922],\n",
       "        [0.3643, 0.0984, 0.5095, 0.9770],\n",
       "        [0.4404, 0.4540, 0.3107, 0.7564],\n",
       "        [0.9338, 0.4731, 0.7105, 0.7569],\n",
       "        [0.2781, 0.0632, 0.0631, 0.3123],\n",
       "        [0.7358, 0.7800, 0.0331, 0.0961]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create a matrix with random numbers \n",
    "torch.rand(6,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#full ones\n",
    "torch.ones(4,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 6, 12, 14, 10],\n",
      "        [10, 11, 12, 16]])\n",
      "sum: 91\n"
     ]
    }
   ],
   "source": [
    "#Create Tensor and find their sum \n",
    "var = torch.tensor([[6,12,14,10], [10,11,12,16]])\n",
    "print(var)\n",
    "print(f'sum: {var.sum()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6, 10],\n",
       "        [12, 11],\n",
       "        [14, 12],\n",
       "        [10, 16]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Transpose \n",
    "var.t()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6, 10],\n",
       "        [12, 11],\n",
       "        [14, 12],\n",
       "        [10, 16]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transpose (via permute)\n",
    "var.permute(-1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6, 12, 14, 10],\n",
       "        [10, 11, 12, 16]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reshape via view\n",
    "var.view(2,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6],\n",
       "        [12],\n",
       "        [14],\n",
       "        [10],\n",
       "        [10],\n",
       "        [11],\n",
       "        [12],\n",
       "        [16]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View again...\n",
    "var.view(8,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([22., 23., 17.])\n"
     ]
    }
   ],
   "source": [
    "# Slicing\n",
    "t = torch.Tensor([[10,23,38,22], [22,32,11,23], [19,29,49,17]])\n",
    "# Every row, only the last column\n",
    "print(t[:, -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[10., 23., 38., 22.],\n",
      "        [22., 32., 11., 23.]])\n"
     ]
    }
   ],
   "source": [
    "# First 2 rows, all columns\n",
    "print(t[:2, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[17.]])\n"
     ]
    }
   ],
   "source": [
    "# Lower right most corner\n",
    "print(t[-1:, -1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 4])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Size\n",
    "t.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a new tensor\n",
    "t2 = torch.tensor([[16,13,24,23], [23,36,78,25],[26,34,68,12]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 26.,  36.,  62.,  45.],\n",
       "        [ 45.,  68.,  89.,  48.],\n",
       "        [ 45.,  63., 117.,  29.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sum\n",
    "t3 =t.add(t2)\n",
    "t3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 26.,  36.,  62.,  45.],\n",
       "        [ 45.,  68.,  89.,  48.],\n",
       "        [ 45.,  63., 117.,  29.]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#In-place\n",
    "t.add_(t2)\n",
    "t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More tensor operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-2.3493,  0.5739, -4.0265],\n",
       "        [-0.9982,  0.7558, -0.2420],\n",
       "        [ 0.3225, -1.4824, -0.8597],\n",
       "        [-0.0962, -1.5052,  0.0313]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cross product\n",
    "val1 = torch.randn(4,3)\n",
    "val2 = torch.randn(4,3)\n",
    "val1.cross(val2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[380.],\n",
       "        [860.]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#matrix product\n",
    "var = (torch.Tensor([[10, 14], [26, 30]]).mm(torch.Tensor([[10], [20]])))\n",
    "var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[400., 144.],\n",
       "        [100., 441.]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Elementwise multiplication\n",
    "val= torch.Tensor([[20,12], [10,21]])\n",
    "val.mul(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running on GPU\n",
    "#### Basic Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking GPU available\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[10, 23, 34, 57],\n",
       "        [23, 18, 37, 27]], device='cuda:0')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create a tensor\n",
    "x = torch.tensor([[10,23,34,57], [23,18,37,27]])\n",
    "x.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[20, 57, 62, 86],\n",
       "        [79, 47, 47, 55]], device='cuda:0')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Tensor is now on device cuda:0\n",
    "x = x.to(device)\n",
    "#Create another tensor and convert it into cuda:0\n",
    "y = torch.tensor([[10,34,28,29], [56,29,10,28]])\n",
    "y = y.to(device)\n",
    "#sum\n",
    "x.add(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000e+00, 0.0000e+00],\n",
      "        [0.0000e+00, 0.0000e+00],\n",
      "        [0.0000e+00, 0.0000e+00],\n",
      "        [3.3055e+21, 2.7329e-06]])\n",
      "tensor([[0., 0.],\n",
      "        [0., 0.],\n",
      "        [0., 0.],\n",
      "        [0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "# Create uninitialized tensor\n",
    "x = torch.FloatTensor(4,2)\n",
    "print(x)\n",
    "# Initialize to zeros\n",
    "x.zero_()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 0., 0., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 0., 1.]])\n",
      "tensor([[1., 1., 1.],\n",
      "        [1., 1., 1.]])\n",
      "tensor([[0., 0., 0.],\n",
      "        [0., 0., 0.]])\n",
      "tensor([0, 1, 2])\n"
     ]
    }
   ],
   "source": [
    "# special tensors\n",
    "print(torch.eye(4))\n",
    "print(torch.ones(2,3))\n",
    "print(torch.zeros(2,3))\n",
    "print(torch.arange(0,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic of Maths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-1.])\n",
      "tensor(-1.)\n",
      "tensor(0.3679)\n",
      "tensor(-1.)\n"
     ]
    }
   ],
   "source": [
    "#Torch arange\n",
    "x = torch.arange(-1.0,1.0,5)\n",
    "print(x)\n",
    "print(torch.sum(x))\n",
    "print(torch.sum(torch.exp(x)))\n",
    "print(torch.mean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.5109, 0.6509],\n",
      "        [0.6475, 0.4615],\n",
      "        [0.9806, 0.7422]])\n",
      "tensor([0.6475, 0.4615])\n"
     ]
    }
   ],
   "source": [
    "#Random\n",
    "x = torch.rand(3,2)\n",
    "print(x)\n",
    "print(x[1,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.\n"
     ]
    }
   ],
   "source": [
    "# create a tensor\n",
    "x = torch.rand(4,2)\n",
    "# copy to GPU\n",
    "y = x.cuda()\n",
    "# copy back to CPU\n",
    "z = y.cpu()\n",
    "# get CPU tensor as numpy array\n",
    "# cannot get GPU tensor as numpy array directly\n",
    "try:\n",
    "    y.numpy()\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mm\n"
     ]
    }
   ],
   "source": [
    "#create pytorch tensor\n",
    "x = torch.rand(3,5)  # CPU tensor\n",
    "y = torch.rand(5,4).cuda()  # GPU tensor\n",
    "try:\n",
    "    torch.mm(x,y)  # Operation between CPU and GPU fails\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2241, 0.2719],\n",
      "        [0.4515, 0.0441],\n",
      "        [0.5389, 0.4390],\n",
      "        [0.7164, 0.9530]], device='cuda:0') torch.float32\n"
     ]
    }
   ],
   "source": [
    "# Put tensor on CUDA\n",
    "x = torch.rand(4,2)\n",
    "if torch.cuda.is_available():\n",
    "    x = x.cuda()\n",
    "    print(x, x.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0502, 0.0739],\n",
      "        [0.2039, 0.0019],\n",
      "        [0.2905, 0.1927],\n",
      "        [0.5132, 0.9081]], device='cuda:0')\n",
      "tensor([[0.0502, 0.0739],\n",
      "        [0.2039, 0.0019],\n",
      "        [0.2905, 0.1927],\n",
      "        [0.5132, 0.9081]]) torch.float32\n"
     ]
    }
   ],
   "source": [
    "# Do some calculations\n",
    "y = x ** 2 \n",
    "print(y)\n",
    "# Copy to CPU if on GPU\n",
    "if y.is_cuda:\n",
    "    y = y.cpu()\n",
    "    print(y, y.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.4013e-45, 0.0000e+00]])\n",
      "tensor([[0.0502, 0.0739]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "#Example\n",
    "x1 = torch.rand(4,2)\n",
    "x2 = x1.new(1,2)  # create cpu tensor\n",
    "print(x2)\n",
    "x1 = torch.rand(4,2).cuda()\n",
    "x2 = x1.new(1,2)  # create cuda tensor\n",
    "print(x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU: 1314.0355000000027ms\n",
      "GPU: 1909.7302999999997ms\n"
     ]
    }
   ],
   "source": [
    "#Time taken for processing CPU vs GPU\n",
    "from timeit import timeit\n",
    "# Create random data\n",
    "val1 = torch.rand(5000,64) #creating cpu tensor\n",
    "val2 = torch.rand(64,32)  #creating cpu tensor\n",
    "number = 10000  # number of iterations\n",
    "def square():\n",
    "    '''dot product (mm=matrix multiplication)'''\n",
    "    val=torch.mm(val1, val2) \n",
    "\n",
    "# Time CPU\n",
    "print('CPU: {}ms'.format(timeit(square, number=number)*1000))\n",
    "# Time GPU\n",
    "val1, val2 = val1.cuda(), val2.cuda()\n",
    "print('GPU: {}ms'.format(timeit(square, number=number)*1000))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x177906144c8>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3deVxVdf7H8deXXRZRxBVEEBB3FFGzsjRzqUzLVKymbDHbN6uZmmampuU3LWZOZYtlaeWktmuru1m54b6yuoEoCIrs6/f3x7kKKijLvZzL5fN8PHwA9xzu+XC4vjn3e875fpTWGiGEEI7JyewChBBC2I6EvBBCODAJeSGEcGAS8kII4cAk5IUQwoG5mF1AZf7+/jo4ONjsMoQQolHZvHnzca1166qW2VXIBwcHExsba3YZQgjRqCilDla3TIZrhBDCgUnICyGEA5OQF0IIByYhL4QQDkxCXgghHFiNQ14p9bFSKl0ptavSY35KqWVKqQTLx5aWx5VS6i2lVKJSaodSKsoWxQshhLiw2hzJzwVGnfPY08AKrXU4sMLyNcA1QLjl31TgvfqVKYQQoi5qHPJa69+ArHMeHgvMs3w+D7ih0uOfasN6oIVSqn19i63Owcw8/r1kNyVl5bbahBBC2MzM5fFsPnjCJs9d3zH5tlrrNADLxzaWxwOAw5XWS7E8dh6l1FSlVKxSKjYjI6NORSQcy+WTPw7wZWxKnb5fCCHMsv3wSWYuT2BtQt3y72JsdeJVVfFYld1JtNaztdbRWuvo1q2rvCv3ooZ1a0Ofji14e2UChSVldXoOIYQww/SlcbT0dOXuy0Ns8vz1Dfljp4dhLB/TLY+nAB0rrRcIHKnntqqllOKpkRGkZRcyf8MhW21GCCGsakNyJmsTjnPflaH4eLjaZBv1DfnFwGTL55OB7ys9frvlKptLgOzTwzq2clmYP5eGtuLdVYnkFZXaclNCCFFvWmumL42jjY87tw8Kttl2anMJ5RfAOiBCKZWilLobeAUYrpRKAIZbvgb4CUgGEoEPgQesWnU1nhwZQWZeMXP/PNAQmxNCiDpbE5/BpgMnePiqMJq5OdtsOzWehVJrfXM1i4ZVsa4GHqxrUXUVFdSSYV3b8MGaJP5ySSd8m9nm7Y8QQtSH1po3lsYT2LIZMf2DbLoth7vjddqILpwqLOXD35LNLkUIIar06+6j7EzN5tFh4bi52DaGHS7ke3Tw5bre7fn4j/0czy0yuxwhhDhLWblxFN+5tRc39q3yynKrcriQB3j86i4UlpTx3uoks0sRQoizLN6eSkJ6LtOGd8HF2fYR7JAhH9bGm3FRgXy2/iBp2QVmlyOEEACUlJXz5rIEurVvzrU9bTYJwFkcMuQBHh0Wjtaat1cmml2KEEIA8GVsCoey8nlyRBecnKq6Z9T6HDbkO/p5Mql/EIs2HeZQZr7Z5QghmrjCkjLeXplA36AWXNW1zcW/wUocNuQBHroqDGcnxczl8WaXIoRo4uZvOERadiFPjYhAqYY5igcHD/m2zT2YfGkw325LJeFYjtnlCCGaqLyiUt5dlciloa24NMy/Qbft0CEPcN+VoXi5uTBjmRzNCyHMMffPA2TmFfPkyIgG37bDh7yflxt3XR7Cz7uOsis12+xyhBBNTHZBCR+sSWJY1zZEBbVs8O07fMgDTBkcgm8zV6YvjTO7FCFEE/Phb8mcKixl2ogupmy/SYR8cw9X7rsylNVxGcQeOLe5lRBC2Mbx3CI+/mM/1/VuT48OvqbU0CRCHmDypZ3w93bn9V/jMOZPE0II23pvdRKFJWU8frU5R/HQhELe082Fh4aGsmF/Fr8nHje7HCGEg0vLLuCz9QcZFxVIWBtv0+poMiEPcPPAIAJaNGO6HM0LIWzs7ZWJaK15dFi4qXU0qZB3d3HmkWFhbE/JZtmeY2aXI4RwUIcy81m06TCT+gfR0c/T1FqaVMgD3BQVSIi/FzOWxVNeLkfzQgjrm7k8HmcnxUNXhZldStMLeRdnJx67Opx9R3NYssNmvcWFEE1UwrEcvt2WyuRLg2nb3MPscppeyANc37sDXdv5MHN5AqVl5WaXI4RwIDOWxePl5sJ9V4aaXQrQREPeyUkxbXgX9h/P4+stKWaXI4RwELtSs/l511HuujwEPy83s8sBmmjIAwzv3pbIji14a0UiRaVlZpcjhHAA05fG4dvMlSmDQ8wu5YwmG/JKKZ4c0YXUkwV8seGQ2eUIIRq52ANZrI7L4L4rQ2nu4Wp2OWc02ZAHuDzMn4EhfryzKon84lKzyxFCNFJaa177NQ5/b3cmX9rJ7HLO0qRDXinFUyMjOJ5bxNw/D5hdjhCikVqbcJyN+7N4aGgonm4uZpdzFquEvFLqcaXUbqXULqXUF0opD6VUiFJqg1IqQSm1UCllH2chzhEd7MeQiNZ8sCaZ7IISs8sRQjQyWmumL40joEUzbh4YZHY556l3yCulAoBHgGitdU/AGZgEvAq8qbUOB04Ad9d3W7by5IgIsgtKmLM22exShBCNzNI9x9iRks0jw8Jwd3E2u5zzWGu4xgVoppRyATyBNOAq4CvL8nnADVbaltX1DPDl2l7tmPP7fjJzi8wuRwjRSJSVa2YsjSfE34ubogLNLqdK9Q55rXUqMB04hBHu2cBm4KTW+vTZzBQgoKrvV0pNVUrFKqViMzIy6ltOnU0b3oWCkjLeX5NkWg1CiMblhx1HiDuWw+PDu+DibJ+nOK0xXNMSGAuEAB0AL+CaKlatcqIYrfVsrXW01jq6devW9S2nzsLa+HBD3wA+XXeQY6cKTatDCNE4lJSV8+ayeLq282F0r/Zml1Mta/zpuRrYr7XO0FqXAN8AlwItLMM3AIGA3U8U89iwLpSVa95emWB2KUIIO/f15hQOZObzxIgInJyU2eVUyxohfwi4RCnlqZRSwDBgD7AKGG9ZZzLwvRW2ZVNBrTyJ6d+RBRsPczgr3+xyhBB2qqi0jLdWJBDZsQVXd2tjdjkXZI0x+Q0YJ1i3ADstzzkb+BswTSmVCLQC5tR3Ww3h4avCcXZSzFwuR/NCiKr9b8MhjmQX8tSICIxjW/tllTMFWuvntNZdtdY9tda3aa2LtNbJWusBWuswrfUErXWjuGylna8Ht13SiW+3ppCYnmN2OUIIO5NfXMqsVYlc0tmPy8JamV3ORdnn6WCT3T8klGauzry5TI7mhRBnm/vnAY7nFvPUSPs/igcJ+Sq18nbnrstD+HFnGrtSs80uRwhhJ7ILSvhgTTJDI1rTr5Of2eXUiIR8NaYM7kxzDxdmLIs3uxQhhJ2Ys9aY/uSJERFml1JjEvLV8G3myr1XhrJyXzqbD54wuxwhhMkyc4uY8/t+ru3Vjp4BvmaXU2MS8hdw52XB+Hu7Mf3XOLNLEUKY7P01SRSUlDFteBezS6kVCfkL8HRz4YEhYaxLzuSPxONmlyOEMMmxU4V8uu4gN/QNIKyNj9nl1IqE/EXcMjCI9r4evP5rHFpXOTODEMLBvb0ygbJyzWPDGtdRPEjIX5SHqzOPDAtn2+GTrNibbnY5QogGdjgrnwUbDxPTvyNBrTzNLqfWJORrYHy/QDq18mT60jjKy+VoXoimZObyBJydFA9fFW52KXUiIV8Drs5OPH51F/YdzeHHnWlmlyOEaCCJ6Tl8uzWF2y7pRDtfD7PLqRMJ+Rq6PrIDXdp68+ayeErLys0uRwjRAN5clkAzV2fuHxJqdil1JiFfQ85OimnDI0g+nsc3W1PNLkcIYWO7UrP5cWcad10eQitvd7PLqTMJ+VoY2aMtvQN9+e/yBIpKy8wuRwhhQzOWxdPcw4UpgzubXUq9SMjXglKKJ0ZEkHqygIWbDptdjhDCRjYfPMHKfence2Uovs1czS6nXiTka+mKcH8GBPvx9spECorlaF4IRzT91zj8vd2487Jgs0upNwn5WlJK8eTICDJyivh03QGzyxFCWNkficdZl5zJA0PC8HRzufg32DkJ+ToYEOLHFV1a896aJHIKS8wuRwhhJVprXv81jva+HtwyMMjscqxCQr6OnhzRhZP5Jcz5fb/ZpQghrGTF3nS2HT7JI8PC8XB1Nrscq5CQr6PegS0Y2aMtH63dz4m8YrPLEULUU3m5ZvrSODq18mR8v0Czy7EaCfl6eGJEBHnFpbz/W5LZpQgh6unHnWnsO5rD41d3wdXZcaLRcX4SE3Rp68PYyA7M+/MA6acKzS5HCFFHpWXlvLksni5tvbk+soPZ5ViVhHw9PXZ1F0rKNLNWJZpdihCijr7Zmkry8TymDY/A2cn+m3PXhoR8PQX7ezExuiP/23iIlBP5ZpcjhKilotIy/rs8gd6Bvozs0dbscqxOQt4KHhkWhlKKt1YkmF2KEKKWFm46TOrJAp4cEYFSjnUUD1YKeaVUC6XUV0qpfUqpvUqpQUopP6XUMqVUguVjS2tsyx61923GXwZ24ustqSRn5JpdjhCihgqKy3h7ZSIDQvwYHO5vdjk2Ya0j+f8Cv2ituwKRwF7gaWCF1jocWGH52mE9MDQUdxcnZiyLN7sUIUQNfbruABk5RTw10jGP4sEKIa+Uag5cAcwB0FoXa61PAmOBeZbV5gE31Hdb9szf2527Lw/hhx1prInPMLscIcRFHM7K560VCQyJaE3/YD+zy7EZaxzJdwYygE+UUluVUh8ppbyAtlrrNADLxzZVfbNSaqpSKlYpFZuR0bjD8cGhYYS38eapL7dzMl9ukBLCXpWVa6Yt2oaTUrx0Q0+zy7Epa4S8CxAFvKe17gvkUYuhGa31bK11tNY6unXr1lYoxzwers68GdOHE/nF/OO7XWaXI4Soxodrk9l04ATPj+lBYMvG15y7NqwR8ilAitZ6g+XrrzBC/5hSqj2A5WO6FbZl93oG+PLY1V34YUca32+TDlJC2Js9R07xxtI4RvVox7ioALPLsbl6h7zW+ihwWCkVYXloGLAHWAxMtjw2Gfi+vttqLO69ojNRQS3453e7SMsuMLscIYRFUWkZ0xZtw7eZG/83rpfDnmytzFpX1zwMzFdK7QD6AP8HvAIMV0olAMMtXzcJLs5OzJjYh9JyzZNfbqe8XJtdkhACmLE0nn1Hc3htfC/8vNzMLqdBWCXktdbbLOPqvbXWN2itT2itM7XWw7TW4ZaPWdbYVmMR7O/FP67rzh+Jmcxbd8DscoRo8tYnZzJ7bTI3Dwjiqq6Od2drdeSOVxu6eUBHrurahld+3kdieo7Z5QjRZOUUlvDEou0E+Xnyj+u6mV1Og5KQtyGlFK/c1AtPN2ceX7idkrJys0sSokl6Ycke0rILmDGxD17ujb+lX21IyNtYGx8P/jOuFztTs3lb5rYRosH9uvsoX25O4f4hofTr5LCzq1RLQr4BjOrZnpuiApm1Ookth06YXY4QTUZGThHPfLOTHh2a8+iwLmaXYwoJ+Qby3JjutGvuwbSF28gvLjW7HCEcntaap7/eQW5RKTNj+uDm0jTjrmn+1CZo7uHK9AmRHMzK5/9+2mt2OUI4vIWbDrNiXzp/G9WV8LY+ZpdjGgn5BjQotBVTLg/h8/WHWBXXJG4AFsIUBzPzeOGHPVwa2oo7Lw02uxxTScg3sCdGRBDR1oe/frWDE3kyiZkQ1lZWrnli0XacleL1CZE4OVg7v9qSkG9gHq7OzIiJ5KRlEjOt5W5YIazpg9+SiD14ghdu6EFAi2Zml2M6CXkT9Ojgy+PDu/DjzjS+33bE7HKEcBi7j2Tz5rJ4ru3Vjhv6OP7kYzUhIW+Se68IJbpTS/75/S6OnJRJzISor8KSMh5fuI2Wnm68fEPTmHysJiTkTeLspJgxsQ/lMomZEFbxxtI44o/l8ur43rRsIpOP1YSEvImCWnnyz9Hd+TMpk0/+PGB2OUI0WuuSMvno9/3cOjCIoRFVNqFrsiTkTRbTvyNXd2vDq7/sI+GYTGImRG2dKizhyS+308nPk2eb2ORjNSEhbzKlFP8Z1xtvdxceX7SN4lKZxEyI2vj3YsvkYzF98HRrWpOP1YSEvB1o7ePOf8b1YlfqKd6SScyEqLFfdqXx9ZYUHhoaRlRQ05t8rCYk5O3EyB7tmNAvkHdXJ7L5oExiJsTFpOcU8sw3O+kV4MvDw8LNLsduScjbkX9d350OLZoxbdE28opkEjMhqqO15m9f7SC/uIw3YyJxdZYoq47sGTvi4+HKGxMiOZSVz8syiZkQ1fpi42FWxWXw9DVdCWvTdCcfqwkJeTszsHMrpg7uzP82HGLVPpnETIhzHTiex0s/7uGysFZMHhRsdjl2T0LeDk0b0YWu7Xx46qsdZMkkZkKcUVpWzrRF23BxUkyXycdqRELeDrm7ODNjYh+yC4r5+zc7ZRIzISw++C2ZLYdO8uINPWnvK5OP1YSEvJ3q3qE5T4yI4JfdR/lmS6rZ5Qhhul2pxuRj1/Vuz5jIDmaX02hIyNuxewZ3ZkCwH88v3k2qTGImmrDTk4/5ebnx8g09ZfKxWrBayCulnJVSW5VSP1i+DlFKbVBKJSilFiqlZMagWnJ2UrwxMZJyrXli0TaZxEw0Wa//GkdCei6vT4ikhadESW1Y80j+UaDydX+vAm9qrcOBE8DdVtxWk9HRz5Pnru/B+uQsPv5jv9nlCNHg/kw8zpzf93P7oE5c2aW12eU0OlYJeaVUIHAd8JHlawVcBXxlWWUecIM1ttUUTYgO5OpubXnt1zjiZRIz0YRkFxiTj3X29+KZa2Tysbqw1pH8TOCvwOnZtVoBJ7XWp2/bTAGqbNOilJqqlIpVSsVmZGRYqRzHopTilZt64ePuwmMLZBIz0XT8e/FujuUUMSOmD83cnM0up1Gqd8grpUYD6VrrzZUfrmLVKgeUtdaztdbRWuvo1q3lrVh1/L3deeWm3uxJO8XM5fFmlyOEzf20M41vtqby0NAw+nRsYXY5jZY1juQvA8YopQ4ACzCGaWYCLZRSp+f9DASkmWk9De/elpjojry/JonYA1lmlyOEzaSfKuTv3+6kd6AvD10VZnY5jVq9Q15r/YzWOlBrHQxMAlZqrW8FVgHjLatNBr6v77YE/PP67gS0bMa0RdvJlUnMhAPSWvPXr3dQWFLGmzF9ZPKxerLl3vsbME0plYgxRj/HhttqMrzdXXhjQh8On8jn5R/3mF2OEFY3f8MhVsdl8Mw13Qht7W12OY2eVUNea71aaz3a8nmy1nqA1jpMaz1Ba11kzW01ZQNC/Lj3ilC+2HiYFXuPmV2OEFaz/3geL/+4l8Hh/tx2SSezy3EI8j6okXp8eDhd2/nwt693kJkrfz9F41daVs7jC7fh5uLE6+Nl8jFrkZBvpNxdnJk5qQ+nCkp5RiYxEw7gvdVJbDtsTD7WztfD7HIchoR8I9a1XXOeHNmFpXuO8dXmFLPLEaLOdqZk898VCYyJ7CCTj1mZhHwjd/flnRkQ4se/l+zhcFa+2eUIUWuFJWU8tnAr/t7uvDi2p9nlOBwJ+UbO2UnxxoRIAJ74cjtlMomZaGRe/WUfSRl5vD6hN76ermaX43Ak5B2AMYlZdzbuz2LO78lmlyNEjf2ecJxP/jjAHZcGMzhc7ni3BQl5BzG+XyAjurdl+q/x7Dt6yuxyhLio7IISnvpqO51be/G3UV3NLsdhScg7CKUU/xnXi+bNXHh84XaKSsvMLkmIC3ru+11k5BQxUyYfsykJeQfSytudV2/qzd60U7z2S5zZ5QhRrW+3pvDdtiM8fFU4vQNl8jFbkpB3MMO6teX2QZ2Y8/t+Xv91n1w/L+zO4u1HePLLHQwI9uPBoaFml+PwXC6+imhsnr++B6XlmlmrkigoLuefo7tJT0xhFxbFHuZvXxsBP+eO/rjI5GM2JyHvgJycFC/f0BN3Fyc+/mM/haVlvDS2p9wmLkz12boD/PP73QwO92f2bdEyDt9AJOQdlFKKf43uTjNXZ95dnURhSRmv3dRbjpyEKT5am8xLP+7l6m5tmXVrX9xdJOAbioS8A1NK8ddRXWnm6swby+IpKi1npszPLRrY2ysSeGNZPNf1as/MSfL6a2gS8k3Aw8PC8XB15uWf9lJUUi5HUqJBaK2ZvjSOWauSGBcVIO8kTSJ7vIm454rOvDi2B8v3HmPKvFgKiuU6emE7Wmte+GEPs1YlccvAIKaPj5SAN4ns9SbktkHBvDa+N78nHueOTzZK+0BhE+Xlmr9/u4tP/jjAnZcF8/INctLfTBLyTczE6I7MjOlD7MET3DZnA9kFJWaXJBxIaVk5T365nS82HuKBIaH8a3R3uXzXZBLyTdDYPgHMuiWKXanZ3PLherLyis0uSTiAkrJyHl2wjW+2pvLE8C78dVRXCXg7ICHfRI3q2Y7Zt0eTmJ7LzbPXk55TaHZJohErLCnj/s838+PONJ69thsPDws3uyRhISHfhA2NaMMnd/TnUFY+kz5YT1p2gdkliUaooLiMez6NZfnedF4c24N7ruhsdkmiEgn5Ju7SMH8+u3sA6TlFTPxgnXSXErWSW1TKHZ9s5PfE47w2vje3DQo2uyRxDgl5QXSwH/OnDORUQSkTP1hHckau2SWJRiC7oITb5mwg9uAJZsb0YWJ0R7NLElWQkBcARHZswYKpl1BcWs7ED9YTdzTH7JKEHcvKK+aWD9ezKzWbWbdEMbZPgNkliWrUO+SVUh2VUquUUnuVUruVUo9aHvdTSi1TSiVYPrasf7nClrq1b87Cewfh7ASTZq9jV2q22SUJO5SeU8jNs9eTmJ7Lh7dHM6pnO7NLEhdgjSP5UuAJrXU34BLgQaVUd+BpYIXWOhxYYfla2LmwNt4suncQnm4u3PzherYcOmF2ScKOpGUXMOmD9Rw+kc8nd/RnSEQbs0sSF1HvkNdap2mtt1g+zwH2AgHAWGCeZbV5wA313ZZoGJ1aebHovkH4eblx20cb2JCcaXZJwg4czspn4gfryMgp4tO7BnBpmL/ZJTVuRTmw/zdYOwMW3Apb59tkM1adoEwpFQz0BTYAbbXWaWD8IVBKVfknXyk1FZgKEBQUZM1yRD0EtGjGonsHcetHG5j8yUY+vD2aweGtzS5LmCQ5I5dbP9pAfnEZ8+8ZKC37aqu8DI7HQ8omSIk1/mXsBV1uLPfrDJ2H2GTTylrt4ZRS3sAa4GWt9TdKqZNa6xaVlp/QWl9wXD46OlrHxsZapR5hHcdzi7htzkaS0nN599Yoru7e1uySRAOLO5rDrR9tQGvN51MG0q19c7NLsn+56UaQp8YawZ66FYotFzN4+EJANARGQ2B/COgHnn712pxSarPWOrqqZVY5kldKuQJfA/O11t9YHj6mlGpvOYpvD6RbY1uiYfl7u/PFPQOZ/PFG7vt8M/+d1Jfrerc3uyzRQHalZnPbnA24uTgxf8ogwtp4m12S/SkphKM7LEfom4xgP3nIWKacoV1P6D3RCPTAaPALBaeGu7Cx3iGvjMkp5gB7tdYzKi1aDEwGXrF8/L6+2xLmaOHpxudTBnLX3E08/MUWissiubFvoNllCRvbcugEkz/eSHMPV/53z0A6tfIyuyTzaQ1ZyZC6uSLUj+6EcstEf80DIbAfDJhqHK23jwQ3T1NLrvdwjVLqcmAtsBOwDDDxd4xx+UVAEHAImKC1zrrQc8lwjX3LLy5lyrxY1iVn8vINvbhloJxDcVTrkzO5e+4m/H3c+d89lxDQopnZJZmj4KQR6KmbK8bTCywx5uoFHfpahl2ijVBvbs67XJsO12itfweqm2puWH2fX9gPTzcXPr6jP/d/vpm/f7uTwpIy7ro8xOyyhJX9Fp/B1M9iCWzpyfwpA2nb3MPskhpGWSmk7644MZoaa5wsBUBB6wjoem3FeHrrbuBs/8317L9CYVc8XJ354LZoHvliKy/8sIfC0jIeGBJmdlnCSpbvOcYD87cQ2sabz+4egL+3u9kl2U52quXEqOVf2jYosczd5OlvjKH3nmiEekCUccK0EZKQF7Xm5uLEO7f05Ykvt/PaL3EUFpfx+PAuMnd4I/fjjjQeXbCVHh2aM++uAbTwdDO7JOspzoMj2yqudknZDDlHjGXObsbYedTkiqGXFp3AQV7PEvKiTlycnZgxsQ/uLk68tTKRwtJynrlGmkQ0Vt9sSeHJL7fTr1NLPr6jPz4ermaXVHfl5ZCZcPbVLsf2gLb0NW4ZAsGXWYZd+htXv7g47jsWCXlRZ85OilfG9cbD1ZnZvyVTUFzGv8f0kH6ejcz/Nhzi2e92MqhzKz6aHI2nWyOLhbzMSkfosZC6BYos8y65NzeuQx88reKadK+mdaduI/ttCnvj5KT495geZ4K+qLSM/4zrjbMEfaPw8e/7eeGHPQyNaM17f+mHh6uz2SVdWGmxccni6SP0lFg4sd9YppygbQ/oOa7iRqNW4Q16Tbo9kpAX9aaU4plruuLh6sxbKxIoLCnnjYmRuDo37f9c9u7d1Ym89ksco3q0462b++LmYme/L63h5MGzr3ZJ2w5llp7EPu2NMO93hxHoHfqAm1zLfy4JeWEVSimmDe+Ch6sTr/0SR1FpGW/fHGV/wSHQWvPmsnjeWpnI2D4deGNCJC728Ae58BQc2VJxYjRlE+QfN5a5NDOuSR94b8VYuq/MYV8TEvLCqh4YEkYzV2f+vWQP934W2ziGAJoQrTX/+Xkfs39LZmJ0oHlDa+VlkL630rDLZsjYB1huzmwVDuEjjLtHA/tDm+7g3IhPBptIQl5Y3Z2XheDu4syz3+1k1MzfuH9IKDf2DZSjehNprVkdl8E7qxLZfPAEtw/qxPPXN+BJ8pyjla522WycHC3JM5Y1a2kEeY8bjVAP6Gc8JqzCarNQWoNMa+BYVselM31pHLtST9HB14N7rwwlpn9HObJvQOXlml92H2XWqkR2HzlFQItmPDA0lFsGBNnucteSAmPsvPK0uqdSjGVOLtCul+VKF8s16X6dHeaadLNcaFoDCXlhU1prVsdnMGtlIrEHT+Dv7c6UwSH85ZJOeLvLG0lbKSkrZ/G2I7y7OpGkjDxC/L0s76gCrHtCXGvITDr7EsZju6C81FjuG3T23C7te4NrE50Hx4Yk5IXptNZs2J/FrFWJrE04jm8zV+68LJg7Lg12rDsrTVZUWvdLba8AABVMSURBVMZXm1N4b3USKScK6NrOhweHhnFtr/bWGXvPzzKGWipfwlh40ljm5m3c/h9QKdR9pP9AQ5CQF3Zl2+GTvLMykeV7j+Hl5sxfBnViyuWdae3juHcd2lp+cSn/23CID9cmc+xUEZEdW/Dw0DCGdWtT92GZshLjqLzyJYyZiZaFyjgZGtiv4mqX1hHgJENxZpCQF3Zp39FTzFqVxI87juDq7MSk/h2ZemVo053Wtg5OFZbw6Z8H+PiPA2TlFXNJZz8eGhrOZWGtahfuWkN2yvkTdpUWGsu92liaXliudunQF9x9bPNDiVqTkBd2bf/xPN5bncg3W1JRCm7sG8D9Q8II8ZcbW6qTlVfMx7/vZ966A+QUljI0ojUPXRVGv041bCNXlAtHtlZc7ZKyCXKPGcuc3Y0bi860qIsG345yctSOSciLRiH1ZAGz1ySxYNNhSsrKGd27Aw8ODSOinRwxnnbsVCGzf0vmfxsOUVhaxqge7XhwaBg9Ay4wDW55ORyPO/sSxvQ9lZpIh57db7RtT3CR8ySNiYS8aFTScwqZs3Y/n68/SF5xGcO7t+WhoWFEdmxx8W92UIez8nlvTRJfxaZQpjVjIzvwwNBQwtpU8QcwN+P8CbvOayJt6TdqhSbSwnwS8qJROplfzCd/HGDunwfILihhcLg/Dw0NY2DnVmaX1mAS03N5d3Ui3287grNSjI8O5L4rQglqZekbWloEaTsqXe2yqaKJtJOLMWFX5VBv4CbSomFIyItGLbeolM/XH+Sjtckczy2mf3BLHhwaxpVdWjvs/PW7j2Qza1UiP+86iruLE7cM6MTUwSG0K087Z8KuHec3kT59o5EdNJEWDUNCXjiEwpIyFmw8xAe/JZOWXUivAF8eHBrKiO7tHGYO+80HTzBrVSIr96XTwb2Ix7uf4jq/I3imW7oa5WcaK9pRE2lhPgl54VCKS8v5dqtxw8+BzHzC23jz4NAwRvdubx+zKdaS1po/E46xZNkKnI/EMsA1mcHNDuBXcMCyhqWJ9Okwb0RNpEXDkJAXDqm0rJwfd6bx7qok4o7lEOTnyf1DQhkXFYC7i53flHPqCPrwRg7uWEte8jpCihPxVEUAaE9/1Olr0ht5E2nRMCTkhUMrL9cs33uMWasS2Z6STbvmHky9ojM3DwiimZsdhP1ZTaRj0SmxKEsT6SLtQqJTZ5yC+hPadwhunQY4VBNp0TAk5EWToLVmbcJx3lmVyMb9WbTycuOuy0O4fVCnhmtMXbmJ9OmrXSo1kc71DGR9cWd+Lwgm3bcXw4dczeh+wdJFS9SLqSGvlBoF/BdwBj7SWr9S3boS8sJaNh3I4p2ViayJz8DH3YVAP9tcZdK8PJuIsni6lsbRtTSOLmXxeGtjnvQ8PIlziWCfSxfinCP4syiYhFwPurVvzkNDwxjVs530whVWYVrIK6WcgXhgOJACbAJu1lrvqWp9CXlhbTtTsvl03QFOFpTU+7mcdQkdi5LoXLSXzkV7CSncS5tSY9ilHCdS3Dqz370ryR7dSHbvxjHXjmhVcYTu5uLETVEBDI2ox6RhQlThQiFv69PzA4BErXWypZAFwFigypAXwtp6Bfry+oTI2n9jTZpIh0dDwL0Q2B+nDn0IcvMiCLjSqj+BEPVj65APAA5X+joFGFh5BaXUVGAqQFBQkI3LEaIa5zaRTo2FvAxjWeUm0qdvNJIm0qKRsHXIV/We9KzxIa31bGA2GMM1Nq5HiIs3kfbvAmHDK240kibSohGzdcinAB0rfR0IHLHxNoU42wWbSPsZQd5znDFZV0CUNJEWDsXWIb8JCFdKhQCpwCTgFhtvUzRlF2wi7Wo0ke77l4oZGKWJtHBwNg15rXWpUuoh4FeMSyg/1lrvtuU2RRNyuol05RkYj+2uaCLdIgiCBkLAg0aot+sNrh7m1ixEA7P55Bda65+An2y9HdEE5GdZuhhVGno500TaBwL6wmWPVszv4t3G3HqbkJKSElJSUigsLDS7FIfm4eFBYGAgrq41P0ckMxwJ+1RabDSRrhzqWUnGMuVkTNDVfWzFpF3SRNpUKSkp+Pj4EBwcLPcA2IjWmszMTFJSUggJCanx90nIC/NpDdmHLdejW/qNpm2vaCLt3dYI8tNj6dJE2u4UFhZKwNuYUopWrVqRkZFRq++TkBcNryjH0kS60o1Gp5tIu3gYzS76TzFOjEoT6UZDAt726rKPJeSFbZWXQUZcpZ6jmyFj79lNpDsPkSbSQtiIhLywrtz0SidGYyF16/lNpLtdL02khdVorRk8eDDPPvss11xzDQCLFi3i448/5pdffjG5OvNJyIu6KymEozsqrklPja1oIq2coV1P6D1RmkgLm1JK8f777zNhwgSGDh1KWVkZzz77rAS8hYS8qBmtISu54sRoSiwc3Xl+E+kBU6WJdBP37yW72XPklFWfs3uH5jx3fY9ql/fs2ZPrr7+eV199lby8PG6//XZCQ0OtWkNjJSEvqlZwsuLyRUtHIwqyjGWnm0gPelCaSAu78dxzzxEVFYWbmxsyZXkFCXkBZaWQvrvixGjKJqO7EXCmiXTEtRUTdkkTaXEBFzritiUvLy9iYmLw9vbG3d3dlBrskfxPbYqyU8+egfHIVigtMJZ5+htBHhkjTaRFo+Pk5ISTnPc5i4S8oyvOq7gm/fSwS06asczZzZjPpd8dFVe7tAyWa9KFcCAS8o7kTBPpSjMwplc0kaZlMARfXjG3S7te4CJva4VwZBLyjVne8UpH6JuMa9KLso1l7s2NI/PB0ypC3cvf3HqFsLHnn3/e7BLsjoR8Y1FaBEd3nT2t7okDxjLlBG16GI0vAqON69Jbhcs16UIICXm7dG4T6ZRNxk1HlZtIB0ZDvzuNQO/QB9y8zK1ZCGGXJOTtwblNpFM2Qf5xY5k0kRZC1IOEfEM7r4l0rDGBV+Um0uEjjLtHA/tLE2khRL1IyNtaTZpI97CMpUsTaSGElUnIW1ONmkjfWjGtrjSRFkLYmIR8XZ3XRDrWaFdXuYl0xwEQKE2khWhozz//PN7e3jz55JM1Wn/x4sXs2bOHp59+utbb+u677+jSpQvdu3cH4F//+hdXXHEFV199da2fyxYk5GsqP8sYaqkc6tJEWohGr7S0lDFjxjBmzJg6ff93333H6NGjz4T8Cy+8YM3y6k1CviplJcZReeVLGKWJtBA18/PTxjTU1tSuF1zzygVXefnll/n000/p2LEjrVu3pl+/fiQlJfHggw+SkZGBp6cnH374IV27duWOO+7Az8+PrVu3EhUVRa9evYiNjeXll18mMjKS5ORknJycyM/PJyIiguTkZObOncvs2bMpLi4mLCyMzz77jG3btrF48WLWrFnDSy+9xNdff82LL77I6NGj8fLy4pNPPmHRokUArF69mjfeeIMlS5awdOlSnnvuOYqKiggNDeWTTz7B29ubp59+msWLF+Pi4sKIESOYPn16vXedhLzWkJ1ScWI0JRbStkkTaSEakc2bN7NgwQK2bt1KaWkpUVFR9OvXj6lTp/L+++8THh7Ohg0beOCBB1i5ciUA8fHxLF++HGdnZ+bOnQuAr68vkZGRrFmzhqFDh7JkyRJGjhyJq6sr48aN45577gHgH//4B3PmzOHhhx9mzJgxjB49mvHjx59V0/Dhw7n33nvJy8vDy8uLhQsXEhMTw/Hjx3nppZdYvnw5Xl5evPrqq8yYMYOHHnqIb7/9ln379qGU4uTJk1bZN00v5ItyLRN2bapogFFtE+n+4BsoJ0eFqI2LHHHbwtq1a7nxxhvx9DQa1YwZM4bCwkL+/PNPJkyYcGa9oqKiM59PmDABZ+fz34HHxMSwcOFChg4dyoIFC3jggQcA2LVrF//4xz84efIkubm5jBw58oI1ubi4MGrUKJYsWcL48eP58ccfee2111izZg179uzhsssuA6C4uJhBgwbRvHlzPDw8mDJlCtdddx2jR4+u936Beoa8Uup14HqgGEgC7tRan7Qsewa4GygDHtFa/1rPWmuvvAyOx599tct5TaSHVszAKE2khWi01DkHY+Xl5bRo0YJt27ZVub6XV9V3iY8ZM4ZnnnmGrKwsNm/ezFVXXQXAHXfcwXfffUdkZCRz585l9erVF60pJiaGWbNm4efnR//+/fHx8UFrzfDhw/niiy/OW3/jxo2sWLGCBQsW8M4775x511Ef9Z3cZBnQU2vdG4gHngFQSnUHJgE9gFHAu0op2w9a56bDvp9gxQsw73p4pRO8ewksfhj2fG90L7rir3DrV/DX/fDIFhj3AQy4x7hGXQJeiEbpiiuu4Ntvv6WgoICcnByWLFmCp6cnISEhfPnll4DR8Hv79u0XfS5vb28GDBjAo48+yujRo88c7efk5NC+fXtKSkqYP3/+mfV9fHzIycmp8rmGDBnCli1b+PDDD4mJiQHgkksu4Y8//iAxMRGA/Px84uPjyc3NJTs7m2uvvZaZM2dW+8eptup1JK+1Xlrpy/XA6UGpscACrXURsF8plQgMANbVZ3vVil8KPz1R0UTayQXa9qhofBHYH1qFyrCLEA4qKiqKmJgY+vTpQ6dOnRg8eDAA8+fP5/777+ell16ipKSESZMmERkZedHni4mJYcKECWcdrb/44osMHDiQTp060atXrzPBPmnSJO655x7eeustvvrqq7Oex9nZmdGjRzN37lzmzZsHQOvWrZk7dy4333zzmeGjl156CR8fH8aOHUthYSFaa958801r7BqU1to6T6TUEmCh1vpzpdQ7wHqt9eeWZXOAn7XWX1XxfVOBqQBBQUH9Dh48WPuNH9kGv79ZcbWLNJEWokHt3buXbt26mV1Gk1DVvlZKbdZaR1e1/kWP5JVSy4F2VSx6Vmv9vWWdZ4FS4PR7mKoOmav8a6K1ng3MBoiOjq7bX5wOfWDivDp9qxBCOLKLhrzW+oK3bSmlJgOjgWG64m1BCtCx0mqBwJG6FimEEKJu6nXiVSk1CvgbMEZrnV9p0WJgklLKXSkVAoQDG+uzLSGEfbPW0K+oXl32cX2vk38HcAeWWS5fWq+1vk9rvVsptQjYgzGM86DWpxuNCiEcjYeHB5mZmbRq1eq8SxmFdWityczMxMOjdnNg1ffqmrALLHsZeLk+zy+EaBwCAwNJSUkhIyPD7FIcmoeHB4GBgbX6nqZ3x6sQwupcXV0JCQkxuwxRBen0LIQQDkxCXgghHJiEvBBCODCr3fFqDUqpDKAOt7wC4A8ct2I51mKvdYH91iZ11Y7UVTuOWFcnrXXrqhbYVcjXh1Iqtrrbes1kr3WB/dYmddWO1FU7Ta0uGa4RQggHJiEvhBAOzJFCfrbZBVTDXusC+61N6qodqat2mlRdDjMmL4QQ4nyOdCQvhBDiHBLyQgjhwBpVyCulJiildiulypVS0ecse0YplaiUilNKVdlGXSkVopTaoJRKUEotVEpZvamr5Xm3Wf4dUEpV2ajRsmynZb1Ya9dRxfaeV0qlVqrt2mrWG2XZh4lKqacboK7XlVL7lFI7lFLfKqVaVLNeg+yvi/38lumzF1qWb1BKBduqlkrb7KiUWqWU2mt5/T9axTpDlFLZlX6//7J1XZW2fcHfjTK8ZdlnO5RSUQ1QU0SlfbFNKXVKKfXYOes0yD5TSn2slEpXSu2q9JifUmqZJYuWKaVaVvO9ky3rJFh6d9Se1rrR/AO6ARHAaiC60uPdge0Y0x6HAEmAcxXfvwiYZPn8feB+G9f7BvCvapYdAPwbcN89Dzx5kXWcLfuuM+Bm2afdbVzXCMDF8vmrwKtm7a+a/PzAA8D7ls8nYbS8tPXvrj0QZfncB4ivoq4hwA8N9Xqqze8GuBb4GaNj3CXAhgauzxk4inHDUIPvM+AKIArYVemx14CnLZ8/XdXrHvADki0fW1o+b1nb7TeqI3mt9V6tdVwVi840Dtda7wdONw4/QxmTXF8FnO4zOw+4wVa1WrY3EfjCVtuwgQFAotY6WWtdDCzA2Lc2o7VeqrUutXy5HqOLmFlq8vOPxXjtgPFaGqZsPIG61jpNa73F8nkOsBcIsOU2rWws8Kk2rAdaKKXaN+D2hwFJWuu63k1fL1rr34Cscx6u/DqqLotGAsu01lla6xPAMmBUbbffqEL+AgKAw5W+TuH8/wStgJOVAqWqdaxpMHBMa51QzXINLFVKbbY0M28ID1neLn9czdvDmuxHW7oL44ivKg2xv2ry859Zx/JaysZ4bTUIy/BQX2BDFYsHKaW2K6V+Vkr1aKiauPjvxuzX1SSqP9gya5+11VqngfFHHGhTxTpW2W92N5+8qkHj8Kq+rYrHzr02tMbNxS+mhjXezIWP4i/TWh9RSrXB6Ky1z/IXv84uVBfwHvAixs/8IsZQ0l3nPkUV31vva2xrsr/U+c3gz2X1/VVVqVU8ZrPXUW0ppbyBr4HHtNanzlm8BWM4ItdyvuU7jLabDeFivxsz95kbMAZ4porFZu6zmrDKfrO7kNcXaRxejZo0Dj+O8TbRxXIEVufm4herUSnlAowD+l3gOY5YPqYrpb7FGCqoV2jVdN8ppT4EfqhikU0asNdgf1XVDP7c57D6/qpCTX7+0+ukWH7Pvpz/VtzqlFKuGAE/X2v9zbnLK4e+1vonpdS7Sil/rbXNJ+Kqwe/GJq+rGroG2KK1PnbuAjP3GXBMKdVea51mGbpKr2KdFIzzBqcFYpyPrBVHGa65aONwS3isAsZbHpoMVPfOoL6uBvZprVOqWqiU8lJK+Zz+HOPk466q1rWWc8ZAb6xme5uAcGVcheSG8TZ3sY3rqq4ZfOV1Gmp/1eTnX4zx2gHjtbSyuj9M1mIZ858D7NVaz6hmnXanzw0opQZg/N/OtGVdlm3V5HezGLjdcpXNJUD26aGKBlDtO2qz9plF5ddRdVn0KzBCKdXSMrw6wvJY7dj6zLI1/2GEUwpQBBwDfq207FmMKyPigGsqPf4T0MHyeWeM8E8EvgTcbVTnXOC+cx7rAPxUqY7tln+7MYYtbL3vPgN2AjssL7D259Zl+fpajKs3khqorkSMccdtln/vn1tXQ+6vqn5+4AWMP0IAHpbXTqLltdS5AfbR5Rhv03dU2k/XAvedfp0BD1n2zXaME9iX2rquC/1uzqlNAbMs+3Qnla6Ms3Ftnhih7VvpsQbfZxh/ZNKAEkt+3Y1xHmcFkGD56GdZNxr4qNL33mV5rSUCd9Zl+zKtgRBCODBHGa4RQghRBQl5IYRwYBLyQgjhwCTkhRDCgUnICyGEA5OQF0IIByYhL4QQDuz/AQfDP3Hv2ZqSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Differentiation\n",
    "import torch\n",
    "import matplotlib.pyplot as plt \n",
    "x = torch.linspace(-10.0,10.0,10, requires_grad=True)\n",
    "Y = x**2\n",
    "y = torch.sum(x**2)     \n",
    "y.backward()\n",
    "\n",
    "plt.plot(x.detach().numpy(), Y.detach().numpy(), label=\"Y\")\n",
    "plt.plot(x.detach().numpy(), x.grad.detach().numpy(), label=\"derivatives\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-20.0000, -15.5556, -11.1111,  -6.6667,  -2.2222,   2.2222,   6.6667,\n",
      "         11.1111,  15.5556,  20.0000])\n",
      "tensor([-40.0000, -31.1111, -22.2222, -13.3333,  -4.4444,   4.4444,  13.3333,\n",
      "         22.2222,  31.1111,  40.0000])\n",
      "tensor([-20.0000, -15.5556, -11.1111,  -6.6667,  -2.2222,   2.2222,   6.6667,\n",
      "         11.1111,  15.5556,  20.0000])\n"
     ]
    }
   ],
   "source": [
    "# Create a variable\n",
    "x=torch.linspace(-10.0,10.0,10, requires_grad=True)\n",
    "# Differentiate\n",
    "torch.sum(x**2).backward()\n",
    "print(x.grad)\n",
    "# Differentiate again (accumulates gradient)\n",
    "torch.sum(x**2).backward()\n",
    "print(x.grad)\n",
    "# Zero gradient before differentiating\n",
    "x.grad.data.zero_()\n",
    "torch.sum(x**2).backward()\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Can't call numpy() on Variable that requires grad. Use var.detach().numpy() instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kunal\\Anaconda3\\envs\\gputest\\lib\\site-packages\\ipykernel_launcher.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "#Example \n",
    "x=torch.tensor(torch.arange(0.0,4.0), requires_grad=True)\n",
    "try:\n",
    "    x.numpy() # raises an exception\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kunal\\Anaconda3\\envs\\gputest\\lib\\site-packages\\ipykernel_launcher.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.,  1., 16., 81.], dtype=float32)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Example\n",
    "x=torch.tensor(torch.arange(0,4.0), requires_grad=True)\n",
    "y=x**2\n",
    "z=y**2\n",
    "z.detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2., 3., 4., 5.],\n",
      "        [1., 2., 3., 4., 5.],\n",
      "        [1., 2., 3., 4., 5.],\n",
      "        [1., 2., 3., 4., 5.]])\n",
      "tensor([[1., 1., 1., 1., 1.],\n",
      "        [2., 2., 2., 2., 2.],\n",
      "        [3., 3., 3., 3., 3.],\n",
      "        [4., 4., 4., 4., 4.]])\n",
      "The size of tensor a (5) must match the size of tensor b (4) at non-singleton dimension 1\n"
     ]
    }
   ],
   "source": [
    "#Example\n",
    "x = torch.ones(4,5)\n",
    "y = torch.arange(5)\n",
    "print(x+y)\n",
    "y = torch.arange(4).view(-1,1)\n",
    "print(x+y)\n",
    "y = torch.arange(4)\n",
    "try:\n",
    "        print(x+y) # exception\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Neural Network Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.3718, -0.0525],\n",
      "        [-0.0180, -0.5750],\n",
      "        [ 0.6493, -0.2669]], grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "#Simple Linear Module\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "x = torch.tensor([[1.0, -1.0],\n",
    "                  [0.0,  1.0],\n",
    "                  [0.0,  0.0]])\n",
    "\n",
    "in_features = x.shape[1]  # = 2\n",
    "out_features = 2\n",
    "\n",
    "net = nn.Linear(in_features, out_features)\n",
    "y = net(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''create a simple sequential network (`nn.Module` object) from layers (other `nn.Module` objects).\n",
    "Here a MLP with 2 layers and sigmoid activation.'''\n",
    "net = torch.nn.Sequential(torch.nn.Linear(32,128),\n",
    "                           torch.nn.Sigmoid(),\n",
    "                           torch.nn.Linear(128,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# customizable network module creation\n",
    "class MyNetwork(torch.nn.Module):\n",
    "    # you can use the layer sizes as initialization arguments if you want to\n",
    "    def __init__(self,input_size, hidden_size, output_size):\n",
    "        super().__init__()\n",
    "        self.layer1 = torch.nn.Linear(input_size,hidden_size)\n",
    "        self.layer2 = torch.nn.Sigmoid()\n",
    "        self.layer3 = torch.nn.Linear(hidden_size,output_size)\n",
    "\n",
    "    def forward(self, input_val):\n",
    "        h = input_val\n",
    "        h = self.layer1(h)\n",
    "        h = self.layer2(h)\n",
    "        h = self.layer3(h)\n",
    "        return h\n",
    "\n",
    "net = MyNetwork(32,128,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[ 0.1360, -0.0284, -0.0086,  ..., -0.0159,  0.1162, -0.0504],\n",
      "        [-0.1099, -0.1526,  0.0903,  ...,  0.0514, -0.0052,  0.0328],\n",
      "        [-0.0984,  0.1504, -0.0676,  ..., -0.1648, -0.0766, -0.0695],\n",
      "        ...,\n",
      "        [ 0.0974, -0.1409,  0.0556,  ..., -0.1619, -0.1213, -0.0107],\n",
      "        [ 0.0583, -0.0089, -0.0757,  ...,  0.1432,  0.0265, -0.0930],\n",
      "        [ 0.0554,  0.1741,  0.0142,  ...,  0.1484,  0.0122,  0.0475]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0634,  0.1129,  0.0302,  0.0251,  0.0826, -0.0598,  0.0225, -0.1387,\n",
      "        -0.1001, -0.0158,  0.0360, -0.0147, -0.1109, -0.0297, -0.0536, -0.1105,\n",
      "         0.1009,  0.1710,  0.0129,  0.0361,  0.0561,  0.0378,  0.0229, -0.1738,\n",
      "        -0.0235, -0.1062,  0.1160, -0.0145, -0.0852,  0.0300,  0.0580,  0.0899,\n",
      "         0.0394,  0.0112,  0.1486,  0.1304, -0.1139, -0.1477,  0.0111, -0.0970,\n",
      "         0.1640, -0.0697, -0.1020,  0.0890, -0.0425,  0.1383, -0.1235,  0.0103,\n",
      "        -0.1360, -0.1344,  0.0830,  0.0121,  0.1746,  0.0990,  0.1633,  0.0312,\n",
      "        -0.1403, -0.1050,  0.0582, -0.1615, -0.0497,  0.1503,  0.1761,  0.1287,\n",
      "         0.0601,  0.1223,  0.0958, -0.1276, -0.1301,  0.0645, -0.1265, -0.0814,\n",
      "        -0.1529, -0.0311, -0.1464,  0.0307, -0.1715,  0.1629,  0.0181, -0.0261,\n",
      "        -0.1220, -0.0299, -0.0401,  0.0865,  0.1679, -0.1457,  0.0313,  0.0262,\n",
      "         0.0435, -0.1165,  0.0967,  0.1496, -0.1344,  0.0425,  0.1400, -0.0781,\n",
      "        -0.1651, -0.0230,  0.1116, -0.1092, -0.0523,  0.0265,  0.0283, -0.1537,\n",
      "        -0.1259, -0.1614, -0.0668,  0.0408,  0.1024, -0.0146,  0.1429, -0.1604,\n",
      "        -0.0551,  0.0916, -0.0264, -0.1678, -0.1368, -0.0873,  0.1515,  0.0132,\n",
      "         0.1572,  0.1244,  0.1749,  0.1131,  0.1467,  0.1653,  0.0988,  0.0259],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0086, -0.0273, -0.0087,  ..., -0.0526, -0.0100, -0.0254],\n",
      "        [-0.0417, -0.0870, -0.0315,  ..., -0.0325, -0.0256, -0.0809],\n",
      "        [ 0.0577, -0.0201, -0.0246,  ...,  0.0289,  0.0747,  0.0169],\n",
      "        ...,\n",
      "        [ 0.0054,  0.0206,  0.0134,  ...,  0.0395, -0.0525, -0.0414],\n",
      "        [-0.0608,  0.0649, -0.0276,  ...,  0.0341, -0.0318, -0.0161],\n",
      "        [-0.0230, -0.0281, -0.0245,  ..., -0.0216, -0.0567,  0.0068]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0193,  0.0853,  0.0432, -0.0507,  0.0826,  0.0163, -0.0099, -0.0773,\n",
      "         0.0617, -0.0315], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#print\n",
    "for param in net.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# customizable network module creation with parameters\n",
    "class MyNetworkWithParams(nn.Module):\n",
    "    def __init__(self,input_size, hidden_size, output_size):\n",
    "        super(MyNetworkWithParams,self).__init__()\n",
    "        self.layer1_weights = nn.Parameter(torch.randn(input_size,hidden_size))\n",
    "        self.layer1_bias = nn.Parameter(torch.randn(hidden_size))\n",
    "        self.layer2_weights = nn.Parameter(torch.randn(hidden_size,output_size))\n",
    "        self.layer2_bias = nn.Parameter(torch.randn(output_size))\n",
    "        \n",
    "    def forward(self,x):\n",
    "        h1 = torch.matmul(x,self.layer1_weights) + self.layer1_bias\n",
    "        h1_act = torch.max(h1, torch.zeros(h1.size())) # ReLU\n",
    "        output = torch.matmul(h1_act,self.layer2_weights) + self.layer2_bias\n",
    "        return output\n",
    "\n",
    "net = MyNetworkWithParams(32,128,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[ 2.4859, -1.6750,  0.0665,  ...,  2.0735,  0.0214, -0.8285],\n",
      "        [ 0.8156,  0.2098,  1.8313,  ...,  1.2195,  1.2943,  0.1101],\n",
      "        [ 0.2275,  1.5312,  0.5497,  ...,  0.6853,  0.1547,  0.8728],\n",
      "        ...,\n",
      "        [-0.9488, -1.2855, -1.0407,  ...,  0.8077, -1.0747, -1.1283],\n",
      "        [-0.5637, -1.8714,  0.3407,  ..., -1.1948, -1.1256, -2.5971],\n",
      "        [ 1.3683, -0.7136, -0.4424,  ..., -0.4152,  0.6378, -1.6716]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 1.0322,  0.2763,  0.3048, -0.7789,  0.0931, -0.3728, -0.1038,  0.1592,\n",
      "         0.9872,  0.6193, -0.0070, -1.6444,  1.0843, -0.3614, -1.7638, -0.0678,\n",
      "        -0.5887, -0.8783, -0.4525, -0.4402,  0.5575,  0.9461, -1.6742, -0.5997,\n",
      "         1.2233, -0.6891, -0.1648, -0.9454, -0.4194,  0.0820, -0.2522,  1.1522,\n",
      "        -0.8768, -0.1290, -0.7117,  0.1403, -0.8592,  0.1403,  0.6877, -0.3550,\n",
      "         0.5132,  1.3831,  1.5951, -0.5626,  0.4352,  0.3921,  0.6587, -0.9680,\n",
      "        -0.8038, -0.1219,  1.1492, -0.6421,  1.5422,  0.4082,  1.9388, -0.9265,\n",
      "         0.6924,  0.1857,  2.3242, -1.4982, -0.0161, -0.2262, -0.5634, -0.5055,\n",
      "         0.5325, -1.2848,  1.1673,  0.8299, -0.4175,  1.5027, -0.0238, -2.0027,\n",
      "         1.1958, -0.4833, -0.5936,  1.6980,  0.9303, -0.4908, -0.1062, -0.5990,\n",
      "         0.4472,  0.7449,  0.2746,  0.0952, -1.4634,  0.7321, -1.3929, -1.8057,\n",
      "         0.3438,  0.7394,  0.5607, -0.4249,  0.5143,  0.2506,  1.8925, -0.2775,\n",
      "        -0.2353,  0.0534, -0.5535, -1.6067, -0.4587,  0.0545,  1.1376, -0.8353,\n",
      "         1.3098, -0.1114, -1.5111, -0.7908,  1.0264,  1.3152, -1.8947,  0.0807,\n",
      "        -1.4589,  1.2128,  1.7217, -1.2520, -0.2345, -1.2595, -0.1060, -0.5888,\n",
      "         2.0078,  0.6311,  2.6570, -0.7415,  2.4827, -1.2755, -0.0423,  1.8519],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 8.5953e-01,  1.0571e+00, -8.6706e-01,  ...,  9.3725e-02,\n",
      "         -2.2232e-01, -1.0822e-01],\n",
      "        [ 8.5755e-01,  9.4221e-02,  2.6763e-01,  ..., -8.0788e-01,\n",
      "          1.8593e+00, -4.9471e-01],\n",
      "        [ 7.9399e-01,  9.9622e-01, -3.5686e-01,  ..., -8.8539e-01,\n",
      "          1.9726e+00, -1.6100e+00],\n",
      "        ...,\n",
      "        [-1.6319e-01, -7.8182e-02,  7.9671e-01,  ...,  1.4996e+00,\n",
      "         -6.8693e-01,  1.3367e+00],\n",
      "        [ 2.3125e+00, -8.0135e-01,  9.7648e-01,  ...,  2.5029e+00,\n",
      "          1.3604e-01, -6.1746e-01],\n",
      "        [-1.1717e-01, -1.5412e+00,  7.4699e-01,  ...,  1.1892e+00,\n",
      "          1.8495e-03,  1.0445e+00]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 1.1729,  1.1718, -0.5595, -1.6339,  0.0506,  1.2522,  0.3895, -1.6335,\n",
      "         0.4181,  0.9003], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#print\n",
    "for param in net.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Training with above Customized Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.2683, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "#Assign parameters value\n",
    "net = MyNetwork(32,128,10)\n",
    "#Processing\n",
    "x = torch.tensor([np.arange(32), np.zeros(32),np.ones(32)]).float()\n",
    "y = torch.tensor([0,3,9])\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "output = net(x)\n",
    "loss = criterion(output,y)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.2683, grad_fn=<NllLossBackward>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kunal\\Anaconda3\\envs\\gputest\\lib\\site-packages\\ipykernel_launcher.py:5: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# equivalent\n",
    "criterion2 = nn.NLLLoss()\n",
    "sf = nn.LogSoftmax()\n",
    "output = net(x)\n",
    "loss = criterion(sf(output),y)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-6.2433e-03, -2.3473e-03,  1.5488e-03,  ...,  1.0674e-01,\n",
      "          1.1064e-01,  1.1454e-01],\n",
      "        [-3.9086e-03,  2.8617e-04,  4.4810e-03,  ...,  1.1774e-01,\n",
      "          1.2193e-01,  1.2613e-01],\n",
      "        [ 5.6422e-03,  5.6422e-03,  5.6423e-03,  ...,  5.6424e-03,\n",
      "          5.6424e-03,  5.6424e-03],\n",
      "        ...,\n",
      "        [ 6.0403e-03,  6.0403e-03,  6.0404e-03,  ...,  6.0416e-03,\n",
      "          6.0417e-03,  6.0417e-03],\n",
      "        [-3.0794e-04,  6.0789e-04,  1.5237e-03,  ...,  2.6251e-02,\n",
      "          2.7167e-02,  2.8083e-02],\n",
      "        [-1.5046e-04,  1.0213e-04,  3.5471e-04,  ...,  7.1745e-03,\n",
      "          7.4271e-03,  7.6797e-03]])\n",
      "tensor([ 2.4880e-03,  1.3506e-03,  1.9591e-03, -2.4919e-03, -1.3297e-03,\n",
      "         2.1950e-03,  9.7842e-04,  1.1026e-03, -7.9888e-03, -7.0191e-03,\n",
      "         2.1729e-03,  1.0002e-02,  4.0039e-03, -5.3891e-03,  9.3931e-03,\n",
      "        -1.2038e-03, -9.2948e-03,  3.3240e-03, -6.7636e-03, -2.5661e-03,\n",
      "        -8.7596e-04, -9.7515e-03, -4.7362e-03, -8.0909e-03,  2.0943e-03,\n",
      "         6.4211e-03, -1.1803e-02, -5.9803e-03, -5.5095e-03,  2.1891e-03,\n",
      "        -9.9183e-03,  2.7918e-03, -4.9605e-03, -1.2306e-03,  1.6825e-03,\n",
      "         7.0822e-03, -1.3154e-02,  1.9268e-03, -6.2594e-03, -1.7602e-03,\n",
      "        -4.8123e-03, -5.7283e-04, -4.4458e-04,  1.1963e-02, -2.5485e-03,\n",
      "        -3.6888e-03, -4.2277e-04,  1.0732e-02,  5.8711e-03, -2.3400e-03,\n",
      "        -1.5351e-03, -5.1208e-03,  3.4827e-04, -3.2950e-04, -6.2716e-03,\n",
      "        -1.6888e-03,  2.2092e-03,  2.8852e-03, -5.2232e-03, -1.2002e-02,\n",
      "        -6.7666e-05,  1.0160e-03, -5.5188e-03, -2.8548e-03,  2.0163e-03,\n",
      "         4.0323e-03, -8.3441e-03,  7.1691e-03, -3.4216e-03,  3.9296e-03,\n",
      "        -6.4048e-03,  6.3774e-04,  3.7926e-03, -1.9106e-03, -3.2035e-03,\n",
      "         6.5983e-03,  8.6829e-03,  8.3870e-03,  4.7461e-03,  7.9977e-03,\n",
      "         4.3901e-03,  6.0687e-03, -1.3774e-03,  1.5947e-03,  4.3264e-03,\n",
      "         4.5636e-03, -4.4153e-03, -8.7386e-04, -3.9728e-03, -2.0357e-03,\n",
      "        -1.8032e-03, -3.6621e-03,  6.3510e-03,  1.9214e-03,  2.5753e-03,\n",
      "        -6.7886e-03,  7.3203e-03,  2.9525e-03, -7.4490e-03,  4.3045e-03,\n",
      "        -8.8767e-03, -9.1790e-03, -7.9343e-03, -5.4787e-03,  2.4240e-03,\n",
      "         1.0189e-03, -2.1324e-03, -3.5684e-03,  2.4360e-03, -8.1527e-03,\n",
      "         2.5368e-03, -3.5064e-04,  2.4365e-03, -4.1165e-03,  7.2402e-03,\n",
      "        -6.1084e-03,  7.8424e-04,  4.1840e-03,  5.0558e-03, -4.0893e-03,\n",
      "        -5.7947e-03, -9.9748e-04,  3.3573e-03,  3.0548e-03, -1.7751e-03,\n",
      "        -9.9471e-04, -2.5606e-03,  5.7150e-03])\n",
      "tensor([[-0.2179, -0.2250, -0.2846,  ...,  0.0189,  0.0133,  0.0090],\n",
      "        [ 0.0445,  0.0529,  0.0597,  ...,  0.0284,  0.0357,  0.0311],\n",
      "        [ 0.0499,  0.0585,  0.0673,  ...,  0.0259,  0.0330,  0.0288],\n",
      "        ...,\n",
      "        [ 0.0595,  0.0695,  0.0794,  ...,  0.0325,  0.0412,  0.0360],\n",
      "        [ 0.0662,  0.0770,  0.0890,  ...,  0.0324,  0.0415,  0.0363],\n",
      "        [-0.0528, -0.1105, -0.1273,  ..., -0.1078, -0.1387, -0.1125]])\n",
      "tensor([-0.2663,  0.0871,  0.0920, -0.2219,  0.0737,  0.0980,  0.1155,  0.1108,\n",
      "         0.1200, -0.2090])\n"
     ]
    }
   ],
   "source": [
    "#accumulates gradient\n",
    "loss.backward()\n",
    "# Check that the parameters now have gradients\n",
    "for param in net.parameters():\n",
    "    print(param.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.2487e-02, -4.6945e-03,  3.0976e-03,  ...,  2.1349e-01,\n",
      "          2.2128e-01,  2.2907e-01],\n",
      "        [-7.8172e-03,  5.7235e-04,  8.9619e-03,  ...,  2.3548e-01,\n",
      "          2.4387e-01,  2.5226e-01],\n",
      "        [ 1.1284e-02,  1.1284e-02,  1.1285e-02,  ...,  1.1285e-02,\n",
      "          1.1285e-02,  1.1285e-02],\n",
      "        ...,\n",
      "        [ 1.2081e-02,  1.2081e-02,  1.2081e-02,  ...,  1.2083e-02,\n",
      "          1.2083e-02,  1.2083e-02],\n",
      "        [-6.1589e-04,  1.2158e-03,  3.0475e-03,  ...,  5.2503e-02,\n",
      "          5.4334e-02,  5.6166e-02],\n",
      "        [-3.0092e-04,  2.0425e-04,  7.0942e-04,  ...,  1.4349e-02,\n",
      "          1.4854e-02,  1.5359e-02]])\n",
      "tensor([ 0.0050,  0.0027,  0.0039, -0.0050, -0.0027,  0.0044,  0.0020,  0.0022,\n",
      "        -0.0160, -0.0140,  0.0043,  0.0200,  0.0080, -0.0108,  0.0188, -0.0024,\n",
      "        -0.0186,  0.0066, -0.0135, -0.0051, -0.0018, -0.0195, -0.0095, -0.0162,\n",
      "         0.0042,  0.0128, -0.0236, -0.0120, -0.0110,  0.0044, -0.0198,  0.0056,\n",
      "        -0.0099, -0.0025,  0.0034,  0.0142, -0.0263,  0.0039, -0.0125, -0.0035,\n",
      "        -0.0096, -0.0011, -0.0009,  0.0239, -0.0051, -0.0074, -0.0008,  0.0215,\n",
      "         0.0117, -0.0047, -0.0031, -0.0102,  0.0007, -0.0007, -0.0125, -0.0034,\n",
      "         0.0044,  0.0058, -0.0104, -0.0240, -0.0001,  0.0020, -0.0110, -0.0057,\n",
      "         0.0040,  0.0081, -0.0167,  0.0143, -0.0068,  0.0079, -0.0128,  0.0013,\n",
      "         0.0076, -0.0038, -0.0064,  0.0132,  0.0174,  0.0168,  0.0095,  0.0160,\n",
      "         0.0088,  0.0121, -0.0028,  0.0032,  0.0087,  0.0091, -0.0088, -0.0017,\n",
      "        -0.0079, -0.0041, -0.0036, -0.0073,  0.0127,  0.0038,  0.0052, -0.0136,\n",
      "         0.0146,  0.0059, -0.0149,  0.0086, -0.0178, -0.0184, -0.0159, -0.0110,\n",
      "         0.0048,  0.0020, -0.0043, -0.0071,  0.0049, -0.0163,  0.0051, -0.0007,\n",
      "         0.0049, -0.0082,  0.0145, -0.0122,  0.0016,  0.0084,  0.0101, -0.0082,\n",
      "        -0.0116, -0.0020,  0.0067,  0.0061, -0.0036, -0.0020, -0.0051,  0.0114])\n",
      "tensor([[-0.4358, -0.4500, -0.5692,  ...,  0.0378,  0.0266,  0.0179],\n",
      "        [ 0.0891,  0.1058,  0.1194,  ...,  0.0568,  0.0714,  0.0622],\n",
      "        [ 0.0999,  0.1169,  0.1346,  ...,  0.0518,  0.0661,  0.0576],\n",
      "        ...,\n",
      "        [ 0.1189,  0.1390,  0.1588,  ...,  0.0650,  0.0824,  0.0720],\n",
      "        [ 0.1325,  0.1540,  0.1779,  ...,  0.0649,  0.0831,  0.0725],\n",
      "        [-0.1055, -0.2209, -0.2546,  ..., -0.2157, -0.2774, -0.2250]])\n",
      "tensor([-0.5327,  0.1742,  0.1841, -0.4438,  0.1474,  0.1960,  0.2310,  0.2216,\n",
      "         0.2401, -0.4180])\n"
     ]
    }
   ],
   "source": [
    "#forward prop and backward prop again\n",
    "output = net(x)\n",
    "loss = criterion(output,y)\n",
    "loss.backward()\n",
    "for param in net.parameters():\n",
    "    print(param.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-6.2433e-03, -2.3473e-03,  1.5488e-03,  ...,  1.0674e-01,\n",
      "          1.1064e-01,  1.1454e-01],\n",
      "        [-3.9086e-03,  2.8617e-04,  4.4810e-03,  ...,  1.1774e-01,\n",
      "          1.2193e-01,  1.2613e-01],\n",
      "        [ 5.6422e-03,  5.6422e-03,  5.6423e-03,  ...,  5.6424e-03,\n",
      "          5.6424e-03,  5.6424e-03],\n",
      "        ...,\n",
      "        [ 6.0403e-03,  6.0403e-03,  6.0404e-03,  ...,  6.0416e-03,\n",
      "          6.0417e-03,  6.0417e-03],\n",
      "        [-3.0794e-04,  6.0789e-04,  1.5237e-03,  ...,  2.6251e-02,\n",
      "          2.7167e-02,  2.8083e-02],\n",
      "        [-1.5046e-04,  1.0213e-04,  3.5471e-04,  ...,  7.1745e-03,\n",
      "          7.4271e-03,  7.6797e-03]])\n",
      "tensor([ 2.4880e-03,  1.3506e-03,  1.9591e-03, -2.4919e-03, -1.3297e-03,\n",
      "         2.1950e-03,  9.7842e-04,  1.1026e-03, -7.9888e-03, -7.0191e-03,\n",
      "         2.1729e-03,  1.0002e-02,  4.0039e-03, -5.3891e-03,  9.3931e-03,\n",
      "        -1.2038e-03, -9.2948e-03,  3.3240e-03, -6.7636e-03, -2.5661e-03,\n",
      "        -8.7596e-04, -9.7515e-03, -4.7362e-03, -8.0909e-03,  2.0943e-03,\n",
      "         6.4211e-03, -1.1803e-02, -5.9803e-03, -5.5095e-03,  2.1891e-03,\n",
      "        -9.9183e-03,  2.7918e-03, -4.9605e-03, -1.2306e-03,  1.6825e-03,\n",
      "         7.0822e-03, -1.3154e-02,  1.9268e-03, -6.2594e-03, -1.7602e-03,\n",
      "        -4.8123e-03, -5.7283e-04, -4.4458e-04,  1.1963e-02, -2.5485e-03,\n",
      "        -3.6888e-03, -4.2277e-04,  1.0732e-02,  5.8711e-03, -2.3400e-03,\n",
      "        -1.5351e-03, -5.1208e-03,  3.4827e-04, -3.2950e-04, -6.2716e-03,\n",
      "        -1.6888e-03,  2.2092e-03,  2.8852e-03, -5.2232e-03, -1.2002e-02,\n",
      "        -6.7667e-05,  1.0160e-03, -5.5188e-03, -2.8548e-03,  2.0163e-03,\n",
      "         4.0323e-03, -8.3441e-03,  7.1691e-03, -3.4216e-03,  3.9296e-03,\n",
      "        -6.4048e-03,  6.3774e-04,  3.7926e-03, -1.9106e-03, -3.2035e-03,\n",
      "         6.5983e-03,  8.6829e-03,  8.3870e-03,  4.7461e-03,  7.9977e-03,\n",
      "         4.3901e-03,  6.0687e-03, -1.3774e-03,  1.5947e-03,  4.3264e-03,\n",
      "         4.5636e-03, -4.4153e-03, -8.7386e-04, -3.9728e-03, -2.0357e-03,\n",
      "        -1.8032e-03, -3.6621e-03,  6.3510e-03,  1.9214e-03,  2.5753e-03,\n",
      "        -6.7886e-03,  7.3203e-03,  2.9525e-03, -7.4490e-03,  4.3045e-03,\n",
      "        -8.8767e-03, -9.1790e-03, -7.9343e-03, -5.4787e-03,  2.4240e-03,\n",
      "         1.0189e-03, -2.1324e-03, -3.5684e-03,  2.4360e-03, -8.1527e-03,\n",
      "         2.5368e-03, -3.5064e-04,  2.4365e-03, -4.1165e-03,  7.2402e-03,\n",
      "        -6.1084e-03,  7.8424e-04,  4.1840e-03,  5.0558e-03, -4.0893e-03,\n",
      "        -5.7947e-03, -9.9748e-04,  3.3573e-03,  3.0548e-03, -1.7751e-03,\n",
      "        -9.9471e-04, -2.5606e-03,  5.7150e-03])\n",
      "tensor([[-0.2179, -0.2250, -0.2846,  ...,  0.0189,  0.0133,  0.0090],\n",
      "        [ 0.0445,  0.0529,  0.0597,  ...,  0.0284,  0.0357,  0.0311],\n",
      "        [ 0.0499,  0.0585,  0.0673,  ...,  0.0259,  0.0330,  0.0288],\n",
      "        ...,\n",
      "        [ 0.0595,  0.0695,  0.0794,  ...,  0.0325,  0.0412,  0.0360],\n",
      "        [ 0.0662,  0.0770,  0.0890,  ...,  0.0324,  0.0415,  0.0363],\n",
      "        [-0.0528, -0.1105, -0.1273,  ..., -0.1078, -0.1387, -0.1125]])\n",
      "tensor([-0.2663,  0.0871,  0.0920, -0.2219,  0.0737,  0.0980,  0.1155,  0.1108,\n",
      "         0.1200, -0.2090])\n"
     ]
    }
   ],
   "source": [
    "#Removing this behavior by reinitializing the gradients\n",
    "net.zero_grad()\n",
    "output = net(x)\n",
    "loss = criterion(output,y)\n",
    "loss.backward()\n",
    "for param in net.parameters():\n",
    "    print(param.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters before gradient descent :\n",
      "Parameter containing:\n",
      "tensor([[-0.1649,  0.1513,  0.0330,  ...,  0.1573,  0.1200, -0.0095],\n",
      "        [ 0.0444,  0.1322,  0.0374,  ...,  0.0672,  0.0717, -0.0374],\n",
      "        [ 0.1023,  0.0648, -0.1038,  ...,  0.0435, -0.0335,  0.1224],\n",
      "        ...,\n",
      "        [ 0.1634,  0.1388, -0.1146,  ..., -0.1763, -0.0560,  0.0331],\n",
      "        [ 0.1346, -0.0478,  0.1085,  ...,  0.0375,  0.0739, -0.1695],\n",
      "        [-0.1013,  0.0362, -0.0493,  ..., -0.0083,  0.1447, -0.1094]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0915,  0.0328, -0.0236, -0.1396,  0.0802, -0.0272, -0.0981,  0.1701,\n",
      "        -0.0970, -0.1459,  0.0119, -0.0557,  0.1270, -0.0069,  0.1329,  0.1481,\n",
      "        -0.0677, -0.0281,  0.0498,  0.1110,  0.0125,  0.1635,  0.1149,  0.0696,\n",
      "        -0.0144, -0.1624, -0.1656,  0.1704,  0.1119,  0.0063, -0.1017,  0.0892,\n",
      "        -0.0928,  0.1499,  0.1617, -0.1395,  0.1070,  0.1460,  0.0486, -0.1104,\n",
      "         0.1359,  0.0875,  0.1597,  0.0517, -0.0639, -0.1594,  0.0875,  0.0763,\n",
      "         0.1075, -0.1316,  0.0483,  0.1347,  0.1268, -0.1239,  0.1000, -0.1712,\n",
      "        -0.0308,  0.0187,  0.1515, -0.1057,  0.1294, -0.0432, -0.0350, -0.1228,\n",
      "         0.1660,  0.1764, -0.0739, -0.0457, -0.0923,  0.0239,  0.1697,  0.0507,\n",
      "         0.0206,  0.0130,  0.1735, -0.1547,  0.0021,  0.0790, -0.0599, -0.1170,\n",
      "         0.1331, -0.0261, -0.0416, -0.0712,  0.0999, -0.0717,  0.1531, -0.1559,\n",
      "        -0.0301, -0.0896,  0.0038,  0.1220, -0.0254,  0.1335, -0.0616,  0.0775,\n",
      "        -0.0698,  0.0897,  0.1184,  0.0767, -0.1504,  0.0532,  0.0229, -0.1534,\n",
      "        -0.0842, -0.1078,  0.1469, -0.0727,  0.1247, -0.1228,  0.1013,  0.1646,\n",
      "        -0.0211, -0.1345, -0.0002, -0.1400, -0.0224, -0.0056,  0.1195,  0.0568,\n",
      "         0.1636, -0.1572, -0.0232,  0.1469,  0.0228, -0.1740,  0.1638, -0.0433],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0766, -0.0872,  0.0091,  ..., -0.0437, -0.0740, -0.0121],\n",
      "        [-0.0470, -0.0423, -0.0106,  ...,  0.0653,  0.0583, -0.0347],\n",
      "        [-0.0552,  0.0204, -0.0541,  ..., -0.0801,  0.0461,  0.0683],\n",
      "        ...,\n",
      "        [ 0.0264,  0.0052, -0.0030,  ..., -0.0470,  0.0376, -0.0006],\n",
      "        [ 0.0136,  0.0211,  0.0683,  ..., -0.0189,  0.0668,  0.0363],\n",
      "        [ 0.0687,  0.0384, -0.0626,  ..., -0.0766,  0.0174,  0.0079]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0822,  0.0624, -0.0606, -0.0532, -0.0798,  0.0708,  0.0840, -0.0214,\n",
      "         0.0542,  0.0284], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#Optimizing\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=0.01)\n",
    "\n",
    "print(\"Parameters before gradient descent :\")\n",
    "for param in net.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters after gradient descent :\n",
      "Parameter containing:\n",
      "tensor([[-0.1648,  0.1513,  0.0330,  ...,  0.1562,  0.1189, -0.0107],\n",
      "        [ 0.0444,  0.1322,  0.0373,  ...,  0.0660,  0.0705, -0.0387],\n",
      "        [ 0.1023,  0.0647, -0.1038,  ...,  0.0434, -0.0335,  0.1223],\n",
      "        ...,\n",
      "        [ 0.1634,  0.1387, -0.1147,  ..., -0.1764, -0.0561,  0.0330],\n",
      "        [ 0.1346, -0.0478,  0.1085,  ...,  0.0372,  0.0736, -0.1698],\n",
      "        [-0.1013,  0.0362, -0.0493,  ..., -0.0084,  0.1447, -0.1094]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0915,  0.0327, -0.0236, -0.1396,  0.0802, -0.0273, -0.0981,  0.1701,\n",
      "        -0.0970, -0.1458,  0.0119, -0.0558,  0.1270, -0.0068,  0.1328,  0.1481,\n",
      "        -0.0677, -0.0281,  0.0498,  0.1110,  0.0125,  0.1636,  0.1150,  0.0697,\n",
      "        -0.0144, -0.1625, -0.1654,  0.1705,  0.1120,  0.0063, -0.1016,  0.0892,\n",
      "        -0.0928,  0.1499,  0.1617, -0.1395,  0.1072,  0.1460,  0.0486, -0.1104,\n",
      "         0.1359,  0.0875,  0.1597,  0.0515, -0.0638, -0.1593,  0.0875,  0.0762,\n",
      "         0.1075, -0.1316,  0.0483,  0.1348,  0.1268, -0.1239,  0.1000, -0.1712,\n",
      "        -0.0308,  0.0187,  0.1516, -0.1056,  0.1294, -0.0432, -0.0349, -0.1228,\n",
      "         0.1659,  0.1763, -0.0738, -0.0458, -0.0923,  0.0239,  0.1697,  0.0507,\n",
      "         0.0205,  0.0130,  0.1735, -0.1548,  0.0020,  0.0789, -0.0599, -0.1171,\n",
      "         0.1331, -0.0262, -0.0416, -0.0712,  0.0999, -0.0717,  0.1531, -0.1559,\n",
      "        -0.0300, -0.0896,  0.0038,  0.1220, -0.0255,  0.1335, -0.0617,  0.0775,\n",
      "        -0.0699,  0.0896,  0.1185,  0.0766, -0.1503,  0.0533,  0.0230, -0.1533,\n",
      "        -0.0842, -0.1079,  0.1469, -0.0727,  0.1246, -0.1227,  0.1012,  0.1646,\n",
      "        -0.0212, -0.1344, -0.0003, -0.1400, -0.0225, -0.0057,  0.1194,  0.0569,\n",
      "         0.1637, -0.1572, -0.0232,  0.1469,  0.0228, -0.1740,  0.1638, -0.0433],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0744, -0.0850,  0.0119,  ..., -0.0439, -0.0741, -0.0122],\n",
      "        [-0.0474, -0.0428, -0.0112,  ...,  0.0650,  0.0580, -0.0350],\n",
      "        [-0.0557,  0.0198, -0.0547,  ..., -0.0803,  0.0458,  0.0680],\n",
      "        ...,\n",
      "        [ 0.0258,  0.0045, -0.0038,  ..., -0.0473,  0.0372, -0.0009],\n",
      "        [ 0.0129,  0.0203,  0.0674,  ..., -0.0192,  0.0664,  0.0359],\n",
      "        [ 0.0692,  0.0395, -0.0613,  ..., -0.0755,  0.0188,  0.0090]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0849,  0.0615, -0.0616, -0.0510, -0.0806,  0.0698,  0.0828, -0.0225,\n",
      "         0.0530,  0.0305], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#Optimize\n",
    "optimizer.step()\n",
    "print(\"Parameters after gradient descent :\")\n",
    "for param in net.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.1482, grad_fn=<NllLossBackward>)\n",
      "tensor(2.0354, grad_fn=<NllLossBackward>)\n",
      "tensor(1.9303, grad_fn=<NllLossBackward>)\n",
      "tensor(1.8339, grad_fn=<NllLossBackward>)\n",
      "tensor(1.7471, grad_fn=<NllLossBackward>)\n",
      "tensor(1.6694, grad_fn=<NllLossBackward>)\n",
      "tensor(1.5995, grad_fn=<NllLossBackward>)\n",
      "tensor(1.5361, grad_fn=<NllLossBackward>)\n",
      "tensor(1.4784, grad_fn=<NllLossBackward>)\n",
      "tensor(1.4256, grad_fn=<NllLossBackward>)\n",
      "tensor(1.3773, grad_fn=<NllLossBackward>)\n",
      "tensor(1.3329, grad_fn=<NllLossBackward>)\n",
      "tensor(1.2920, grad_fn=<NllLossBackward>)\n",
      "tensor(1.2544, grad_fn=<NllLossBackward>)\n",
      "tensor(1.2198, grad_fn=<NllLossBackward>)\n",
      "tensor(1.1879, grad_fn=<NllLossBackward>)\n",
      "tensor(1.1585, grad_fn=<NllLossBackward>)\n",
      "tensor(1.1312, grad_fn=<NllLossBackward>)\n",
      "tensor(1.1059, grad_fn=<NllLossBackward>)\n",
      "tensor(1.0822, grad_fn=<NllLossBackward>)\n",
      "tensor(1.0601, grad_fn=<NllLossBackward>)\n",
      "tensor(1.0393, grad_fn=<NllLossBackward>)\n",
      "tensor(1.0197, grad_fn=<NllLossBackward>)\n",
      "tensor(1.0013, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9838, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9673, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9516, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9367, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9225, grad_fn=<NllLossBackward>)\n",
      "tensor(0.9090, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8960, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8837, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8719, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8605, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8497, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8392, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8292, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8196, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8103, grad_fn=<NllLossBackward>)\n",
      "tensor(0.8013, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7927, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7844, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7764, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7686, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7611, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7539, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7469, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7401, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7335, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7271, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7208, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7148, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7090, grad_fn=<NllLossBackward>)\n",
      "tensor(0.7033, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6977, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6924, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6871, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6820, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6770, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6722, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6675, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6628, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6583, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6539, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6496, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6454, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6413, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6373, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6334, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6295, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6257, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6220, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6184, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6148, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6113, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6079, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6046, grad_fn=<NllLossBackward>)\n",
      "tensor(0.6013, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5980, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5948, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5917, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5886, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5856, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5826, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5797, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5768, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5740, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5712, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5684, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5657, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5630, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5604, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5578, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5553, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5527, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5502, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5478, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5454, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5430, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5406, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5383, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5360, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5337, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5314, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5292, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5270, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5248, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5227, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5206, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5185, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5164, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5143, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5123, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5103, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5083, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5063, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5044, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5024, grad_fn=<NllLossBackward>)\n",
      "tensor(0.5005, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4986, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4967, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4949, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4930, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4912, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4894, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4876, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4858, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4841, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4823, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4806, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4789, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4772, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4755, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4738, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4722, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4705, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4689, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4673, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4657, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4641, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4625, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4609, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4593, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4578, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4562, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4547, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4532, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4517, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4502, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4487, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4472, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4458, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4443, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4429, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4414, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4400, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4386, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4372, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4358, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4344, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4330, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4316, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4303, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4289, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4276, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4262, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4249, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4236, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4223, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4209, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4196, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4183, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4171, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4158, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4145, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4132, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4120, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4107, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4095, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4082, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4070, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4058, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4045, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4033, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4021, grad_fn=<NllLossBackward>)\n",
      "tensor(0.4009, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3997, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3985, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3973, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3962, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3950, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3938, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3926, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3915, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3903, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3892, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3880, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3869, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3858, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3846, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3835, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3824, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3813, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3802, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3791, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3780, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3769, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3758, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3747, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3736, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3726, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3715, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3704, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3694, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3683, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3673, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3662, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3652, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3641, grad_fn=<NllLossBackward>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3631, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3620, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3610, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3600, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3590, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3580, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3569, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3559, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3549, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3539, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3529, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3519, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3510, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3500, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3490, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3480, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3470, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3461, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3451, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3441, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3432, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3422, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3413, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3403, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3394, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3384, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3375, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3365, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3356, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3347, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3337, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3328, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3319, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3310, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3301, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3292, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3282, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3273, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3264, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3255, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3246, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3238, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3229, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3220, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3211, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3202, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3193, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3185, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3176, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3167, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3158, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3150, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3141, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3133, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3124, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3116, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3107, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3099, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3090, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3082, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3073, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3065, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3057, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3048, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3040, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3032, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3024, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3015, grad_fn=<NllLossBackward>)\n",
      "tensor(0.3007, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2999, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2991, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2983, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2975, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2967, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2959, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2951, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2943, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2935, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2927, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2919, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2911, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2903, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2896, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2888, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2880, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2872, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2865, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2857, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2849, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2842, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2834, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2826, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2819, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2811, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2804, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2796, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2789, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2781, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2774, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2766, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2759, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2752, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2744, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2737, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2730, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2723, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2715, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2708, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2701, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2694, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2687, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2679, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2672, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2665, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2658, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2651, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2644, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2637, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2630, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2623, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2616, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2609, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2602, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2595, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2589, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2582, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2575, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2568, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2561, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2555, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2548, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2541, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2535, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2528, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2521, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2515, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2508, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2501, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2495, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2488, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2482, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2475, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2469, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2462, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2456, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2449, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2443, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2437, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2430, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2424, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2418, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2411, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2405, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2399, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2392, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2386, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2380, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2374, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2368, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2362, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2355, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2349, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2343, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2337, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2331, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2325, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2319, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2313, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2307, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2301, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2295, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2289, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2283, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2277, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2271, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2266, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2260, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2254, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2248, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2242, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2237, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2231, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2225, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2219, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2214, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2208, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2202, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2197, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2191, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2186, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2180, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2174, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2169, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2163, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2158, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2152, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2147, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2141, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2136, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2130, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2125, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2120, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2114, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2109, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2104, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2098, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2093, grad_fn=<NllLossBackward>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2088, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2082, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2077, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2072, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2067, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2061, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2056, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2051, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2046, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2041, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2036, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2030, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2025, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2020, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2015, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2010, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2005, grad_fn=<NllLossBackward>)\n",
      "tensor(0.2000, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1995, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1990, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1985, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1980, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1975, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1970, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1965, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1961, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1956, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1951, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1946, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1941, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1936, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1931, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1927, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1922, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1917, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1912, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1908, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1903, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1898, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1894, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1889, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1884, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1880, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1875, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1870, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1866, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1861, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1857, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1852, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1848, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1843, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1839, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1834, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1830, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1825, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1821, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1816, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1812, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1807, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1803, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1799, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1794, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1790, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1786, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1781, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1777, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1773, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1768, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1764, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1760, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1756, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1751, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1747, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1743, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1739, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1735, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1731, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1726, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1722, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1718, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1714, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1710, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1706, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1702, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1698, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1694, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1690, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1686, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1682, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1678, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1674, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1670, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1666, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1662, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1658, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1654, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1650, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1646, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1642, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1639, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1635, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1631, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1627, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1623, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1619, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1616, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1612, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1608, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1604, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1601, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1597, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1593, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1589, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1586, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1582, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1578, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1575, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1571, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1567, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1564, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1560, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1557, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1553, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1549, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1546, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1542, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1539, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1535, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1532, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1528, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1525, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1521, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1518, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1514, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1511, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1507, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1504, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1501, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1497, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1494, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1490, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1487, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1484, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1480, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1477, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1474, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1470, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1467, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1464, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1460, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1457, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1454, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1451, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1447, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1444, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1441, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1438, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1434, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1431, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1428, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1425, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1422, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1419, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1415, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1412, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1409, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1406, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1403, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1400, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1397, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1394, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1391, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1388, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1384, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1381, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1378, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1375, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1372, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1369, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1366, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1363, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1360, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1357, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1354, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1352, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1349, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1346, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1343, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1340, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1337, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1334, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1331, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1328, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1325, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1323, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1320, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1317, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1314, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1311, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1308, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1306, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1303, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1300, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1297, grad_fn=<NllLossBackward>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1295, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1292, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1289, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1286, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1284, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1281, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1278, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1275, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1273, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1270, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1267, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1265, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1262, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1259, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1257, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1254, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1251, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1249, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1246, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1244, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1241, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1238, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1236, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1233, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1231, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1228, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1226, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1223, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1221, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1218, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1215, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1213, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1210, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1208, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1205, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1203, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1201, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1198, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1196, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1193, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1191, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1188, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1186, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1183, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1181, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1179, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1176, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1174, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1171, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1169, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1167, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1164, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1162, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1160, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1157, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1155, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1153, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1150, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1148, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1146, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1143, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1141, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1139, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1137, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1134, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1132, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1130, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1128, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1125, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1123, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1121, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1119, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1116, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1114, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1112, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1110, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1108, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1105, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1103, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1101, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1099, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1097, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1095, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1092, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1090, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1088, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1086, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1084, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1082, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1080, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1078, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1076, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1073, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1071, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1069, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1067, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1065, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1063, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1061, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1059, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1057, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1055, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1053, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1051, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1049, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1047, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1045, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1043, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1041, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1039, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1037, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1035, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1033, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1031, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1029, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1027, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1025, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1023, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1021, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1019, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1017, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1016, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1014, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1012, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1010, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1008, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1006, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1004, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1002, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1000, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0999, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0997, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0995, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0993, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0991, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0989, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0987, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0986, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0984, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0982, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0980, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0978, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0977, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0975, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0973, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0971, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0969, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0968, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0966, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0964, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0962, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0961, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0959, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0957, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0955, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0954, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0952, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0950, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0948, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0947, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0945, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0943, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0942, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0940, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0938, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0937, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0935, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0933, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0932, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0930, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0928, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0927, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0925, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0923, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0922, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0920, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0918, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0917, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0915, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0913, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0912, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0910, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0909, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0907, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0905, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0904, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0902, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0901, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0899, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0897, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0896, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0894, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0893, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0891, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0890, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0888, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0887, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0885, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0883, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0882, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0880, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0879, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0877, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0876, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0874, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0873, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0871, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0870, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0868, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0867, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0865, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0864, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0862, grad_fn=<NllLossBackward>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0861, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0859, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0858, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0857, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0855, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0854, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0852, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0851, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0849, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0848, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0846, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0845, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0844, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0842, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0841, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0839, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0838, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0836, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0835, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0834, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0832, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0831, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0829, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0828, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0827, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0825, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0824, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0823, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0821, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0820, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0818, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0817, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0816, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0814, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0813, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0812, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0810, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0809, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0808, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0806, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0805, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0804, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0802, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0801, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0800, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0799, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0797, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0796, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0795, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0793, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0792, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0791, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0789, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0788, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0787, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0786, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0784, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0783, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0782, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0781, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0779, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0778, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0777, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0776, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0774, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0773, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0772, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0771, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0769, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0768, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0767, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0766, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0765, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0763, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0762, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0761, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0760, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0759, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0757, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0756, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0755, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0754, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0753, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0751, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0750, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0749, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0748, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0747, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0746, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0744, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0743, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0742, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0741, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0740, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0739, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0737, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0736, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0735, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0734, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0733, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0732, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0731, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0729, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0728, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0727, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0726, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0725, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0724, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0723, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0722, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0721, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0719, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0718, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0717, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0716, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0715, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0714, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0713, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0712, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0711, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0710, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0709, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0708, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0706, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0705, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0704, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0703, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0702, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0701, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0700, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0699, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0698, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0697, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0696, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0695, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0694, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0693, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0692, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0691, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0690, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0689, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0688, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0687, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0686, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0685, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0684, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0683, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0682, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0681, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0680, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0679, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0678, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0677, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0676, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0675, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0674, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0673, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "# In training loop\n",
    "n_iter = 1000\n",
    "for i in range(n_iter):\n",
    "    optimizer.zero_grad() # equivalent to net.zero_grad()\n",
    "    output = net(x)\n",
    "    loss = criterion(output,y)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 7.9251, -1.6502, -1.3369, -0.1119, -1.5680, -1.1583, -1.1655, -1.2301,\n",
      "         -1.2132,  4.0209],\n",
      "        [ 0.4119, -1.2414, -1.3830,  6.1773, -1.3616, -1.3325, -1.2789, -1.2657,\n",
      "         -1.3175,  3.6486],\n",
      "        [ 1.9097, -1.3356, -1.3215,  3.2266, -1.3391, -1.2312, -1.2595, -1.3256,\n",
      "         -1.2687,  5.8029]], grad_fn=<AddmmBackward>)\n",
      "tensor([0, 3, 9])\n"
     ]
    }
   ],
   "source": [
    "#print\n",
    "output = net(x)\n",
    "print(output)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving and Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "odict_keys(['0.weight', '0.bias', '2.weight', '2.bias'])\n"
     ]
    }
   ],
   "source": [
    "# get dictionary of keys to weights using `state_dict`\n",
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28,256),\n",
    "    torch.nn.Sigmoid(),\n",
    "    torch.nn.Linear(256,10))\n",
    "print(net.state_dict().keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save a dictionary\n",
    "torch.save(net.state_dict(),'test.t7')\n",
    "# load a dictionary\n",
    "net.load_state_dict(torch.load('test.t7'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(4.7919, device='cuda:0', grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "#Linear Model\n",
    "net = nn.Sequential(nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,2048),nn.ReLU(),\n",
    "                   nn.Linear(2048,120))\n",
    "x = torch.ones(256,2048).cuda()\n",
    "y = torch.zeros(256).long().cuda()\n",
    "net.cuda()\n",
    "x.cuda()\n",
    "crit=nn.CrossEntropyLoss()\n",
    "out = net(x)\n",
    "loss = crit(out,y)\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define Class \n",
    "class MyNet(nn.Module):\n",
    "    def __init__(self,n_hidden_layers):\n",
    "        super(MyNet,self).__init__()\n",
    "        self.n_hidden_layers=n_hidden_layers\n",
    "        self.final_layer = nn.Linear(128,10)\n",
    "        self.act = nn.ReLU()\n",
    "        self.hidden = []\n",
    "        for i in range(n_hidden_layers):\n",
    "            self.hidden.append(nn.Linear(128,128))\n",
    "        self.hidden = nn.ModuleList(self.hidden)\n",
    "            \n",
    "    def forward(self,x):\n",
    "        h = x\n",
    "        for i in range(self.n_hidden_layers):\n",
    "            h = self.hidden[i](h)\n",
    "            h = self.act(h)\n",
    "        out = self.final_layer(h)\n",
    "        return out\n",
    "net = MyNet(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters are :\n",
      "Parameter containing:\n",
      "tensor([[ 0.0717,  0.0513,  0.0815,  ..., -0.0377,  0.0310,  0.0840],\n",
      "        [-0.0625,  0.0012, -0.0812,  ...,  0.0289,  0.0664, -0.0781],\n",
      "        [-0.0878,  0.0651,  0.0861,  ..., -0.0518, -0.0007,  0.0788],\n",
      "        ...,\n",
      "        [-0.0297, -0.0465, -0.0235,  ...,  0.0201,  0.0271, -0.0508],\n",
      "        [-0.0142, -0.0709,  0.0085,  ..., -0.0399, -0.0225,  0.0174],\n",
      "        [ 0.0270, -0.0014, -0.0501,  ..., -0.0304, -0.0042,  0.0599]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0443,  0.0627, -0.0259, -0.0351, -0.0433,  0.0378, -0.0082, -0.0006,\n",
      "         0.0335, -0.0860], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0828,  0.0700,  0.0743,  ..., -0.0233, -0.0215, -0.0347],\n",
      "        [ 0.0684,  0.0133,  0.0487,  ..., -0.0067, -0.0204, -0.0628],\n",
      "        [-0.0655, -0.0135,  0.0782,  ..., -0.0486, -0.0820,  0.0587],\n",
      "        ...,\n",
      "        [-0.0343,  0.0022, -0.0537,  ...,  0.0565, -0.0794, -0.0318],\n",
      "        [-0.0147, -0.0110,  0.0083,  ...,  0.0791, -0.0026,  0.0478],\n",
      "        [ 0.0514, -0.0165, -0.0806,  ...,  0.0227, -0.0376, -0.0086]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0570, -0.0768,  0.0091, -0.0040, -0.0851,  0.0691, -0.0518,  0.0063,\n",
      "         0.0256, -0.0479,  0.0296,  0.0566,  0.0087, -0.0596, -0.0070, -0.0627,\n",
      "         0.0728,  0.0689,  0.0376, -0.0358, -0.0410,  0.0115, -0.0126,  0.0260,\n",
      "         0.0263, -0.0403,  0.0057,  0.0779,  0.0372,  0.0680, -0.0808, -0.0296,\n",
      "         0.0284, -0.0068,  0.0247,  0.0830, -0.0749, -0.0771, -0.0138, -0.0492,\n",
      "         0.0136, -0.0156,  0.0715, -0.0286,  0.0637, -0.0390,  0.0579,  0.0070,\n",
      "         0.0023,  0.0310, -0.0222,  0.0783,  0.0400,  0.0409,  0.0473, -0.0110,\n",
      "        -0.0053, -0.0054, -0.0002,  0.0863, -0.0495,  0.0657,  0.0165, -0.0717,\n",
      "         0.0078,  0.0169, -0.0686,  0.0445,  0.0049,  0.0501,  0.0782, -0.0703,\n",
      "         0.0675,  0.0120, -0.0042,  0.0556,  0.0485,  0.0500, -0.0394, -0.0651,\n",
      "        -0.0075, -0.0064, -0.0588,  0.0517,  0.0274, -0.0486, -0.0495,  0.0670,\n",
      "        -0.0248, -0.0623,  0.0201, -0.0656, -0.0198, -0.0764,  0.0434, -0.0132,\n",
      "         0.0556, -0.0646,  0.0695, -0.0275, -0.0415,  0.0236, -0.0175, -0.0779,\n",
      "         0.0410,  0.0558, -0.0634,  0.0861,  0.0768,  0.0532, -0.0234,  0.0638,\n",
      "        -0.0743,  0.0068, -0.0643, -0.0394, -0.0564, -0.0157, -0.0427, -0.0814,\n",
      "         0.0401,  0.0758,  0.0512,  0.0689,  0.0721, -0.0084,  0.0875,  0.0854],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0168, -0.0129,  0.0663,  ..., -0.0089, -0.0301,  0.0149],\n",
      "        [-0.0567, -0.0861, -0.0282,  ...,  0.0585, -0.0098,  0.0352],\n",
      "        [-0.0137,  0.0832, -0.0101,  ..., -0.0871,  0.0376,  0.0646],\n",
      "        ...,\n",
      "        [ 0.0497, -0.0491, -0.0484,  ..., -0.0386, -0.0466, -0.0337],\n",
      "        [-0.0031,  0.0411, -0.0146,  ...,  0.0696,  0.0480, -0.0879],\n",
      "        [ 0.0590, -0.0629,  0.0287,  ..., -0.0716,  0.0535,  0.0881]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0289,  0.0874, -0.0360,  0.0782,  0.0541, -0.0658, -0.0182, -0.0027,\n",
      "        -0.0026, -0.0112,  0.0823, -0.0467, -0.0491, -0.0286,  0.0862, -0.0729,\n",
      "         0.0871,  0.0815,  0.0448, -0.0103, -0.0650, -0.0045,  0.0872, -0.0187,\n",
      "         0.0372, -0.0035,  0.0213,  0.0090,  0.0125, -0.0317,  0.0121,  0.0404,\n",
      "         0.0320, -0.0181,  0.0642,  0.0433, -0.0293, -0.0326, -0.0303, -0.0673,\n",
      "        -0.0223,  0.0217, -0.0333,  0.0842, -0.0020, -0.0520,  0.0677,  0.0638,\n",
      "         0.0045,  0.0207, -0.0707,  0.0466,  0.0442, -0.0761,  0.0163, -0.0651,\n",
      "        -0.0023,  0.0591, -0.0543,  0.0036, -0.0453,  0.0667, -0.0111,  0.0451,\n",
      "        -0.0832,  0.0419, -0.0633,  0.0792, -0.0210, -0.0108, -0.0254, -0.0220,\n",
      "         0.0379, -0.0383, -0.0568,  0.0421, -0.0532, -0.0863,  0.0785,  0.0855,\n",
      "        -0.0251,  0.0672, -0.0541, -0.0231, -0.0050, -0.0155,  0.0502,  0.0646,\n",
      "        -0.0369,  0.0189,  0.0409,  0.0365, -0.0619, -0.0667,  0.0363, -0.0446,\n",
      "         0.0557,  0.0309,  0.0767, -0.0124, -0.0680, -0.0809,  0.0408, -0.0877,\n",
      "        -0.0568, -0.0391, -0.0368, -0.0112,  0.0520, -0.0556,  0.0803,  0.0865,\n",
      "         0.0025, -0.0088,  0.0673,  0.0597, -0.0067, -0.0406, -0.0564, -0.0311,\n",
      "         0.0237, -0.0161, -0.0593,  0.0732,  0.0404, -0.0267, -0.0400, -0.0548],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0880, -0.0786,  0.0388,  ..., -0.0172, -0.0051,  0.0177],\n",
      "        [-0.0640,  0.0405, -0.0183,  ..., -0.0739,  0.0462,  0.0676],\n",
      "        [ 0.0321,  0.0196,  0.0764,  ..., -0.0405, -0.0089, -0.0854],\n",
      "        ...,\n",
      "        [ 0.0743, -0.0014, -0.0050,  ...,  0.0512,  0.0611,  0.0115],\n",
      "        [ 0.0650,  0.0044, -0.0837,  ...,  0.0560, -0.0405,  0.0316],\n",
      "        [-0.0299,  0.0848,  0.0653,  ...,  0.0718, -0.0181, -0.0253]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0395,  0.0017,  0.0302, -0.0340,  0.0332,  0.0675, -0.0614,  0.0315,\n",
      "        -0.0772,  0.0192,  0.0872,  0.0559,  0.0866,  0.0553, -0.0487, -0.0620,\n",
      "         0.0775, -0.0852,  0.0362, -0.0766, -0.0347,  0.0703,  0.0149, -0.0395,\n",
      "         0.0879, -0.0157,  0.0238, -0.0828, -0.0205, -0.0868,  0.0348, -0.0259,\n",
      "         0.0676,  0.0411, -0.0379,  0.0450, -0.0477, -0.0595,  0.0839, -0.0727,\n",
      "        -0.0649,  0.0823, -0.0799, -0.0214, -0.0654,  0.0732,  0.0009, -0.0445,\n",
      "         0.0634,  0.0426,  0.0365, -0.0363, -0.0129,  0.0396, -0.0761, -0.0854,\n",
      "         0.0268,  0.0636, -0.0022, -0.0450,  0.0133, -0.0122, -0.0865,  0.0788,\n",
      "         0.0362, -0.0218,  0.0298, -0.0612,  0.0140, -0.0116, -0.0029, -0.0095,\n",
      "        -0.0381, -0.0061,  0.0572, -0.0170, -0.0584, -0.0775,  0.0359, -0.0080,\n",
      "         0.0809, -0.0287, -0.0040,  0.0663,  0.0334, -0.0834, -0.0084,  0.0790,\n",
      "         0.0105,  0.0383,  0.0304,  0.0383, -0.0404, -0.0664, -0.0872, -0.0381,\n",
      "         0.0719,  0.0406, -0.0104, -0.0599,  0.0848, -0.0223, -0.0072,  0.0405,\n",
      "        -0.0661, -0.0210, -0.0711, -0.0783, -0.0696,  0.0124, -0.0236, -0.0710,\n",
      "         0.0310,  0.0568,  0.0032,  0.0713, -0.0321, -0.0759,  0.0610,  0.0311,\n",
      "         0.0401, -0.0594, -0.0357, -0.0310, -0.0796,  0.0004, -0.0813,  0.0622],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0055, -0.0393,  0.0615,  ..., -0.0576, -0.0219, -0.0212],\n",
      "        [-0.0276,  0.0747,  0.0806,  ..., -0.0191, -0.0623,  0.0590],\n",
      "        [-0.0335, -0.0157, -0.0592,  ...,  0.0039,  0.0219,  0.0651],\n",
      "        ...,\n",
      "        [-0.0424,  0.0466, -0.0707,  ...,  0.0778, -0.0084, -0.0065],\n",
      "        [ 0.0146, -0.0788,  0.0315,  ...,  0.0665,  0.0609,  0.0676],\n",
      "        [-0.0641, -0.0769, -0.0751,  ...,  0.0858,  0.0256,  0.0085]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0852, -0.0553, -0.0230, -0.0843, -0.0097,  0.0365,  0.0649, -0.0735,\n",
      "        -0.0536, -0.0405, -0.0362, -0.0539, -0.0023, -0.0379,  0.0336,  0.0552,\n",
      "         0.0334,  0.0516,  0.0254, -0.0447, -0.0196, -0.0018, -0.0653,  0.0714,\n",
      "         0.0032,  0.0633,  0.0065,  0.0144, -0.0388, -0.0739,  0.0240,  0.0309,\n",
      "         0.0836,  0.0237, -0.0487,  0.0369,  0.0056, -0.0197,  0.0434,  0.0636,\n",
      "         0.0045, -0.0558,  0.0011, -0.0079, -0.0646, -0.0591,  0.0590,  0.0187,\n",
      "         0.0112, -0.0841,  0.0529,  0.0233,  0.0082, -0.0503,  0.0602,  0.0504,\n",
      "        -0.0564,  0.0398, -0.0541, -0.0553,  0.0051,  0.0462, -0.0695, -0.0225,\n",
      "         0.0846,  0.0335,  0.0843, -0.0600,  0.0002, -0.0590, -0.0643,  0.0511,\n",
      "         0.0849,  0.0427,  0.0419, -0.0452,  0.0574, -0.0454,  0.0329, -0.0197,\n",
      "        -0.0309, -0.0857,  0.0269, -0.0135, -0.0080,  0.0759,  0.0698,  0.0815,\n",
      "        -0.0691, -0.0685,  0.0122, -0.0131, -0.0578, -0.0581, -0.0498,  0.0029,\n",
      "        -0.0164,  0.0002, -0.0326, -0.0471, -0.0175, -0.0281,  0.0181, -0.0318,\n",
      "         0.0498, -0.0141, -0.0179, -0.0225, -0.0074,  0.0683,  0.0277,  0.0091,\n",
      "         0.0149,  0.0555,  0.0594, -0.0047,  0.0202,  0.0796,  0.0257,  0.0584,\n",
      "        -0.0078,  0.0051,  0.0815,  0.0813, -0.0065, -0.0260, -0.0050, -0.0756],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0355, -0.0747,  0.0710,  ...,  0.0342, -0.0272,  0.0093],\n",
      "        [ 0.0882,  0.0611, -0.0061,  ...,  0.0490,  0.0550, -0.0448],\n",
      "        [ 0.0122,  0.0430, -0.0119,  ..., -0.0413,  0.0422, -0.0725],\n",
      "        ...,\n",
      "        [ 0.0445, -0.0848,  0.0524,  ..., -0.0121, -0.0122,  0.0776],\n",
      "        [-0.0079,  0.0378, -0.0490,  ...,  0.0414, -0.0492, -0.0334],\n",
      "        [ 0.0296,  0.0368, -0.0711,  ...,  0.0600,  0.0493,  0.0672]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0513,  0.0181,  0.0685, -0.0457, -0.0069,  0.0135,  0.0798, -0.0296,\n",
      "        -0.0410,  0.0011, -0.0228,  0.0063,  0.0507, -0.0850, -0.0585,  0.0487,\n",
      "         0.0810, -0.0274,  0.0368, -0.0442,  0.0480,  0.0544,  0.0666,  0.0691,\n",
      "         0.0252, -0.0247, -0.0614,  0.0739, -0.0644, -0.0032,  0.0770,  0.0455,\n",
      "         0.0387, -0.0822, -0.0309,  0.0438,  0.0719, -0.0583,  0.0698,  0.0086,\n",
      "         0.0876, -0.0698,  0.0491, -0.0546, -0.0331, -0.0115,  0.0351, -0.0591,\n",
      "        -0.0139, -0.0140, -0.0652, -0.0871, -0.0798,  0.0767,  0.0440,  0.0188,\n",
      "        -0.0719, -0.0240,  0.0556, -0.0301, -0.0220, -0.0400, -0.0778,  0.0005,\n",
      "         0.0300,  0.0110, -0.0599,  0.0099,  0.0510,  0.0058,  0.0415,  0.0784,\n",
      "        -0.0411, -0.0764, -0.0637,  0.0799,  0.0711,  0.0065,  0.0740, -0.0500,\n",
      "        -0.0529,  0.0255,  0.0353,  0.0491,  0.0772,  0.0232,  0.0393, -0.0009,\n",
      "         0.0533, -0.0252, -0.0257, -0.0671,  0.0753, -0.0845, -0.0419,  0.0676,\n",
      "        -0.0011, -0.0647,  0.0512,  0.0773, -0.0299,  0.0178,  0.0301,  0.0048,\n",
      "        -0.0529, -0.0032,  0.0166,  0.0634, -0.0659,  0.0228, -0.0176, -0.0753,\n",
      "         0.0013,  0.0122, -0.0697, -0.0490,  0.0715,  0.0337, -0.0087,  0.0587,\n",
      "         0.0821, -0.0412, -0.0328,  0.0513, -0.0401,  0.0790,  0.0724, -0.0774],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0185, -0.0562, -0.0133,  ...,  0.0632,  0.0858, -0.0379],\n",
      "        [-0.0697,  0.0801,  0.0816,  ...,  0.0592,  0.0566, -0.0342],\n",
      "        [ 0.0706, -0.0836, -0.0826,  ..., -0.0270, -0.0209,  0.0479],\n",
      "        ...,\n",
      "        [ 0.0346,  0.0710, -0.0060,  ..., -0.0865, -0.0151, -0.0668],\n",
      "        [ 0.0625,  0.0713,  0.0139,  ..., -0.0278,  0.0565,  0.0158],\n",
      "        [ 0.0345,  0.0192, -0.0763,  ..., -0.0880,  0.0838, -0.0479]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0391,  0.0037, -0.0796, -0.0709,  0.0093,  0.0768, -0.0268,  0.0349,\n",
      "         0.0410, -0.0538,  0.0091, -0.0022,  0.0803, -0.0237,  0.0442, -0.0621,\n",
      "        -0.0114,  0.0714, -0.0617, -0.0854,  0.0695,  0.0159, -0.0471, -0.0049,\n",
      "         0.0802,  0.0590, -0.0783,  0.0551,  0.0715,  0.0581,  0.0337,  0.0673,\n",
      "         0.0811, -0.0254,  0.0215, -0.0497, -0.0486, -0.0685, -0.0721, -0.0845,\n",
      "        -0.0122, -0.0160, -0.0464,  0.0321,  0.0777, -0.0218,  0.0240,  0.0650,\n",
      "         0.0400, -0.0872, -0.0582, -0.0488,  0.0169,  0.0290,  0.0855, -0.0818,\n",
      "        -0.0559, -0.0103,  0.0105, -0.0114,  0.0670, -0.0644, -0.0162,  0.0431,\n",
      "         0.0464, -0.0498,  0.0645,  0.0548, -0.0010, -0.0429,  0.0274,  0.0524,\n",
      "         0.0368, -0.0391, -0.0455,  0.0118, -0.0305, -0.0152,  0.0272,  0.0764,\n",
      "         0.0341,  0.0011,  0.0604,  0.0083, -0.0708,  0.0109,  0.0342, -0.0199,\n",
      "         0.0046,  0.0117, -0.0111,  0.0416, -0.0276,  0.0244, -0.0693, -0.0092,\n",
      "        -0.0749, -0.0760,  0.0029,  0.0883,  0.0062,  0.0096,  0.0423, -0.0380,\n",
      "        -0.0332,  0.0285,  0.0177, -0.0539, -0.0722, -0.0190,  0.0047, -0.0035,\n",
      "        -0.0017,  0.0115, -0.0857, -0.0485,  0.0508,  0.0584,  0.0497, -0.0010,\n",
      "         0.0457,  0.0125, -0.0107,  0.0263,  0.0852,  0.0795,  0.0453, -0.0003],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0601,  0.0607, -0.0729,  ...,  0.0618, -0.0196,  0.0438],\n",
      "        [-0.0599, -0.0259, -0.0530,  ...,  0.0066,  0.0652,  0.0566],\n",
      "        [ 0.0324,  0.0310,  0.0341,  ..., -0.0694,  0.0428,  0.0282],\n",
      "        ...,\n",
      "        [-0.0196, -0.0659, -0.0320,  ...,  0.0591,  0.0860, -0.0551],\n",
      "        [ 0.0461, -0.0692,  0.0259,  ..., -0.0108,  0.0641,  0.0021],\n",
      "        [ 0.0562, -0.0293, -0.0272,  ...,  0.0721,  0.0484,  0.0553]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0599,  0.0665,  0.0084, -0.0651, -0.0188, -0.0596,  0.0243, -0.0227,\n",
      "         0.0602, -0.0323, -0.0112,  0.0141, -0.0186,  0.0805,  0.0320,  0.0321,\n",
      "        -0.0676, -0.0010,  0.0030, -0.0185, -0.0191, -0.0714,  0.0203, -0.0575,\n",
      "        -0.0766,  0.0805,  0.0717,  0.0591,  0.0521, -0.0594,  0.0251, -0.0797,\n",
      "         0.0429,  0.0471, -0.0439,  0.0336, -0.0832, -0.0124, -0.0200,  0.0657,\n",
      "        -0.0860,  0.0692,  0.0672,  0.0269, -0.0507,  0.0058,  0.0296,  0.0854,\n",
      "        -0.0075,  0.0077,  0.0232,  0.0176,  0.0723, -0.0379, -0.0186, -0.0617,\n",
      "         0.0051, -0.0054, -0.0834,  0.0290, -0.0183, -0.0092,  0.0204, -0.0870,\n",
      "        -0.0417,  0.0612, -0.0370, -0.0847, -0.0231, -0.0234,  0.0128,  0.0012,\n",
      "         0.0110, -0.0718, -0.0190,  0.0017, -0.0501,  0.0313, -0.0695, -0.0333,\n",
      "         0.0345,  0.0326, -0.0213, -0.0548,  0.0767,  0.0539,  0.0694, -0.0016,\n",
      "         0.0008, -0.0170,  0.0066, -0.0729, -0.0239, -0.0370, -0.0017,  0.0408,\n",
      "         0.0588,  0.0619, -0.0549, -0.0232,  0.0607,  0.0329, -0.0463,  0.0071,\n",
      "         0.0280,  0.0065, -0.0213,  0.0227,  0.0126,  0.0428,  0.0448, -0.0674,\n",
      "        -0.0847, -0.0262,  0.0183, -0.0766, -0.0709,  0.0164, -0.0834, -0.0213,\n",
      "        -0.0254, -0.0873,  0.0626, -0.0276,  0.0149,  0.0048,  0.0667, -0.0438],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0829,  0.0841,  0.0527,  ...,  0.0729,  0.0054,  0.0881],\n",
      "        [-0.0642, -0.0002,  0.0291,  ..., -0.0093, -0.0459, -0.0482],\n",
      "        [-0.0783,  0.0789, -0.0341,  ..., -0.0594,  0.0806,  0.0122],\n",
      "        ...,\n",
      "        [ 0.0402, -0.0617, -0.0318,  ...,  0.0308, -0.0800, -0.0246],\n",
      "        [ 0.0850, -0.0011, -0.0590,  ..., -0.0422, -0.0803, -0.0056],\n",
      "        [ 0.0635,  0.0240, -0.0668,  ...,  0.0662,  0.0325, -0.0500]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0785,  0.0805, -0.0826,  0.0625, -0.0330, -0.0612, -0.0483,  0.0597,\n",
      "        -0.0220,  0.0762,  0.0462, -0.0769,  0.0217, -0.0123, -0.0197,  0.0450,\n",
      "        -0.0389,  0.0825, -0.0087, -0.0210, -0.0708, -0.0719, -0.0547, -0.0553,\n",
      "        -0.0457,  0.0330,  0.0757, -0.0610, -0.0326, -0.0211, -0.0237,  0.0671,\n",
      "        -0.0169, -0.0485, -0.0461,  0.0154, -0.0845, -0.0139, -0.0675,  0.0767,\n",
      "        -0.0191, -0.0803,  0.0853,  0.0378, -0.0577,  0.0285,  0.0014, -0.0354,\n",
      "        -0.0520, -0.0793, -0.0581, -0.0039,  0.0051,  0.0867, -0.0477,  0.0113,\n",
      "         0.0111, -0.0490,  0.0673,  0.0273,  0.0642, -0.0758,  0.0273, -0.0696,\n",
      "        -0.0167, -0.0359, -0.0295, -0.0039, -0.0559,  0.0473, -0.0577, -0.0225,\n",
      "         0.0817,  0.0530,  0.0654,  0.0488, -0.0438, -0.0813,  0.0881,  0.0443,\n",
      "        -0.0598, -0.0325,  0.0353,  0.0542,  0.0638, -0.0513, -0.0014, -0.0551,\n",
      "         0.0351,  0.0684, -0.0044,  0.0842, -0.0373, -0.0042,  0.0006,  0.0855,\n",
      "         0.0084, -0.0279, -0.0218,  0.0351, -0.0852, -0.0319, -0.0491,  0.0610,\n",
      "        -0.0203, -0.0221,  0.0239, -0.0028,  0.0641, -0.0372,  0.0466,  0.0153,\n",
      "         0.0565, -0.0731, -0.0697, -0.0307,  0.0556,  0.0190, -0.0005,  0.0843,\n",
      "        -0.0204, -0.0762,  0.0579,  0.0341, -0.0138, -0.0375,  0.0313,  0.0514],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0362, -0.0261, -0.0074,  ..., -0.0585, -0.0278,  0.0586],\n",
      "        [ 0.0234, -0.0014,  0.0853,  ..., -0.0502,  0.0769,  0.0582],\n",
      "        [ 0.0126, -0.0370, -0.0483,  ...,  0.0078, -0.0836,  0.0204],\n",
      "        ...,\n",
      "        [ 0.0294,  0.0095,  0.0691,  ...,  0.0480, -0.0572,  0.0562],\n",
      "        [ 0.0384, -0.0119,  0.0682,  ...,  0.0051,  0.0325, -0.0405],\n",
      "        [-0.0696, -0.0164,  0.0202,  ...,  0.0226,  0.0592,  0.0846]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-8.0861e-02, -6.1336e-03,  3.5421e-02, -5.4880e-02,  5.4365e-02,\n",
      "        -1.4961e-02,  4.0504e-02,  2.5746e-02,  7.4896e-02,  1.9823e-02,\n",
      "         8.7413e-02, -1.5861e-02,  5.7215e-02,  4.4591e-02,  7.2207e-02,\n",
      "        -1.8893e-02, -4.0249e-02, -6.6057e-02, -2.6051e-02, -2.6007e-02,\n",
      "         4.2882e-02,  7.1737e-02,  1.9885e-02,  7.4064e-02,  6.5449e-02,\n",
      "        -4.0633e-02, -7.9460e-02,  3.3202e-02, -4.6566e-02, -2.1224e-02,\n",
      "        -5.9982e-02, -4.8979e-02, -4.5220e-02,  4.1778e-02, -1.9951e-02,\n",
      "         1.4998e-02, -2.6933e-02,  5.7422e-05,  8.1462e-03, -6.0363e-02,\n",
      "         6.5255e-02, -5.0802e-02,  3.4266e-02,  2.2184e-02,  8.5042e-02,\n",
      "        -8.3655e-03,  6.7504e-02,  2.8075e-02,  3.9763e-02, -1.5634e-02,\n",
      "         2.9804e-02,  6.3618e-02,  6.5023e-02, -8.2060e-02,  7.4174e-02,\n",
      "        -8.5122e-02,  6.1530e-02, -3.2303e-03, -1.9666e-02,  3.0668e-02,\n",
      "        -3.7198e-02, -4.3024e-02, -8.3036e-02, -3.5111e-02,  3.5568e-02,\n",
      "         7.0486e-02,  3.7113e-02,  7.7078e-02, -5.9764e-02,  1.3096e-02,\n",
      "         6.6460e-02,  8.1075e-02,  3.7664e-02,  2.2548e-02,  1.5056e-02,\n",
      "        -4.1745e-02, -6.8139e-02, -2.9981e-02, -6.7533e-02,  5.1202e-02,\n",
      "        -8.7538e-02,  4.5110e-02,  3.9513e-02, -6.7796e-02, -2.9698e-03,\n",
      "         3.8593e-02, -5.5300e-02,  2.4145e-02,  2.2618e-03, -4.8032e-02,\n",
      "        -4.9852e-02,  3.1082e-02,  8.4193e-02, -7.0822e-02, -7.6588e-03,\n",
      "         8.8051e-02,  8.3090e-03,  3.1772e-02,  2.3320e-02,  7.3868e-02,\n",
      "        -7.7348e-02, -8.4352e-04,  7.5933e-02, -5.4063e-02, -2.7592e-02,\n",
      "         3.2072e-02, -7.6127e-02, -4.2399e-02,  6.8579e-02, -8.1091e-03,\n",
      "         5.4462e-02,  6.5067e-02, -7.3086e-02,  1.7349e-02,  6.9414e-02,\n",
      "         7.0368e-02,  7.6712e-02, -3.9773e-02,  3.2257e-02, -3.4206e-02,\n",
      "         1.5741e-02, -2.1496e-02,  8.8033e-02,  2.6780e-02, -7.5439e-02,\n",
      "         8.3500e-02, -3.1643e-02, -7.5566e-02], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0359, -0.0507, -0.0655,  ...,  0.0811,  0.0659,  0.0505],\n",
      "        [-0.0194, -0.0109, -0.0132,  ..., -0.0713, -0.0006, -0.0793],\n",
      "        [-0.0467,  0.0394, -0.0073,  ...,  0.0119,  0.0213, -0.0344],\n",
      "        ...,\n",
      "        [-0.0862, -0.0479, -0.0567,  ...,  0.0515,  0.0037, -0.0572],\n",
      "        [ 0.0493,  0.0435,  0.0773,  ..., -0.0191, -0.0428, -0.0209],\n",
      "        [-0.0443, -0.0784, -0.0686,  ..., -0.0357,  0.0803, -0.0269]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0743,  0.0292, -0.0527, -0.0255,  0.0613,  0.0176, -0.0107, -0.0530,\n",
      "         0.0317,  0.0122, -0.0683, -0.0676,  0.0811, -0.0370,  0.0411, -0.0398,\n",
      "         0.0647, -0.0087,  0.0166,  0.0401, -0.0172,  0.0453,  0.0204, -0.0857,\n",
      "         0.0250, -0.0253, -0.0739,  0.0812,  0.0326,  0.0126, -0.0148, -0.0760,\n",
      "        -0.0128, -0.0105, -0.0508,  0.0430,  0.0103,  0.0431,  0.0143, -0.0833,\n",
      "        -0.0302,  0.0421,  0.0583,  0.0656,  0.0742, -0.0766,  0.0793,  0.0831,\n",
      "        -0.0856, -0.0281,  0.0366, -0.0820, -0.0223, -0.0385,  0.0672, -0.0828,\n",
      "         0.0491,  0.0099,  0.0037,  0.0388, -0.0756,  0.0629, -0.0263, -0.0099,\n",
      "        -0.0710, -0.0861, -0.0439,  0.0076,  0.0086, -0.0347,  0.0147, -0.0441,\n",
      "        -0.0822,  0.0295, -0.0833,  0.0564, -0.0867, -0.0022,  0.0651, -0.0539,\n",
      "        -0.0678, -0.0580,  0.0856,  0.0294, -0.0577, -0.0881, -0.0399, -0.0524,\n",
      "        -0.0646,  0.0599, -0.0675,  0.0633, -0.0431,  0.0750, -0.0400,  0.0814,\n",
      "         0.0444, -0.0203,  0.0427,  0.0431,  0.0463,  0.0391, -0.0342, -0.0720,\n",
      "         0.0718, -0.0751, -0.0797,  0.0686, -0.0474, -0.0639, -0.0483, -0.0140,\n",
      "         0.0435,  0.0039,  0.0281,  0.0538, -0.0569,  0.0024,  0.0730, -0.0220,\n",
      "        -0.0011,  0.0037,  0.0304, -0.0695, -0.0714, -0.0637, -0.0569, -0.0777],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0099, -0.0031,  0.0882,  ...,  0.0060,  0.0084,  0.0748],\n",
      "        [ 0.0139,  0.0210,  0.0323,  ...,  0.0028,  0.0406,  0.0045],\n",
      "        [ 0.0268, -0.0731,  0.0064,  ...,  0.0451, -0.0279,  0.0327],\n",
      "        ...,\n",
      "        [ 0.0128,  0.0790, -0.0748,  ..., -0.0276, -0.0011, -0.0647],\n",
      "        [ 0.0556, -0.0841,  0.0723,  ...,  0.0846,  0.0285, -0.0243],\n",
      "        [-0.0837, -0.0317, -0.0129,  ..., -0.0773,  0.0633, -0.0640]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0283, -0.0572, -0.0851,  0.0382, -0.0583,  0.0011, -0.0189,  0.0292,\n",
      "        -0.0561,  0.0691,  0.0792, -0.0419, -0.0156, -0.0662, -0.0426, -0.0664,\n",
      "        -0.0221,  0.0012, -0.0821, -0.0461, -0.0150, -0.0039, -0.0212,  0.0841,\n",
      "         0.0859, -0.0701, -0.0071,  0.0683,  0.0052, -0.0517, -0.0696,  0.0484,\n",
      "        -0.0839,  0.0565,  0.0857,  0.0688, -0.0778, -0.0456, -0.0015,  0.0099,\n",
      "        -0.0342, -0.0206,  0.0072,  0.0727, -0.0852,  0.0332, -0.0659,  0.0442,\n",
      "        -0.0860, -0.0323,  0.0563,  0.0492, -0.0491,  0.0062,  0.0655, -0.0348,\n",
      "        -0.0716, -0.0120,  0.0075, -0.0463, -0.0419,  0.0694, -0.0814, -0.0688,\n",
      "         0.0145, -0.0836,  0.0832, -0.0606, -0.0545, -0.0129, -0.0310, -0.0002,\n",
      "         0.0161, -0.0057, -0.0269, -0.0368,  0.0371,  0.0677,  0.0352, -0.0031,\n",
      "        -0.0165, -0.0470,  0.0436,  0.0147, -0.0864,  0.0537,  0.0506,  0.0170,\n",
      "         0.0077, -0.0612,  0.0626,  0.0568, -0.0505,  0.0422, -0.0304,  0.0402,\n",
      "         0.0384, -0.0054, -0.0623, -0.0009, -0.0768,  0.0109, -0.0747, -0.0685,\n",
      "         0.0288, -0.0600, -0.0535, -0.0577,  0.0659,  0.0484,  0.0596, -0.0716,\n",
      "         0.0609,  0.0664,  0.0115, -0.0644, -0.0039, -0.0336,  0.0416,  0.0258,\n",
      "         0.0686, -0.0015, -0.0473, -0.0116,  0.0446, -0.0317, -0.0316, -0.0384],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0180,  0.0178, -0.0821,  ...,  0.0251, -0.0737, -0.0884],\n",
      "        [ 0.0831,  0.0875, -0.0361,  ..., -0.0327,  0.0246, -0.0702],\n",
      "        [-0.0318, -0.0022, -0.0018,  ...,  0.0121, -0.0749, -0.0296],\n",
      "        ...,\n",
      "        [ 0.0187, -0.0687,  0.0434,  ..., -0.0259, -0.0127, -0.0799],\n",
      "        [-0.0274,  0.0364, -0.0580,  ..., -0.0523,  0.0008, -0.0560],\n",
      "        [ 0.0303, -0.0194, -0.0448,  ...,  0.0204,  0.0176,  0.0204]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 7.3690e-02, -3.6852e-02, -4.7312e-02,  5.8374e-02, -2.1846e-02,\n",
      "        -1.4352e-02,  5.2399e-02, -3.2734e-02,  8.4724e-03, -2.4738e-02,\n",
      "         3.1613e-03, -4.5906e-02, -2.7886e-02,  5.3878e-02, -3.1562e-02,\n",
      "         8.1119e-02, -5.4873e-03, -8.0577e-02,  2.6305e-02,  3.1051e-02,\n",
      "        -3.5147e-03,  7.2824e-02, -8.7902e-02, -2.8241e-02,  3.7579e-02,\n",
      "         3.5140e-02, -4.6663e-02,  3.4542e-02,  1.2327e-03,  3.2441e-02,\n",
      "        -2.4012e-02,  7.0324e-02, -3.7578e-02,  7.8169e-02,  7.3518e-02,\n",
      "        -2.8595e-02,  4.9897e-02,  2.7828e-02,  5.8907e-02,  9.3231e-03,\n",
      "        -7.3474e-02, -8.6563e-02, -7.2372e-02, -7.0459e-02,  4.7338e-02,\n",
      "        -2.9084e-02, -6.3402e-02,  3.0008e-02, -3.2182e-02,  5.6102e-02,\n",
      "        -7.0934e-02, -4.4519e-02, -7.3566e-02, -9.4602e-03,  3.6920e-02,\n",
      "        -4.6120e-02, -5.7023e-02,  2.8237e-02, -2.4951e-03, -9.3736e-05,\n",
      "        -3.2737e-02, -1.2955e-02, -3.7493e-02, -5.1857e-02,  4.4577e-02,\n",
      "        -6.0910e-02, -6.4220e-02,  2.1303e-02, -7.9832e-02, -6.4334e-02,\n",
      "         3.8460e-02,  6.4473e-02, -1.5853e-02, -7.2538e-03,  2.8778e-02,\n",
      "         2.7291e-03,  5.8693e-02, -7.4750e-02, -2.3789e-02,  3.0828e-02,\n",
      "        -8.8179e-02, -5.6000e-02,  1.7887e-02,  4.4625e-02, -6.6745e-02,\n",
      "        -5.1651e-02, -2.2369e-02, -2.7465e-02, -4.5580e-02, -1.8419e-02,\n",
      "        -2.1817e-02,  7.0413e-02,  3.3936e-02,  7.8979e-02, -4.6606e-02,\n",
      "         8.0774e-02, -7.3122e-02,  4.4887e-03, -2.0714e-02, -7.1546e-02,\n",
      "         2.9250e-02,  3.0249e-02, -3.9941e-02, -1.7742e-02,  5.8112e-02,\n",
      "         4.0942e-02,  7.0968e-02, -3.5696e-02,  6.8692e-02,  6.8590e-04,\n",
      "        -7.6929e-02, -5.6350e-02, -5.7377e-02,  1.0497e-02,  8.7849e-02,\n",
      "         7.8561e-02, -3.5079e-02, -3.6935e-03,  8.5947e-02,  4.7251e-02,\n",
      "         1.6501e-02,  8.7974e-02, -6.1087e-02, -6.6015e-02,  7.9949e-02,\n",
      "        -6.8334e-02,  2.8127e-02,  6.2016e-02], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0830, -0.0596,  0.0263,  ..., -0.0095,  0.0413, -0.0596],\n",
      "        [ 0.0786, -0.0716,  0.0496,  ..., -0.0560,  0.0625,  0.0551],\n",
      "        [-0.0254,  0.0393, -0.0715,  ...,  0.0214, -0.0820, -0.0816],\n",
      "        ...,\n",
      "        [ 0.0037,  0.0320, -0.0486,  ..., -0.0522,  0.0318,  0.0615],\n",
      "        [-0.0184,  0.0135, -0.0485,  ...,  0.0342, -0.0810,  0.0216],\n",
      "        [-0.0509, -0.0220, -0.0674,  ..., -0.0368,  0.0028,  0.0173]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0537, -0.0146,  0.0458, -0.0477,  0.0227, -0.0247,  0.0014,  0.0541,\n",
      "         0.0686, -0.0526,  0.0400, -0.0100, -0.0431,  0.0566, -0.0096, -0.0474,\n",
      "         0.0398, -0.0428,  0.0449, -0.0620,  0.0185, -0.0225, -0.0345,  0.0274,\n",
      "         0.0578,  0.0650,  0.0581,  0.0817,  0.0361, -0.0411, -0.0511, -0.0791,\n",
      "        -0.0320, -0.0124,  0.0537, -0.0024,  0.0290,  0.0766,  0.0433, -0.0019,\n",
      "        -0.0331,  0.0071, -0.0363, -0.0537, -0.0599,  0.0740,  0.0059,  0.0653,\n",
      "         0.0523,  0.0292,  0.0828,  0.0060, -0.0611,  0.0420,  0.0055,  0.0278,\n",
      "         0.0196, -0.0718, -0.0183, -0.0578, -0.0802,  0.0443,  0.0004,  0.0315,\n",
      "         0.0602,  0.0218,  0.0840, -0.0575, -0.0425, -0.0374, -0.0713,  0.0235,\n",
      "        -0.0223, -0.0063, -0.0175,  0.0798, -0.0099, -0.0040, -0.0728, -0.0118,\n",
      "        -0.0090, -0.0600,  0.0740, -0.0060, -0.0064,  0.0765, -0.0551,  0.0048,\n",
      "         0.0755,  0.0579, -0.0572, -0.0230, -0.0757, -0.0118,  0.0165,  0.0387,\n",
      "         0.0340,  0.0529, -0.0238,  0.0595,  0.0858,  0.0589, -0.0320,  0.0768,\n",
      "         0.0782,  0.0293,  0.0662,  0.0634,  0.0173,  0.0806, -0.0133,  0.0490,\n",
      "         0.0374, -0.0569, -0.0202, -0.0626,  0.0042, -0.0052,  0.0638,  0.0072,\n",
      "         0.0689, -0.0178,  0.0223, -0.0329,  0.0880, -0.0330, -0.0356, -0.0313],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0144, -0.0041,  0.0557,  ..., -0.0702, -0.0756, -0.0831],\n",
      "        [-0.0004,  0.0654,  0.0158,  ...,  0.0599,  0.0278,  0.0607],\n",
      "        [-0.0339, -0.0411, -0.0493,  ...,  0.0557,  0.0018, -0.0708],\n",
      "        ...,\n",
      "        [ 0.0638,  0.0635, -0.0575,  ..., -0.0353, -0.0745, -0.0636],\n",
      "        [ 0.0168,  0.0471, -0.0010,  ..., -0.0879, -0.0770, -0.0878],\n",
      "        [-0.0739,  0.0836, -0.0850,  ..., -0.0165, -0.0303,  0.0083]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0522, -0.0123,  0.0251, -0.0434,  0.0146, -0.0063,  0.0016,  0.0483,\n",
      "         0.0113,  0.0758, -0.0473,  0.0016,  0.0069,  0.0544, -0.0133,  0.0696,\n",
      "         0.0004, -0.0694, -0.0447, -0.0693, -0.0322, -0.0339,  0.0431,  0.0701,\n",
      "        -0.0443,  0.0187,  0.0291, -0.0324,  0.0628,  0.0696,  0.0104, -0.0341,\n",
      "        -0.0297,  0.0478, -0.0486,  0.0627,  0.0158, -0.0172, -0.0419, -0.0030,\n",
      "        -0.0781, -0.0300,  0.0824, -0.0098,  0.0545, -0.0320, -0.0157,  0.0683,\n",
      "         0.0596,  0.0092, -0.0795, -0.0271,  0.0012,  0.0526,  0.0633,  0.0765,\n",
      "         0.0437,  0.0783, -0.0075,  0.0773, -0.0054,  0.0040,  0.0748, -0.0807,\n",
      "         0.0829,  0.0549,  0.0334,  0.0274, -0.0472, -0.0536,  0.0111,  0.0008,\n",
      "        -0.0858, -0.0085, -0.0857,  0.0020, -0.0637,  0.0562, -0.0499,  0.0116,\n",
      "        -0.0760, -0.0765,  0.0123,  0.0333,  0.0678,  0.0697,  0.0786, -0.0235,\n",
      "         0.0374, -0.0522,  0.0740,  0.0416, -0.0296,  0.0771,  0.0329,  0.0139,\n",
      "         0.0314,  0.0313,  0.0081, -0.0475,  0.0540,  0.0406,  0.0536,  0.0637,\n",
      "        -0.0556, -0.0390,  0.0631, -0.0575, -0.0063, -0.0552, -0.0371,  0.0541,\n",
      "         0.0184, -0.0483, -0.0798, -0.0748,  0.0160, -0.0053,  0.0727,  0.0604,\n",
      "         0.0173, -0.0047,  0.0102, -0.0822, -0.0401, -0.0092, -0.0706, -0.0226],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0504, -0.0664,  0.0845,  ...,  0.0725,  0.0829, -0.0220],\n",
      "        [-0.0074, -0.0373,  0.0861,  ..., -0.0231,  0.0786,  0.0727],\n",
      "        [-0.0859,  0.0754, -0.0405,  ..., -0.0874,  0.0555,  0.0432],\n",
      "        ...,\n",
      "        [-0.0642, -0.0089, -0.0108,  ...,  0.0048,  0.0617,  0.0350],\n",
      "        [-0.0533, -0.0154,  0.0641,  ..., -0.0696,  0.0828,  0.0408],\n",
      "        [ 0.0206,  0.0683, -0.0517,  ...,  0.0295,  0.0409,  0.0662]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0753,  0.0574, -0.0390,  0.0473, -0.0358,  0.0288, -0.0492,  0.0142,\n",
      "         0.0382, -0.0541,  0.0189,  0.0068, -0.0828, -0.0016, -0.0717, -0.0831,\n",
      "        -0.0124,  0.0195,  0.0072,  0.0755,  0.0233, -0.0652,  0.0576,  0.0524,\n",
      "        -0.0522, -0.0664,  0.0732,  0.0422,  0.0541, -0.0732, -0.0372,  0.0443,\n",
      "        -0.0571,  0.0460,  0.0829, -0.0309, -0.0341, -0.0273,  0.0698, -0.0723,\n",
      "        -0.0531, -0.0027,  0.0078,  0.0107, -0.0832, -0.0630, -0.0153, -0.0102,\n",
      "         0.0087, -0.0862, -0.0715, -0.0096,  0.0433, -0.0285,  0.0647, -0.0463,\n",
      "        -0.0230,  0.0727,  0.0279, -0.0615, -0.0604, -0.0283, -0.0149, -0.0509,\n",
      "         0.0517, -0.0856,  0.0035, -0.0049,  0.0780,  0.0682,  0.0155,  0.0507,\n",
      "         0.0341,  0.0793,  0.0306,  0.0343,  0.0728, -0.0709,  0.0339, -0.0167,\n",
      "         0.0614,  0.0712, -0.0574,  0.0408, -0.0032,  0.0348,  0.0297, -0.0710,\n",
      "        -0.0331, -0.0273,  0.0022,  0.0839, -0.0187,  0.0050, -0.0108,  0.0295,\n",
      "         0.0117, -0.0536,  0.0532,  0.0861, -0.0844,  0.0088,  0.0306,  0.0468,\n",
      "         0.0825,  0.0342,  0.0718,  0.0712, -0.0591,  0.0679, -0.0127,  0.0451,\n",
      "        -0.0688, -0.0104,  0.0007,  0.0596,  0.0758, -0.0657, -0.0668,  0.0445,\n",
      "         0.0789, -0.0095, -0.0349,  0.0073, -0.0331, -0.0420, -0.0553, -0.0754],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0388, -0.0670, -0.0859,  ..., -0.0011, -0.0467, -0.0081],\n",
      "        [ 0.0377, -0.0196,  0.0548,  ..., -0.0545, -0.0303, -0.0458],\n",
      "        [ 0.0667,  0.0283, -0.0521,  ..., -0.0057, -0.0154,  0.0788],\n",
      "        ...,\n",
      "        [-0.0049,  0.0465,  0.0694,  ..., -0.0482,  0.0383,  0.0544],\n",
      "        [-0.0628, -0.0410,  0.0381,  ..., -0.0070, -0.0617, -0.0386],\n",
      "        [ 0.0045, -0.0473,  0.0005,  ...,  0.0320, -0.0829, -0.0702]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0154,  0.0308,  0.0601,  0.0432, -0.0527, -0.0068,  0.0346,  0.0555,\n",
      "        -0.0863, -0.0793,  0.0758,  0.0223, -0.0489,  0.0781,  0.0105, -0.0565,\n",
      "        -0.0452,  0.0510, -0.0049,  0.0394, -0.0181,  0.0433,  0.0143,  0.0252,\n",
      "        -0.0733, -0.0797, -0.0498,  0.0088,  0.0052, -0.0352,  0.0709,  0.0659,\n",
      "         0.0388, -0.0395, -0.0506,  0.0263,  0.0546, -0.0814, -0.0442,  0.0442,\n",
      "        -0.0461, -0.0829, -0.0559, -0.0090, -0.0320, -0.0857, -0.0780,  0.0672,\n",
      "         0.0159,  0.0068,  0.0343,  0.0003,  0.0480,  0.0619, -0.0161,  0.0604,\n",
      "         0.0864,  0.0250,  0.0503, -0.0499,  0.0603, -0.0090,  0.0045,  0.0667,\n",
      "        -0.0135,  0.0692,  0.0031, -0.0335, -0.0764,  0.0491,  0.0253,  0.0655,\n",
      "         0.0753, -0.0133,  0.0368,  0.0124,  0.0073,  0.0809,  0.0681, -0.0053,\n",
      "        -0.0561,  0.0389,  0.0245, -0.0882, -0.0700,  0.0077, -0.0436, -0.0174,\n",
      "        -0.0270,  0.0356,  0.0100, -0.0777, -0.0857, -0.0646, -0.0002,  0.0207,\n",
      "        -0.0274, -0.0345,  0.0751, -0.0704, -0.0307,  0.0519,  0.0795,  0.0062,\n",
      "        -0.0856,  0.0044, -0.0857, -0.0775, -0.0590, -0.0169,  0.0631, -0.0674,\n",
      "        -0.0038,  0.0599, -0.0535, -0.0149,  0.0501,  0.0824,  0.0669, -0.0486,\n",
      "         0.0688, -0.0489, -0.0460,  0.0826,  0.0426, -0.0828,  0.0301,  0.0590],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0536, -0.0742,  0.0134,  ...,  0.0861,  0.0849, -0.0178],\n",
      "        [-0.0559,  0.0431,  0.0758,  ..., -0.0312,  0.0262,  0.0082],\n",
      "        [-0.0228, -0.0144, -0.0023,  ...,  0.0245, -0.0729, -0.0085],\n",
      "        ...,\n",
      "        [ 0.0771,  0.0607,  0.0078,  ..., -0.0674,  0.0500,  0.0125],\n",
      "        [ 0.0497, -0.0170, -0.0654,  ...,  0.0619,  0.0700, -0.0853],\n",
      "        [ 0.0733,  0.0819, -0.0350,  ..., -0.0759,  0.0228, -0.0728]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0344, -0.0314,  0.0863,  0.0594, -0.0154, -0.0249,  0.0024,  0.0828,\n",
      "         0.0047, -0.0253, -0.0160,  0.0248, -0.0337,  0.0057,  0.0601, -0.0727,\n",
      "        -0.0629,  0.0660,  0.0484, -0.0238, -0.0168, -0.0617,  0.0070, -0.0545,\n",
      "         0.0456,  0.0799, -0.0124,  0.0321,  0.0410, -0.0402,  0.0595, -0.0545,\n",
      "         0.0568,  0.0680,  0.0196, -0.0732, -0.0479,  0.0124,  0.0797, -0.0206,\n",
      "         0.0101, -0.0532,  0.0608, -0.0422, -0.0812, -0.0700,  0.0507,  0.0882,\n",
      "         0.0079, -0.0315,  0.0688, -0.0289,  0.0136, -0.0842, -0.0512, -0.0279,\n",
      "        -0.0287,  0.0423, -0.0036, -0.0763,  0.0590,  0.0117,  0.0275,  0.0137,\n",
      "        -0.0632,  0.0711,  0.0480,  0.0007,  0.0724,  0.0867, -0.0500, -0.0430,\n",
      "         0.0294, -0.0709, -0.0633, -0.0374,  0.0729,  0.0055, -0.0297, -0.0569,\n",
      "        -0.0435,  0.0468,  0.0718,  0.0058,  0.0409,  0.0167,  0.0181,  0.0057,\n",
      "         0.0381,  0.0659, -0.0536, -0.0424,  0.0229,  0.0699, -0.0295,  0.0807,\n",
      "         0.0795, -0.0545, -0.0666, -0.0598, -0.0835, -0.0438, -0.0608,  0.0785,\n",
      "        -0.0688, -0.0093,  0.0835,  0.0859, -0.0678, -0.0395, -0.0220,  0.0672,\n",
      "         0.0206, -0.0248, -0.0568, -0.0683, -0.0247,  0.0423, -0.0657, -0.0613,\n",
      "        -0.0141, -0.0802, -0.0024, -0.0604,  0.0037, -0.0617, -0.0558,  0.0164],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0863, -0.0074, -0.0192,  ...,  0.0417,  0.0517, -0.0363],\n",
      "        [ 0.0699, -0.0380, -0.0785,  ..., -0.0477, -0.0114, -0.0101],\n",
      "        [ 0.0533, -0.0654,  0.0416,  ..., -0.0871, -0.0391, -0.0777],\n",
      "        ...,\n",
      "        [-0.0020,  0.0854, -0.0379,  ...,  0.0393,  0.0859,  0.0541],\n",
      "        [ 0.0237, -0.0875,  0.0069,  ..., -0.0062,  0.0339, -0.0067],\n",
      "        [-0.0716, -0.0466,  0.0165,  ..., -0.0792, -0.0646, -0.0591]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0804, -0.0343,  0.0062, -0.0501,  0.0849, -0.0367, -0.0717,  0.0792,\n",
      "         0.0201, -0.0036, -0.0046, -0.0604,  0.0174, -0.0848,  0.0107, -0.0263,\n",
      "         0.0261, -0.0486, -0.0629, -0.0778,  0.0534,  0.0558, -0.0577,  0.0285,\n",
      "         0.0309,  0.0221, -0.0316, -0.0688, -0.0882, -0.0833, -0.0703,  0.0265,\n",
      "        -0.0354, -0.0453, -0.0316, -0.0137,  0.0740, -0.0222,  0.0615, -0.0304,\n",
      "        -0.0739, -0.0504, -0.0391, -0.0371,  0.0633,  0.0662,  0.0627,  0.0299,\n",
      "         0.0005, -0.0121, -0.0171, -0.0518, -0.0507,  0.0883, -0.0193,  0.0474,\n",
      "         0.0494,  0.0260, -0.0638,  0.0666, -0.0637,  0.0551, -0.0472, -0.0438,\n",
      "         0.0451,  0.0106,  0.0820, -0.0278, -0.0284, -0.0240,  0.0229,  0.0065,\n",
      "         0.0841, -0.0855,  0.0799, -0.0237,  0.0594,  0.0116,  0.0004, -0.0037,\n",
      "        -0.0195,  0.0742,  0.0787, -0.0880, -0.0413,  0.0346,  0.0063,  0.0633,\n",
      "         0.0681,  0.0568, -0.0774, -0.0003, -0.0428,  0.0071, -0.0836,  0.0764,\n",
      "         0.0636,  0.0126,  0.0346, -0.0786, -0.0504,  0.0737, -0.0353,  0.0661,\n",
      "         0.0521, -0.0655, -0.0383,  0.0604,  0.0522,  0.0703, -0.0530,  0.0639,\n",
      "        -0.0765,  0.0145, -0.0694, -0.0715,  0.0701,  0.0436,  0.0211,  0.0520,\n",
      "        -0.0099,  0.0881,  0.0610,  0.0031,  0.0508,  0.0286, -0.0660, -0.0844],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0147, -0.0137, -0.0080,  ...,  0.0528,  0.0342,  0.0505],\n",
      "        [ 0.0442,  0.0235, -0.0072,  ...,  0.0464,  0.0510,  0.0135],\n",
      "        [ 0.0075,  0.0821, -0.0404,  ..., -0.0708, -0.0099, -0.0824],\n",
      "        ...,\n",
      "        [ 0.0226,  0.0651,  0.0072,  ...,  0.0834,  0.0323, -0.0865],\n",
      "        [ 0.0508,  0.0032, -0.0108,  ..., -0.0781, -0.0424,  0.0793],\n",
      "        [ 0.0049,  0.0560,  0.0029,  ..., -0.0645, -0.0537,  0.0245]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0452, -0.0019,  0.0288,  0.0560,  0.0637, -0.0545,  0.0702, -0.0594,\n",
      "         0.0742,  0.0174,  0.0128,  0.0439,  0.0220, -0.0884,  0.0348,  0.0458,\n",
      "        -0.0847,  0.0502,  0.0693,  0.0099,  0.0037,  0.0003, -0.0286, -0.0328,\n",
      "         0.0229, -0.0086, -0.0233,  0.0222, -0.0364, -0.0298,  0.0552, -0.0158,\n",
      "        -0.0382,  0.0812,  0.0717,  0.0360,  0.0198,  0.0862, -0.0142,  0.0368,\n",
      "        -0.0711, -0.0631,  0.0682, -0.0602, -0.0089,  0.0016,  0.0310,  0.0719,\n",
      "        -0.0740,  0.0729,  0.0483,  0.0095, -0.0355, -0.0083, -0.0183,  0.0737,\n",
      "        -0.0402,  0.0616,  0.0060, -0.0029,  0.0457, -0.0022,  0.0535, -0.0234,\n",
      "         0.0234, -0.0650, -0.0804, -0.0637,  0.0291, -0.0123, -0.0254, -0.0459,\n",
      "         0.0321, -0.0757,  0.0036, -0.0069, -0.0407,  0.0703, -0.0219, -0.0271,\n",
      "         0.0410, -0.0321,  0.0410, -0.0515, -0.0785,  0.0410,  0.0456, -0.0649,\n",
      "         0.0325, -0.0095, -0.0565, -0.0086,  0.0254, -0.0303, -0.0269, -0.0188,\n",
      "         0.0550,  0.0550, -0.0582, -0.0284, -0.0788,  0.0851, -0.0287,  0.0797,\n",
      "        -0.0016,  0.0882,  0.0212, -0.0444, -0.0742, -0.0183, -0.0599, -0.0357,\n",
      "        -0.0844, -0.0013, -0.0337,  0.0466, -0.0347, -0.0826, -0.0376,  0.0374,\n",
      "         0.0003, -0.0447, -0.0116, -0.0695,  0.0675, -0.0317, -0.0542, -0.0447],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0117,  0.0389, -0.0758,  ...,  0.0638,  0.0217,  0.0521],\n",
      "        [ 0.0513,  0.0187, -0.0160,  ..., -0.0753, -0.0116,  0.0749],\n",
      "        [ 0.0354,  0.0318, -0.0565,  ..., -0.0674, -0.0506, -0.0111],\n",
      "        ...,\n",
      "        [-0.0296, -0.0856,  0.0766,  ..., -0.0752, -0.0529, -0.0353],\n",
      "        [ 0.0216, -0.0846,  0.0675,  ..., -0.0125, -0.0354,  0.0084],\n",
      "        [ 0.0320, -0.0755, -0.0862,  ...,  0.0627,  0.0279,  0.0857]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-8.5681e-02,  4.7403e-02, -3.4771e-02, -8.4490e-02,  2.1682e-02,\n",
      "         5.8174e-03,  6.6037e-02,  5.7850e-02, -4.1201e-02,  4.8048e-02,\n",
      "         5.2826e-02,  5.3506e-04, -6.1842e-02, -3.7463e-02,  8.2067e-02,\n",
      "         7.6211e-02, -1.7865e-02, -8.7156e-02,  7.7372e-02,  2.3359e-02,\n",
      "         7.3274e-02,  1.3701e-02, -8.6020e-02,  5.9234e-03,  4.1286e-02,\n",
      "        -4.1748e-02,  1.6856e-02,  4.1784e-03, -6.6380e-02,  2.0931e-02,\n",
      "         6.6249e-02, -1.8994e-02,  6.7284e-02,  4.6681e-03, -1.7040e-02,\n",
      "        -8.1157e-02,  5.0485e-02,  5.1170e-02,  5.1213e-02, -2.1009e-02,\n",
      "        -1.9551e-02, -4.4934e-02,  7.3216e-02,  6.3444e-02,  7.0493e-02,\n",
      "        -6.8809e-02, -8.1290e-02,  3.1404e-02,  6.9634e-02,  6.5708e-02,\n",
      "         3.0346e-02,  4.1910e-02, -4.4018e-02, -9.6580e-03, -1.1062e-02,\n",
      "         4.6578e-03, -8.1317e-02,  5.6681e-02, -7.0761e-02, -4.5977e-02,\n",
      "         3.2738e-02, -6.6522e-02,  3.0646e-02,  6.8022e-02, -6.7301e-02,\n",
      "        -8.2464e-02, -2.8146e-02, -6.4822e-02, -4.5517e-02,  4.2299e-02,\n",
      "         7.5826e-02, -8.4032e-02, -7.2999e-02, -7.7859e-02,  5.3690e-02,\n",
      "        -3.5987e-02, -1.1739e-02,  8.1146e-02,  3.1609e-02,  6.8434e-02,\n",
      "         4.1613e-03, -2.5969e-02, -3.0199e-03,  2.6103e-03,  5.8094e-02,\n",
      "         8.8863e-05,  8.5329e-02, -4.2334e-02,  1.1086e-02,  5.0525e-02,\n",
      "         4.8552e-02, -3.7024e-02, -1.4895e-02,  3.6781e-02, -1.2067e-02,\n",
      "        -8.5673e-02, -8.1668e-02,  7.1611e-02, -2.2209e-02,  2.4402e-02,\n",
      "         1.2020e-02, -8.0362e-02, -4.7513e-02, -5.7351e-02, -4.7329e-02,\n",
      "        -6.0321e-02, -1.5024e-02,  3.6717e-02, -1.1615e-02,  1.8287e-03,\n",
      "         7.1998e-02,  3.8852e-03,  4.4711e-02, -7.8407e-02,  7.0925e-03,\n",
      "         3.2646e-02,  3.0655e-02,  8.0930e-02, -6.8417e-02, -7.8838e-02,\n",
      "         2.9265e-02, -8.4786e-02, -1.2008e-02, -2.4297e-02,  6.2244e-02,\n",
      "        -7.3218e-02,  7.0598e-02, -5.0509e-02], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0690, -0.0861,  0.0832,  ..., -0.0519,  0.0469,  0.0488],\n",
      "        [-0.0353,  0.0515, -0.0116,  ...,  0.0812, -0.0879, -0.0761],\n",
      "        [-0.0865, -0.0734, -0.0453,  ...,  0.0643, -0.0487, -0.0583],\n",
      "        ...,\n",
      "        [ 0.0489,  0.0759, -0.0332,  ..., -0.0031, -0.0419, -0.0033],\n",
      "        [ 0.0217,  0.0278,  0.0581,  ..., -0.0716,  0.0395,  0.0426],\n",
      "        [-0.0304, -0.0357, -0.0871,  ..., -0.0720,  0.0596,  0.0545]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-3.1662e-02, -7.2560e-03,  7.6613e-02, -3.5433e-03,  1.7854e-02,\n",
      "         2.7292e-02, -4.0965e-02, -7.8703e-03, -3.8396e-02, -1.7166e-02,\n",
      "        -1.4683e-02,  3.3180e-02, -1.7170e-02, -4.2312e-02,  4.5462e-02,\n",
      "        -1.6012e-02,  8.5025e-02,  1.5737e-02, -5.3873e-02,  5.8771e-02,\n",
      "         3.6876e-02, -5.5706e-03,  8.4873e-02,  3.1775e-02, -6.1998e-02,\n",
      "        -5.2051e-02, -6.4911e-02, -3.8657e-02,  5.7899e-02, -1.3251e-02,\n",
      "        -7.0156e-02, -6.9193e-02,  1.4745e-02, -4.6510e-02,  1.3621e-02,\n",
      "        -7.6682e-02, -1.7369e-02,  2.0919e-02,  2.6468e-02, -1.4330e-02,\n",
      "        -5.9358e-02, -2.0270e-02, -7.7497e-02, -3.7757e-02, -8.5861e-02,\n",
      "         4.0078e-02, -5.3797e-03,  1.1764e-02,  7.6971e-02, -8.0661e-02,\n",
      "         1.8247e-02, -7.1385e-02, -4.4707e-02,  6.4364e-03,  7.6409e-02,\n",
      "        -5.0976e-02,  1.9135e-02, -4.1120e-02,  5.9087e-03, -2.4463e-02,\n",
      "         3.8465e-03, -8.5609e-02,  6.4780e-02, -1.1228e-02,  9.2079e-03,\n",
      "         4.1759e-02,  1.4123e-02,  8.3190e-02,  4.8589e-02, -8.2337e-02,\n",
      "        -2.1211e-02, -6.5684e-02, -5.4886e-03, -5.0153e-02, -4.9505e-02,\n",
      "         2.8036e-02,  8.6097e-02,  8.1906e-02, -8.7065e-02,  2.7641e-02,\n",
      "         4.9134e-02, -7.9477e-03,  5.9246e-02, -3.4439e-02,  5.7491e-02,\n",
      "        -8.7676e-02,  2.1375e-02, -7.6146e-02, -1.9236e-02,  5.6291e-02,\n",
      "        -2.5524e-02, -5.7499e-02,  8.6317e-02, -4.7260e-02,  6.4271e-02,\n",
      "        -2.0561e-02, -2.7468e-02, -7.5667e-02, -7.6936e-02,  2.1594e-02,\n",
      "        -4.8147e-02,  4.0065e-02, -2.2991e-02, -2.7027e-02,  2.5514e-02,\n",
      "         4.1944e-03,  4.7923e-02, -1.7861e-03, -1.0789e-02, -4.5964e-03,\n",
      "        -3.7911e-02, -6.9660e-02, -7.0737e-02,  3.9687e-02, -8.2767e-02,\n",
      "        -2.0478e-02,  2.6750e-02,  6.1486e-02,  4.3855e-03,  6.4442e-03,\n",
      "         7.4916e-05, -7.2522e-02,  6.8150e-02,  5.7143e-02,  7.2002e-02,\n",
      "         2.3916e-02,  6.2998e-02, -2.4226e-02], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0522, -0.0573, -0.0188,  ...,  0.0531, -0.0064, -0.0290],\n",
      "        [-0.0494, -0.0669,  0.0733,  ...,  0.0202,  0.0400, -0.0410],\n",
      "        [ 0.0228, -0.0040,  0.0589,  ...,  0.0252,  0.0362,  0.0761],\n",
      "        ...,\n",
      "        [ 0.0305, -0.0615,  0.0064,  ...,  0.0309, -0.0542,  0.0270],\n",
      "        [ 0.0861, -0.0649, -0.0817,  ...,  0.0706,  0.0320, -0.0168],\n",
      "        [ 0.0185, -0.0501, -0.0839,  ..., -0.0217, -0.0731, -0.0239]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0287,  0.0195,  0.0099, -0.0734, -0.0807, -0.0797,  0.0556,  0.0129,\n",
      "        -0.0159,  0.0379,  0.0755,  0.0802, -0.0119, -0.0182, -0.0584,  0.0515,\n",
      "         0.0170,  0.0117, -0.0672,  0.0249, -0.0679,  0.0562,  0.0432,  0.0315,\n",
      "         0.0717, -0.0168,  0.0473, -0.0314,  0.0533, -0.0251, -0.0207,  0.0833,\n",
      "         0.0656,  0.0572, -0.0777,  0.0187,  0.0818,  0.0798,  0.0026, -0.0808,\n",
      "         0.0717,  0.0364, -0.0853, -0.0879,  0.0831,  0.0235, -0.0212, -0.0019,\n",
      "         0.0715,  0.0310,  0.0153,  0.0378, -0.0297, -0.0456,  0.0071,  0.0656,\n",
      "         0.0240,  0.0416,  0.0776,  0.0064,  0.0810,  0.0338,  0.0450, -0.0596,\n",
      "        -0.0383,  0.0699, -0.0135,  0.0508, -0.0422,  0.0273, -0.0831,  0.0401,\n",
      "        -0.0019, -0.0295,  0.0609, -0.0185, -0.0279,  0.0337, -0.0026,  0.0728,\n",
      "        -0.0793,  0.0281,  0.0765,  0.0767,  0.0421,  0.0305, -0.0775,  0.0699,\n",
      "        -0.0074, -0.0110, -0.0133,  0.0579, -0.0033,  0.0018, -0.0872, -0.0259,\n",
      "         0.0142, -0.0294, -0.0668,  0.0373, -0.0731,  0.0515, -0.0138, -0.0116,\n",
      "         0.0670, -0.0642,  0.0342,  0.0589,  0.0881, -0.0166, -0.0454, -0.0067,\n",
      "         0.0663, -0.0328, -0.0495,  0.0553,  0.0794,  0.0786, -0.0747,  0.0174,\n",
      "         0.0637, -0.0001, -0.0166,  0.0791, -0.0826, -0.0489,  0.0649,  0.0523],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 3.7929e-02, -4.1282e-02, -4.7657e-02,  ...,  2.7247e-02,\n",
      "          8.1579e-02, -3.5235e-02],\n",
      "        [ 4.3739e-02, -4.1741e-02, -2.5798e-02,  ..., -2.5757e-03,\n",
      "         -8.5073e-02, -5.3987e-05],\n",
      "        [-3.5074e-02,  2.5710e-02,  4.7891e-02,  ...,  7.6432e-02,\n",
      "         -8.5259e-02,  6.1917e-02],\n",
      "        ...,\n",
      "        [ 6.5793e-03, -2.8725e-02,  2.3327e-02,  ...,  5.4120e-02,\n",
      "          3.8918e-02,  3.5816e-02],\n",
      "        [-5.5011e-02, -1.5010e-02,  5.8440e-02,  ...,  5.2257e-02,\n",
      "         -4.1600e-02, -2.6033e-02],\n",
      "        [-4.4792e-02,  7.2610e-03,  4.9765e-02,  ...,  5.4880e-02,\n",
      "         -1.9568e-02,  5.4265e-02]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 1.4742e-02, -6.1042e-03,  8.3362e-02, -4.8686e-02, -6.8460e-02,\n",
      "        -1.8009e-02,  5.6593e-02, -2.5295e-03, -1.7148e-02, -2.4397e-02,\n",
      "         1.9347e-02, -5.6961e-02,  2.7406e-02, -6.5894e-02,  6.9785e-02,\n",
      "         6.4148e-02, -7.8617e-02,  8.4163e-02, -4.0120e-03, -1.6369e-03,\n",
      "        -5.8232e-02,  3.2878e-02,  1.3202e-05, -1.2249e-02,  6.8236e-02,\n",
      "        -6.9340e-02, -5.9240e-02, -7.1127e-02, -7.6003e-02,  6.2869e-02,\n",
      "        -5.8226e-02,  5.2993e-02, -3.8613e-02,  7.5005e-02,  6.6139e-02,\n",
      "         6.3873e-02, -3.2752e-02, -1.6395e-02,  7.5281e-02,  2.5504e-02,\n",
      "         1.7089e-02, -7.8159e-02, -8.5502e-02, -5.2352e-02,  3.2337e-02,\n",
      "        -5.5797e-02, -6.6935e-02,  2.6020e-02,  4.8591e-02,  6.9486e-02,\n",
      "         1.7848e-02,  6.0809e-02,  2.0251e-02, -1.2815e-03, -5.5466e-02,\n",
      "        -4.6052e-02, -7.9841e-02, -6.7296e-02,  3.6747e-02,  3.1260e-02,\n",
      "        -7.8100e-02,  6.3521e-02, -1.0330e-02,  5.6342e-02,  1.5462e-02,\n",
      "         5.8436e-02, -6.6565e-02,  7.7197e-02,  8.2056e-02, -5.6403e-02,\n",
      "        -5.3909e-02, -6.7841e-02,  6.9189e-02,  4.7473e-02,  7.3118e-02,\n",
      "        -6.3879e-02,  4.9560e-03,  2.5682e-02, -7.2605e-03, -8.2418e-02,\n",
      "        -1.8058e-02, -8.7787e-02, -5.6910e-02,  4.5705e-02, -4.1047e-02,\n",
      "        -7.0167e-02,  2.4774e-02,  9.5391e-03,  6.1251e-02,  3.2600e-02,\n",
      "        -9.7551e-03,  1.5966e-02, -1.6114e-02,  3.4607e-02,  4.9184e-02,\n",
      "        -5.5171e-02, -2.9473e-02, -6.7803e-02,  3.7913e-02,  4.1189e-02,\n",
      "         2.9397e-02, -5.8189e-02,  1.9278e-02,  2.6331e-03, -1.8262e-02,\n",
      "        -5.7067e-03,  8.1882e-02,  9.6890e-03,  8.0740e-02, -6.6885e-02,\n",
      "         2.6899e-02,  7.1967e-02,  3.6862e-04,  5.4399e-02, -8.0151e-02,\n",
      "         5.4193e-02,  1.6335e-02,  1.7130e-02,  6.5458e-02,  3.2493e-02,\n",
      "        -5.3097e-02, -2.5094e-02,  2.3889e-02, -4.9912e-02, -7.3373e-02,\n",
      "         4.5040e-02,  8.3158e-03,  2.2821e-02], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0679,  0.0394,  0.0702,  ...,  0.0622, -0.0344, -0.0561],\n",
      "        [-0.0446,  0.0466, -0.0039,  ..., -0.0774,  0.0722,  0.0813],\n",
      "        [ 0.0619,  0.0821, -0.0367,  ..., -0.0692, -0.0073, -0.0454],\n",
      "        ...,\n",
      "        [-0.0123,  0.0031,  0.0311,  ..., -0.0737, -0.0181, -0.0834],\n",
      "        [-0.0642, -0.0487,  0.0222,  ..., -0.0030,  0.0607, -0.0222],\n",
      "        [ 0.0687, -0.0709, -0.0566,  ..., -0.0497, -0.0352,  0.0668]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0498,  0.0526, -0.0017,  0.0180, -0.0696,  0.0289, -0.0647, -0.0394,\n",
      "         0.0413,  0.0493,  0.0189, -0.0013, -0.0864,  0.0598, -0.0093, -0.0141,\n",
      "         0.0104, -0.0490, -0.0482,  0.0054,  0.0333, -0.0683, -0.0449, -0.0512,\n",
      "         0.0085, -0.0078,  0.0870,  0.0068,  0.0057, -0.0829, -0.0358, -0.0768,\n",
      "         0.0244,  0.0543,  0.0809,  0.0189,  0.0590, -0.0392, -0.0515, -0.0415,\n",
      "        -0.0160, -0.0498,  0.0835, -0.0545, -0.0163,  0.0181,  0.0511, -0.0616,\n",
      "        -0.0506,  0.0181, -0.0342,  0.0479, -0.0484, -0.0035,  0.0838,  0.0020,\n",
      "        -0.0827, -0.0137, -0.0790, -0.0320, -0.0340,  0.0217, -0.0462,  0.0802,\n",
      "         0.0569, -0.0535, -0.0379,  0.0709,  0.0594,  0.0733,  0.0436, -0.0220,\n",
      "        -0.0016,  0.0407,  0.0782, -0.0582,  0.0752,  0.0321,  0.0702,  0.0694,\n",
      "        -0.0385,  0.0840, -0.0622,  0.0820,  0.0280, -0.0704,  0.0118, -0.0107,\n",
      "         0.0105,  0.0044,  0.0071,  0.0568, -0.0559,  0.0789,  0.0194, -0.0283,\n",
      "        -0.0762, -0.0879,  0.0531,  0.0460,  0.0475,  0.0177,  0.0100,  0.0642,\n",
      "        -0.0864,  0.0171, -0.0266, -0.0246,  0.0351,  0.0018,  0.0279,  0.0542,\n",
      "         0.0601,  0.0019, -0.0338,  0.0167, -0.0239,  0.0256,  0.0379,  0.0585,\n",
      "         0.0263,  0.0570, -0.0144, -0.0615,  0.0339,  0.0381,  0.0602,  0.0801],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-7.1828e-02,  3.1419e-02, -4.3711e-02,  ...,  3.5771e-02,\n",
      "         -1.9876e-02,  4.6847e-02],\n",
      "        [-4.8617e-02,  7.5726e-02,  4.7586e-02,  ..., -7.6204e-02,\n",
      "          7.4674e-02, -4.2653e-02],\n",
      "        [ 4.9676e-02,  7.9855e-05, -1.8221e-02,  ..., -7.5656e-02,\n",
      "         -7.2000e-02, -1.7004e-02],\n",
      "        ...,\n",
      "        [ 4.1370e-02,  7.3799e-02,  2.2579e-02,  ...,  3.2229e-02,\n",
      "          6.6745e-02,  7.9330e-02],\n",
      "        [-4.4839e-02, -5.8942e-02,  3.7648e-02,  ...,  7.1981e-02,\n",
      "          7.2632e-02, -6.5700e-02],\n",
      "        [-1.7561e-02,  2.2968e-02,  1.5996e-02,  ..., -7.1877e-04,\n",
      "          6.5087e-02,  6.1985e-02]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0738, -0.0525, -0.0026, -0.0323, -0.0049,  0.0687, -0.0063,  0.0784,\n",
      "         0.0500,  0.0563,  0.0731, -0.0045, -0.0092, -0.0036, -0.0496,  0.0430,\n",
      "         0.0081,  0.0141,  0.0358,  0.0863, -0.0216,  0.0230, -0.0116, -0.0829,\n",
      "         0.0336, -0.0363, -0.0432, -0.0688, -0.0755,  0.0037, -0.0329, -0.0682,\n",
      "        -0.0606, -0.0805, -0.0653, -0.0073,  0.0723,  0.0844,  0.0338,  0.0375,\n",
      "        -0.0161, -0.0597,  0.0648, -0.0351, -0.0871, -0.0062,  0.0464, -0.0119,\n",
      "         0.0715,  0.0250,  0.0721,  0.0815,  0.0862, -0.0293, -0.0125, -0.0574,\n",
      "        -0.0380,  0.0800,  0.0088,  0.0394, -0.0088, -0.0146, -0.0834,  0.0121,\n",
      "         0.0472,  0.0341, -0.0179, -0.0108, -0.0656, -0.0742,  0.0634, -0.0422,\n",
      "         0.0313,  0.0192, -0.0216, -0.0693,  0.0029, -0.0414,  0.0448, -0.0010,\n",
      "        -0.0647, -0.0425,  0.0660,  0.0178,  0.0405, -0.0099,  0.0370,  0.0038,\n",
      "        -0.0230,  0.0155, -0.0085, -0.0594,  0.0167,  0.0273, -0.0304,  0.0581,\n",
      "        -0.0330,  0.0526,  0.0513, -0.0015, -0.0377, -0.0779, -0.0339,  0.0318,\n",
      "         0.0430, -0.0627, -0.0753,  0.0300, -0.0489, -0.0295, -0.0005, -0.0382,\n",
      "         0.0437,  0.0490,  0.0366,  0.0766, -0.0061, -0.0342, -0.0365, -0.0134,\n",
      "         0.0677,  0.0425,  0.0040,  0.0377, -0.0697, -0.0668, -0.0015, -0.0342],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0094,  0.0607,  0.0080,  ..., -0.0037, -0.0039,  0.0025],\n",
      "        [ 0.0545, -0.0497,  0.0295,  ..., -0.0278,  0.0049, -0.0441],\n",
      "        [-0.0797, -0.0774,  0.0415,  ...,  0.0827, -0.0386,  0.0356],\n",
      "        ...,\n",
      "        [-0.0072,  0.0076, -0.0324,  ..., -0.0464,  0.0454, -0.0388],\n",
      "        [-0.0320,  0.0333, -0.0446,  ..., -0.0446, -0.0088, -0.0426],\n",
      "        [-0.0285,  0.0845,  0.0389,  ..., -0.0794, -0.0488,  0.0422]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0595, -0.0828, -0.0785, -0.0194,  0.0320,  0.0605, -0.0641, -0.0584,\n",
      "         0.0643,  0.0451, -0.0195,  0.0877, -0.0462,  0.0026, -0.0546,  0.0262,\n",
      "         0.0334,  0.0498,  0.0731,  0.0089, -0.0821, -0.0154,  0.0222,  0.0475,\n",
      "         0.0296,  0.0303, -0.0810, -0.0578,  0.0853, -0.0576,  0.0104, -0.0485,\n",
      "        -0.0146, -0.0518,  0.0089,  0.0076,  0.0474, -0.0053, -0.0505,  0.0187,\n",
      "         0.0826,  0.0175, -0.0330, -0.0047,  0.0004, -0.0641, -0.0414,  0.0408,\n",
      "         0.0713, -0.0063,  0.0759,  0.0782, -0.0730,  0.0529, -0.0172,  0.0245,\n",
      "         0.0672,  0.0839, -0.0450,  0.0181,  0.0780,  0.0434, -0.0525,  0.0695,\n",
      "        -0.0589, -0.0147, -0.0776, -0.0740, -0.0225, -0.0529,  0.0115, -0.0073,\n",
      "        -0.0349,  0.0840, -0.0834,  0.0520, -0.0467,  0.0168,  0.0605, -0.0299,\n",
      "         0.0815,  0.0735,  0.0848, -0.0070, -0.0039, -0.0039,  0.0807, -0.0227,\n",
      "        -0.0585, -0.0490, -0.0553, -0.0868, -0.0853, -0.0724, -0.0830,  0.0151,\n",
      "         0.0050, -0.0694, -0.0306, -0.0282,  0.0731, -0.0479,  0.0252,  0.0433,\n",
      "        -0.0724, -0.0351, -0.0409,  0.0807,  0.0126,  0.0352,  0.0425,  0.0512,\n",
      "        -0.0139,  0.0049, -0.0774, -0.0145, -0.0157,  0.0630, -0.0376,  0.0065,\n",
      "        -0.0383,  0.0837,  0.0026,  0.0458, -0.0214,  0.0466, -0.0701,  0.0548],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0178,  0.0021, -0.0348,  ..., -0.0690,  0.0125, -0.0469],\n",
      "        [ 0.0163, -0.0229,  0.0571,  ...,  0.0255, -0.0790,  0.0589],\n",
      "        [-0.0184,  0.0525, -0.0677,  ...,  0.0382, -0.0752,  0.0669],\n",
      "        ...,\n",
      "        [ 0.0071,  0.0612, -0.0372,  ...,  0.0613,  0.0190,  0.0411],\n",
      "        [ 0.0332, -0.0636, -0.0219,  ..., -0.0579, -0.0824,  0.0142],\n",
      "        [ 0.0536, -0.0206, -0.0358,  ..., -0.0653, -0.0021, -0.0118]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0303,  0.0087, -0.0849,  0.0230, -0.0560, -0.0735,  0.0312,  0.0281,\n",
      "         0.0800, -0.0445,  0.0059,  0.0829, -0.0609, -0.0309, -0.0256, -0.0168,\n",
      "        -0.0209,  0.0392,  0.0620,  0.0643, -0.0778,  0.0324, -0.0792, -0.0883,\n",
      "         0.0832, -0.0756, -0.0059,  0.0377, -0.0831, -0.0015, -0.0312, -0.0463,\n",
      "        -0.0753,  0.0548,  0.0426, -0.0496,  0.0244, -0.0831, -0.0598,  0.0428,\n",
      "        -0.0877,  0.0650,  0.0311,  0.0037, -0.0250, -0.0792, -0.0237,  0.0386,\n",
      "        -0.0226,  0.0116, -0.0659, -0.0553,  0.0655, -0.0529, -0.0876, -0.0711,\n",
      "        -0.0805,  0.0560, -0.0549,  0.0468, -0.0818, -0.0612,  0.0323,  0.0720,\n",
      "         0.0146,  0.0514, -0.0704, -0.0776, -0.0520,  0.0560,  0.0315,  0.0867,\n",
      "        -0.0875,  0.0811, -0.0688, -0.0214,  0.0317, -0.0467, -0.0484,  0.0198,\n",
      "        -0.0718, -0.0059,  0.0840, -0.0459, -0.0207,  0.0122,  0.0871,  0.0165,\n",
      "         0.0754,  0.0327, -0.0319,  0.0231,  0.0318, -0.0650,  0.0783, -0.0866,\n",
      "         0.0527, -0.0723,  0.0785, -0.0238,  0.0726,  0.0358, -0.0514,  0.0116,\n",
      "        -0.0697,  0.0399, -0.0576, -0.0778,  0.0660, -0.0783,  0.0854, -0.0790,\n",
      "        -0.0574,  0.0803,  0.0808, -0.0262,  0.0650,  0.0732, -0.0634,  0.0175,\n",
      "         0.0037, -0.0361,  0.0761,  0.0355,  0.0824,  0.0315, -0.0209, -0.0688],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0638, -0.0773, -0.0524,  ...,  0.0530, -0.0013, -0.0559],\n",
      "        [-0.0736,  0.0033, -0.0030,  ...,  0.0587,  0.0322, -0.0094],\n",
      "        [-0.0677,  0.0670, -0.0515,  ...,  0.0226, -0.0845,  0.0501],\n",
      "        ...,\n",
      "        [ 0.0064,  0.0846, -0.0037,  ...,  0.0279,  0.0821, -0.0291],\n",
      "        [ 0.0337, -0.0326,  0.0494,  ...,  0.0406,  0.0878, -0.0357],\n",
      "        [ 0.0323,  0.0343, -0.0578,  ..., -0.0006,  0.0801,  0.0012]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0018,  0.0313, -0.0763,  0.0675,  0.0762, -0.0016, -0.0181,  0.0427,\n",
      "         0.0202, -0.0299,  0.0050,  0.0532,  0.0781,  0.0707,  0.0557,  0.0663,\n",
      "         0.0476,  0.0711, -0.0459, -0.0807,  0.0209,  0.0851,  0.0166, -0.0743,\n",
      "         0.0250,  0.0526,  0.0382, -0.0645, -0.0847, -0.0652,  0.0776, -0.0664,\n",
      "        -0.0817, -0.0721,  0.0841, -0.0270, -0.0043,  0.0233, -0.0803,  0.0191,\n",
      "        -0.0721,  0.0856, -0.0754,  0.0296, -0.0415, -0.0013, -0.0614,  0.0434,\n",
      "        -0.0600,  0.0456, -0.0025,  0.0852, -0.0344, -0.0230, -0.0807, -0.0451,\n",
      "         0.0763, -0.0530,  0.0041,  0.0409,  0.0657, -0.0749, -0.0811, -0.0615,\n",
      "        -0.0304, -0.0332,  0.0669,  0.0048,  0.0649, -0.0870, -0.0602, -0.0576,\n",
      "        -0.0166,  0.0447,  0.0124, -0.0870, -0.0266, -0.0306, -0.0121, -0.0248,\n",
      "         0.0570,  0.0157,  0.0816, -0.0607,  0.0663, -0.0208,  0.0750,  0.0147,\n",
      "         0.0068,  0.0662, -0.0687, -0.0428,  0.0767,  0.0808, -0.0315,  0.0193,\n",
      "         0.0099, -0.0257, -0.0303,  0.0609, -0.0075,  0.0860,  0.0370, -0.0546,\n",
      "         0.0631,  0.0787, -0.0130, -0.0703, -0.0352, -0.0757, -0.0641,  0.0105,\n",
      "        -0.0638,  0.0117,  0.0094, -0.0579,  0.0864,  0.0533,  0.0851, -0.0838,\n",
      "        -0.0671, -0.0377, -0.0046, -0.0453, -0.0320, -0.0865, -0.0504, -0.0393],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-0.0300,  0.0724, -0.0269,  ...,  0.0136,  0.0417, -0.0008],\n",
      "        [ 0.0418,  0.0074,  0.0295,  ..., -0.0801,  0.0558, -0.0695],\n",
      "        [-0.0755, -0.0246,  0.0588,  ...,  0.0069,  0.0497,  0.0554],\n",
      "        ...,\n",
      "        [ 0.0352,  0.0036, -0.0696,  ...,  0.0369,  0.0534, -0.0307],\n",
      "        [ 0.0207, -0.0855, -0.0822,  ..., -0.0050, -0.0331,  0.0286],\n",
      "        [-0.0866, -0.0074, -0.0606,  ...,  0.0583, -0.0373,  0.0191]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0369, -0.0461, -0.0550, -0.0029,  0.0299, -0.0153,  0.0651,  0.0516,\n",
      "         0.0332, -0.0842, -0.0397, -0.0545,  0.0543, -0.0097,  0.0241, -0.0387,\n",
      "         0.0047,  0.0520,  0.0552,  0.0319, -0.0198,  0.0332,  0.0043,  0.0252,\n",
      "         0.0173,  0.0555,  0.0212, -0.0243, -0.0782,  0.0344, -0.0731,  0.0400,\n",
      "        -0.0566,  0.0208,  0.0308,  0.0675, -0.0835,  0.0752,  0.0311, -0.0124,\n",
      "         0.0349,  0.0469,  0.0860, -0.0870, -0.0681,  0.0245,  0.0447, -0.0472,\n",
      "         0.0130,  0.0737,  0.0555, -0.0231, -0.0652,  0.0646, -0.0360, -0.0070,\n",
      "         0.0757,  0.0194,  0.0186, -0.0114,  0.0139, -0.0225,  0.0646, -0.0405,\n",
      "        -0.0497,  0.0567,  0.0236,  0.0459, -0.0606, -0.0046, -0.0069, -0.0662,\n",
      "         0.0083, -0.0670,  0.0503,  0.0339, -0.0285,  0.0746,  0.0116,  0.0569,\n",
      "        -0.0077,  0.0805,  0.0533,  0.0102, -0.0076, -0.0758, -0.0056,  0.0522,\n",
      "        -0.0386,  0.0292,  0.0152,  0.0567,  0.0232,  0.0400,  0.0301,  0.0279,\n",
      "        -0.0081, -0.0694,  0.0569, -0.0515,  0.0103, -0.0602, -0.0545,  0.0422,\n",
      "        -0.0810, -0.0468,  0.0148, -0.0266,  0.0679, -0.0473, -0.0156,  0.0206,\n",
      "         0.0389, -0.0423, -0.0668, -0.0125, -0.0483, -0.0443, -0.0457,  0.0078,\n",
      "         0.0255,  0.0824, -0.0204,  0.0124,  0.0233, -0.0542, -0.0851,  0.0398],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0871,  0.0854,  0.0634,  ..., -0.0254,  0.0240, -0.0073],\n",
      "        [-0.0653, -0.0127, -0.0730,  ...,  0.0805, -0.0095, -0.0344],\n",
      "        [ 0.0769,  0.0186,  0.0288,  ...,  0.0118, -0.0578,  0.0230],\n",
      "        ...,\n",
      "        [ 0.0518, -0.0078, -0.0220,  ...,  0.0554, -0.0452, -0.0291],\n",
      "        [-0.0438,  0.0328,  0.0424,  ...,  0.0443,  0.0282, -0.0253],\n",
      "        [-0.0430, -0.0046, -0.0660,  ...,  0.0234, -0.0874,  0.0135]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0334, -0.0406, -0.0178,  0.0363,  0.0535,  0.0145, -0.0667, -0.0351,\n",
      "         0.0863, -0.0375,  0.0821, -0.0098,  0.0184,  0.0731,  0.0744,  0.0618,\n",
      "         0.0022, -0.0652,  0.0828,  0.0518, -0.0622, -0.0376, -0.0474, -0.0480,\n",
      "         0.0643, -0.0456, -0.0755,  0.0737, -0.0568,  0.0398,  0.0181, -0.0261,\n",
      "        -0.0111, -0.0229,  0.0813, -0.0267, -0.0631,  0.0764, -0.0150, -0.0264,\n",
      "        -0.0181, -0.0500,  0.0279, -0.0649, -0.0508,  0.0499,  0.0271,  0.0269,\n",
      "         0.0693,  0.0716, -0.0196, -0.0862, -0.0485, -0.0044,  0.0124,  0.0376,\n",
      "         0.0238, -0.0568,  0.0134,  0.0363, -0.0429, -0.0095,  0.0626, -0.0830,\n",
      "        -0.0473, -0.0632, -0.0435,  0.0337, -0.0317,  0.0280,  0.0764, -0.0628,\n",
      "         0.0726, -0.0004, -0.0314,  0.0662,  0.0855,  0.0753, -0.0370,  0.0104,\n",
      "        -0.0058, -0.0359,  0.0230, -0.0414,  0.0190,  0.0028, -0.0086,  0.0746,\n",
      "         0.0258,  0.0194,  0.0883,  0.0083, -0.0742,  0.0880,  0.0690,  0.0364,\n",
      "        -0.0149, -0.0751,  0.0369, -0.0852,  0.0460,  0.0760,  0.0837, -0.0349,\n",
      "        -0.0077,  0.0641, -0.0475,  0.0293, -0.0335,  0.0473,  0.0210, -0.0800,\n",
      "        -0.0103,  0.0660, -0.0276,  0.0223,  0.0771, -0.0234, -0.0300,  0.0207,\n",
      "         0.0750, -0.0506,  0.0167,  0.0183,  0.0647, -0.0663, -0.0479,  0.0297],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0136, -0.0853,  0.0529,  ...,  0.0068,  0.0522, -0.0098],\n",
      "        [ 0.0143,  0.0657,  0.0559,  ..., -0.0460,  0.0212, -0.0125],\n",
      "        [-0.0446,  0.0480,  0.0183,  ...,  0.0363, -0.0039,  0.0667],\n",
      "        ...,\n",
      "        [-0.0169,  0.0282,  0.0830,  ..., -0.0729, -0.0465,  0.0312],\n",
      "        [-0.0560,  0.0413,  0.0589,  ..., -0.0776, -0.0392,  0.0880],\n",
      "        [ 0.0783,  0.0019,  0.0752,  ..., -0.0186,  0.0724, -0.0193]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.0297,  0.0438,  0.0165, -0.0557, -0.0643, -0.0500, -0.0284, -0.0292,\n",
      "         0.0288, -0.0301, -0.0063,  0.0169,  0.0244, -0.0255,  0.0361,  0.0785,\n",
      "        -0.0049, -0.0385,  0.0766, -0.0229,  0.0802, -0.0378,  0.0584,  0.0032,\n",
      "         0.0711,  0.0247, -0.0361,  0.0389, -0.0389,  0.0791,  0.0699, -0.0620,\n",
      "         0.0275,  0.0154,  0.0323,  0.0240,  0.0266,  0.0014,  0.0181,  0.0074,\n",
      "        -0.0074,  0.0412, -0.0104, -0.0570,  0.0782,  0.0291,  0.0475, -0.0035,\n",
      "        -0.0748, -0.0360,  0.0813, -0.0602,  0.0094, -0.0668, -0.0689, -0.0520,\n",
      "         0.0596, -0.0191,  0.0717, -0.0301,  0.0167,  0.0363,  0.0186,  0.0791,\n",
      "         0.0851,  0.0203,  0.0036,  0.0789,  0.0376,  0.0067,  0.0225, -0.0195,\n",
      "        -0.0673, -0.0500,  0.0512,  0.0790,  0.0822, -0.0720,  0.0459,  0.0326,\n",
      "         0.0711,  0.0644, -0.0792, -0.0479, -0.0515,  0.0360, -0.0264, -0.0712,\n",
      "        -0.0785, -0.0852, -0.0752,  0.0567,  0.0554,  0.0796,  0.0546,  0.0459,\n",
      "         0.0472,  0.0106, -0.0119,  0.0779,  0.0397, -0.0513, -0.0556, -0.0062,\n",
      "         0.0519, -0.0309, -0.0470,  0.0318,  0.0254,  0.0842,  0.0693, -0.0053,\n",
      "        -0.0760, -0.0767,  0.0134,  0.0848,  0.0738, -0.0747,  0.0432,  0.0236,\n",
      "        -0.0495, -0.0839, -0.0405, -0.0067,  0.0830, -0.0643,  0.0433,  0.0362],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 0.0093,  0.0879,  0.0034,  ..., -0.0496,  0.0458,  0.0685],\n",
      "        [-0.0789,  0.0690,  0.0559,  ..., -0.0528, -0.0539, -0.0735],\n",
      "        [ 0.0310, -0.0166, -0.0085,  ...,  0.0273, -0.0003,  0.0471],\n",
      "        ...,\n",
      "        [-0.0008,  0.0047, -0.0598,  ...,  0.0351,  0.0110,  0.0047],\n",
      "        [-0.0832,  0.0273, -0.0037,  ..., -0.0845,  0.0143,  0.0850],\n",
      "        [ 0.0335, -0.0812, -0.0319,  ..., -0.0063, -0.0192, -0.0189]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([ 0.0834, -0.0038,  0.0615,  0.0741, -0.0795,  0.0155, -0.0721, -0.0332,\n",
      "        -0.0185, -0.0807, -0.0652, -0.0076, -0.0455, -0.0050, -0.0064,  0.0733,\n",
      "        -0.0737, -0.0615, -0.0193,  0.0477, -0.0078,  0.0471,  0.0445, -0.0722,\n",
      "         0.0830,  0.0076,  0.0538, -0.0157, -0.0154, -0.0644, -0.0386,  0.0790,\n",
      "        -0.0095, -0.0198,  0.0345,  0.0019, -0.0052,  0.0812, -0.0674, -0.0837,\n",
      "        -0.0348, -0.0200, -0.0837, -0.0763,  0.0805, -0.0094, -0.0182,  0.0816,\n",
      "        -0.0004, -0.0173, -0.0616,  0.0334,  0.0499,  0.0413,  0.0532, -0.0835,\n",
      "        -0.0668, -0.0421,  0.0404,  0.0800,  0.0762,  0.0015, -0.0526,  0.0272,\n",
      "         0.0555,  0.0223,  0.0243, -0.0574, -0.0267,  0.0077, -0.0341,  0.0712,\n",
      "         0.0489, -0.0508, -0.0045,  0.0839, -0.0199,  0.0299,  0.0383,  0.0487,\n",
      "        -0.0296,  0.0340, -0.0519, -0.0295, -0.0597,  0.0692, -0.0238,  0.0069,\n",
      "        -0.0740,  0.0046, -0.0760,  0.0485,  0.0813, -0.0109, -0.0580, -0.0269,\n",
      "        -0.0498,  0.0029, -0.0542,  0.0293, -0.0279, -0.0881,  0.0651, -0.0186,\n",
      "        -0.0531,  0.0384, -0.0632,  0.0489,  0.0023, -0.0270,  0.0291, -0.0132,\n",
      "         0.0820, -0.0153,  0.0197, -0.0821, -0.0543,  0.0826,  0.0815,  0.0427,\n",
      "        -0.0285,  0.0697,  0.0460, -0.0291,  0.0777,  0.0082,  0.0620, -0.0127],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "print(\"Parameters are :\")\n",
    "for param in net.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep Learning Algorithm Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parameters\n",
    "NUM_INPUTS=100\n",
    "HIDDEN_SIZE=1024\n",
    "NUM_OUTPUTS=20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Logistic Regression\n",
    "lor = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS, 1),\n",
    "    nn.Sigmoid())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Linear Regression\n",
    "lir = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Softmax classifier\n",
    "smx = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS, NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MultiLayer Perceptron\n",
    "mlp = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS, HIDDEN_SIZE),\n",
    "    nn.Tanh(),\n",
    "    nn.Linear(HIDDEN_SIZE, HIDDEN_SIZE),\n",
    "    nn.Tanh(),\n",
    "    nn.Linear(HIDDEN_SIZE, NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Embedding with fully connected layer\n",
    "VOCAB_SIZE = 10000\n",
    "HIDDEN_SIZE=100\n",
    "# mapping a Vocabulary of size 10,000 to HIDDEN_SIZE projections\n",
    "emb_1 = nn.Linear(VOCAB_SIZE, HIDDEN_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input x tensor size:  torch.Size([10, 10000])\n",
      "Output y embedding size:  torch.Size([10, 100])\n"
     ]
    }
   ],
   "source": [
    "# forward example [10, 10000] tensor\n",
    "code = [1] + [0] * 9999\n",
    "# copy 10 times the same code [1 0 0 0 ... 0]\n",
    "x = torch.FloatTensor([code] * 10)\n",
    "print('Input x tensor size: ', x.size())\n",
    "y = emb_1(x)\n",
    "print('Output y embedding size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Embedding with Embedding layer\n",
    "VOCAB_SIZE = 10000\n",
    "HIDDEN_SIZE=100\n",
    "# mapping a Vocabulary of size 10.000 to HIDDEN_SIZE projections\n",
    "emb_2 = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input x tensor size:  torch.Size([10, 1])\n",
      "Output y embedding size:  torch.Size([10, 1, 100])\n"
     ]
    }
   ],
   "source": [
    "# Just make a long tensor with zero-index\n",
    "x = torch.zeros(10, 1).long()\n",
    "print('Input x tensor size: ', x.size())\n",
    "y = emb_2(x)\n",
    "print('Output y embedding size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Recurrent Neural Network\n",
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 512\n",
    "NUM_LAYERS = 1\n",
    "# define a recurrent layer\n",
    "rnn = nn.RNN(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size [seq_len, bsize, hidden_size]:  torch.Size([100, 1, 100])\n",
      "Output tensor h[t] size [seq_len, bsize, hidden_size]:  torch.Size([100, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "SEQ_LEN = 100\n",
    "x = torch.randn(SEQ_LEN, 1, NUM_INPUTS)\n",
    "print('Input tensor size [seq_len, bsize, hidden_size]: ', x.size())\n",
    "ht, state = rnn(x, None)\n",
    "print('Output tensor h[t] size [seq_len, bsize, hidden_size]: ', ht.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 512\n",
    "NUM_LAYERS = 1\n",
    "# define a recurrent layer, swapping batch and time axis\n",
    "rnn = nn.RNN(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS,\n",
    "            batch_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size [bsize, seq_len, hidden_size]:  torch.Size([1, 100, 100])\n",
      "Output tensor h[t] size [bsize, seq_len, hidden_size]:  torch.Size([1, 100, 512])\n"
     ]
    }
   ],
   "source": [
    "SEQ_LEN = 100\n",
    "x = torch.randn(1, SEQ_LEN, NUM_INPUTS)\n",
    "print('Input tensor size [bsize, seq_len, hidden_size]: ', x.size())\n",
    "ht, state = rnn(x, None)\n",
    "print('Output tensor h[t] size [bsize, seq_len, hidden_size]: ', ht.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ht size:  torch.Size([1, 100, 512])\n",
      "state size:  torch.Size([1, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "# let's check ht and state sizes\n",
    "print('ht size: ', ht.size())\n",
    "print('state size: ', state.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 100, 100])\n",
      "Hidden tensor size ht:  torch.Size([1, 100, 512])\n",
      "Output tensor y size:  torch.Size([1, 100, 10])\n"
     ]
    }
   ],
   "source": [
    "NUM_INPUTS = 100\n",
    "NUM_OUTPUTS = 10\n",
    "HIDDEN_SIZE = 512\n",
    "SEQ_LEN = 100\n",
    "NUM_LAYERS = 1\n",
    "# define a recurrent layer, swapping batch and time axis and connect\n",
    "# an FC layer as an output layer to build a full network\n",
    "rnn = nn.RNN(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS,\n",
    "            batch_first=True)\n",
    "fc = nn.Sequential(\n",
    "    nn.Linear(HIDDEN_SIZE, NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=2)\n",
    ")\n",
    "\n",
    "x = torch.randn(1, SEQ_LEN, NUM_INPUTS)\n",
    "print('Input tensor size x: ', x.size())\n",
    "ht, state = rnn(x, None)\n",
    "print('Hidden tensor size ht: ', ht.size())\n",
    "y = fc(ht)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 100, 100])\n",
      "Output tensor ht size:  torch.Size([1, 100, 512])\n",
      "Last state h[T]:  torch.Size([1, 1, 512])\n",
      "Cell state c[T]:  torch.Size([1, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "#LSTM Recurrent Neural Network\n",
    "lstm = nn.LSTM(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS,\n",
    "              batch_first=True)\n",
    "x = torch.randn(1, SEQ_LEN, NUM_INPUTS)\n",
    "print('Input tensor size x: ', x.size())\n",
    "ht, states = lstm(x, None)\n",
    "hT, cT = states[0], states[1]\n",
    "print('Output tensor ht size: ', ht.size())\n",
    "print('Last state h[T]: ', hT.size())\n",
    "print('Cell state c[T]: ', cT.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convolutional Neural Network\n",
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 1, 8])\n",
      "Output tensor y size:  torch.Size([1, 1024, 6])\n"
     ]
    }
   ],
   "source": [
    "SEQ_LEN = 8\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor size x: ', x.size())\n",
    "y = conv1d(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 1, 8])\n",
      "Output tensor y size:  torch.Size([1, 1024, 8])\n"
     ]
    }
   ],
   "source": [
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "PADDING = KERNEL_WIDTH // 2 # = 1\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH, \n",
    "                   padding=PADDING)\n",
    "\n",
    "SEQ_LEN = 8\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor size x: ', x.size())\n",
    "y = conv1d(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 8])\n",
      "Input tensor after padding xpad size:  torch.Size([1, 1, 10])\n",
      "Output tensor y size:  torch.Size([1, 1024, 8])\n"
     ]
    }
   ],
   "source": [
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH)\n",
    "                   \n",
    "SEQ_LEN = 8\n",
    "PADDING = KERNEL_WIDTH - 1 # = 2\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor x size: ', x.size())\n",
    "xpad = F.pad(x, (PADDING, 0))\n",
    "print('Input tensor after padding xpad size: ', xpad.size())\n",
    "y = conv1d(xpad)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 100, 1])\n",
      "Output tensor y size:  torch.Size([1, 20, 1])\n"
     ]
    }
   ],
   "source": [
    "#Convolutional Neural Network as an MLP\n",
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 1024\n",
    "NUM_OUTPUTS= 20\n",
    "# MLP as a CNN\n",
    "mlp = nn.Sequential(\n",
    "    nn.Conv1d(NUM_INPUTS, HIDDEN_SIZE, 1),\n",
    "    nn.Tanh(),\n",
    "    nn.Conv1d(HIDDEN_SIZE, HIDDEN_SIZE, 1),\n",
    "    nn.Tanh(),\n",
    "    nn.Conv1d(HIDDEN_SIZE, NUM_OUTPUTS, 1),\n",
    "    nn.LogSoftmax(dim=1)\n",
    ")\n",
    "\n",
    "x = torch.randn(1, 100, 1)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = mlp(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor y size:  torch.Size([1, 1, 2])\n",
      "Output (interpolated) tensor x size:  torch.Size([1, 1, 12])\n"
     ]
    }
   ],
   "source": [
    "#Deconvolutional Neural Network\n",
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1\n",
    "KERNEL_WIDTH = 8\n",
    "STRIDE = 4\n",
    "\n",
    "deconv = nn.ConvTranspose1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH,\n",
    "                            stride=STRIDE)\n",
    "\n",
    "SEQ_LEN = 2\n",
    "y = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor y size: ', y.size())\n",
    "x = deconv(y)\n",
    "print('Output (interpolated) tensor x size: ', x.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ht size:  torch.Size([1, 10, 100])\n",
      "state size:  torch.Size([1, 100])\n"
     ]
    }
   ],
   "source": [
    "#Quasi Recurrent Neural Network\n",
    "class fQRNNLayer(nn.Module):\n",
    "  \n",
    "  def __init__(self, num_inputs, num_outputs,\n",
    "              kwidth=2):\n",
    "    super().__init__()\n",
    "    self.num_inputs = num_inputs\n",
    "    self.num_outputs = num_outputs\n",
    "    self.kwidth = kwidth\n",
    "    # double feature maps for zt and ft predictions with same conv layer\n",
    "    self.conv = nn.Conv1d(num_inputs, num_outputs * 2, kwidth)\n",
    "    \n",
    "  def forward(self, x, state=None):\n",
    "    # x is [bsz, seq_len, num_inputs]\n",
    "    # state is [bsz, num_outputs] dimensional\n",
    "    # ---------- FEED FORWARD PART\n",
    "    # inference convolutional part\n",
    "    # transpose x axis first to work with CNN layer\n",
    "    x = x.transpose(1, 2)\n",
    "    pad = self.kwidth - 1\n",
    "    xp = F.pad(x, (pad, 0))\n",
    "    conv_h = self.conv(xp)\n",
    "    # split convolutional layer feature maps into zt (new state\n",
    "    # candidate) and forget activation ft\n",
    "    zt, ft = torch.chunk(conv_h, 2, dim=1)\n",
    "    # Convert forget gate into actual forget\n",
    "    ft = torch.sigmoid(ft)\n",
    "    # Convert zt into actual non-linear response\n",
    "    zt = torch.tanh(zt)\n",
    "    # ---------- SEQUENTIAL PART\n",
    "    # iterate through time now to make pooling\n",
    "    seqlen = ft.size(2)\n",
    "    if state is None:\n",
    "      # create the zero state\n",
    "      ht_1 = torch.zeros(ft.size(0), self.num_outputs, 1)\n",
    "    else:\n",
    "      # add the dim=2 to match 3D tensor shape\n",
    "      ht_1 = state.unsqueeze(2)\n",
    "    zts = torch.chunk(zt, zt.size(2), dim=2)\n",
    "    fts = torch.chunk(ft, ft.size(2), dim=2)\n",
    "    hts = []\n",
    "    for t in range(seqlen):\n",
    "      ht = ht_1 * fts[t] + (1 - fts[t]) * zts[t]\n",
    "      # transpose time, channels dims again to match RNN-like shape\n",
    "      hts.append(ht.transpose(1, 2))\n",
    "      # re-assign h[t-1] now\n",
    "      ht_1 = ht\n",
    "    # convert hts list into a 3D tensor [bsz, seq_len, num_outputs]\n",
    "    hts = torch.cat(hts, dim=1)\n",
    "    return hts, ht_1.squeeze(2)\n",
    "      \n",
    "    \n",
    "      \n",
    "fqrnn = fQRNNLayer(1, 100, 2)\n",
    "x = torch.randn(1, 10, 1)\n",
    "ht, state = fqrnn(x)\n",
    "print('ht size: ', ht.size())\n",
    "print('state size: ', state.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#AlexNet classifier\n",
    "class AlexNet(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes=1000):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(256 * 6 * 6, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, num_classes),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), 256 * 6 * 6)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 3, 224, 224])\n",
      "Output tensor y size:  torch.Size([1, 1000])\n"
     ]
    }
   ],
   "source": [
    "alexnet = AlexNet()\n",
    "x = torch.randn(1, 3, 224, 224)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = alexnet(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Time-Delayed Neural Network (TDNN)\n",
    "class StatisticalPooling(nn.Module):\n",
    "\n",
    "    def forward(self, x): \n",
    "        # x is 3-D with axis [B, feats, T]\n",
    "        mu = x.mean(dim=2, keepdim=True)\n",
    "        std = x.std(dim=2, keepdim=True)\n",
    "        return torch.cat((mu, std), dim=1)\n",
    "\n",
    "class TDNN(nn.Module):\n",
    "    # Architecture taken from x-vectors extractor\n",
    "    # https://www.danielpovey.com/files/2018_icassp_xvectors.pdf\n",
    "    def __init__(self, num_inputs=24, num_outputs=2000):\n",
    "        super().__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Conv1d(num_inputs, 512, 5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 3, dilation=2, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 3, dilation=3, padding=3),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 1), \n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 1500, 1), \n",
    "            nn.ReLU(inplace=True),\n",
    "            StatisticalPooling(),\n",
    "            nn.Conv1d(3000, 512, 1), \n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 1), \n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, num_outputs, 1), \n",
    "            nn.LogSoftmax(dim=1)\n",
    "        )   \n",
    "\n",
    "    def forward(self, x): \n",
    "        return self.model(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 24, 10000])\n",
      "Output tensor y size:  torch.Size([1, 2000, 1])\n"
     ]
    }
   ],
   "source": [
    "tdnn = TDNN()\n",
    "x = torch.randn(1, 24, 10000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "# The output has to contain the final pooling through time with \n",
    "# 2000 class activations so [batch_size, num_classes, 1], being the \n",
    "# latter 1 the last time-step after pooling\n",
    "y = tdnn(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 64, 100, 100])\n",
      "Output tensor y size:  torch.Size([1, 64, 100, 100])\n"
     ]
    }
   ],
   "source": [
    "#Residual connections\n",
    "class ResLayer(nn.Module):\n",
    "  \n",
    "  def __init__(self, num_inputs):\n",
    "    super().__init__()\n",
    "    self.num_inputs = num_inputs\n",
    "    num_outputs = num_inputs\n",
    "    self.num_outputs = num_outputs\n",
    "    self.conv1 = nn.Sequential(\n",
    "        nn.Conv2d(num_inputs, num_outputs, 3, padding=1),\n",
    "        nn.BatchNorm2d(num_outputs),\n",
    "        nn.ReLU(inplace=True)\n",
    "    )\n",
    "    self.conv2 = nn.Sequential(\n",
    "        nn.Conv2d(num_outputs, num_outputs, 3, padding=1),\n",
    "        nn.BatchNorm2d(num_outputs),\n",
    "        nn.ReLU(inplace=True)\n",
    "    )\n",
    "    self.out_relu = nn.ReLU(inplace=True)\n",
    "    \n",
    "  def forward(self, x):\n",
    "    # non-linear processing trunk\n",
    "    conv1_h = self.conv1(x)\n",
    "    conv2_h = self.conv2(conv1_h)\n",
    "    # output is result of res connection + non-linear processing\n",
    "    y = self.out_relu(x + conv2_h)\n",
    "    return y\n",
    "    \n",
    "x = torch.randn(1, 64, 100, 100)\n",
    "print('Input tensor x size: ', x.size())\n",
    "reslayer = ResLayer(64)\n",
    "y = reslayer(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([10, 784])\n",
      "Output tensor y size:  torch.Size([10, 784])\n"
     ]
    }
   ],
   "source": [
    "#Auto-Encoder Network\n",
    "class AE(nn.Module):\n",
    "    def __init__(self, num_inputs=784):\n",
    "        super().__init__()\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Linear(num_inputs, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 20)\n",
    "        )\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.Linear(20, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, num_inputs)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.decoder(self.encoder(x))\n",
    "\n",
    "      \n",
    "ae = AE(784)\n",
    "x = torch.randn(10, 784)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = ae(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([10, 784])\n",
      "Input tensor y size:  torch.Size([10, 784])\n",
      "Mean tensor mu size:  torch.Size([10, 20])\n",
      "Covariance tensor logvar size:  torch.Size([10, 20])\n"
     ]
    }
   ],
   "source": [
    "#Variational Auto-Encoder Network\n",
    "# from https://github.com/pytorch/examples/blob/master/vae/main.py\n",
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "\n",
    "        self.fc1 = nn.Linear(784, 400)\n",
    "        self.fc21 = nn.Linear(400, 20)\n",
    "        self.fc22 = nn.Linear(400, 20)\n",
    "        self.fc3 = nn.Linear(20, 400)\n",
    "        self.fc4 = nn.Linear(400, 784)\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = F.relu(self.fc1(x))\n",
    "        return self.fc21(h1), self.fc22(h1)\n",
    "\n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = torch.exp(0.5*logvar)\n",
    "        eps = torch.randn_like(std)\n",
    "        return mu + eps*std\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = F.relu(self.fc3(z))\n",
    "        return torch.sigmoid(self.fc4(h3))\n",
    "\n",
    "    def forward(self, x):\n",
    "        mu, logvar = self.encode(x.view(-1, 784))\n",
    "        z = self.reparameterize(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "\n",
    "vae = VAE()\n",
    "x = torch.randn(10, 784)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y, mu, logvar = vae(x)\n",
    "print('Input tensor y size: ', y.size())\n",
    "print('Mean tensor mu size: ', mu.size())\n",
    "print('Covariance tensor logvar size: ', logvar.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 4000])\n",
      "Output tensor y size:  torch.Size([1, 1, 1000])\n"
     ]
    }
   ],
   "source": [
    "#Deep Convolutional Auto-Encoder with skip connections (SEGAN G)\n",
    "class DownConv1dBlock(nn.Module):\n",
    "  \n",
    "  def __init__(self, ninp, fmap, kwidth, stride):\n",
    "    super().__init__()\n",
    "    assert stride > 1, stride\n",
    "    self.kwidth = kwidth\n",
    "    self.conv = nn.Conv1d(ninp, fmap, kwidth, stride=stride)\n",
    "    self.act = nn.ReLU(inplace=True)\n",
    "  \n",
    "  def forward(self, x):\n",
    "    # calculate padding with stride > 1\n",
    "    pad_left = self.kwidth // 2 - 1\n",
    "    pad_right = self.kwidth // 2\n",
    "    xp = F.pad(x, (pad_left, pad_right))\n",
    "    y = self.act(self.conv(xp))\n",
    "    return y\n",
    "\n",
    "block = DownConv1dBlock(1, 1, 31, 4)\n",
    "x = torch.randn(1, 1, 4000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = block(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 1000])\n",
      "Output tensor y size:  torch.Size([1, 1, 4000])\n"
     ]
    }
   ],
   "source": [
    "#Convolutional Layer\n",
    "class UpConv1dBlock(nn.Module):\n",
    "  \n",
    "  def __init__(self, ninp, fmap, kwidth, stride, act=True):\n",
    "    super().__init__()\n",
    "    assert stride > 1, stride\n",
    "    self.kwidth = kwidth\n",
    "    pad = max(0, (stride - kwidth) // -2)\n",
    "    self.deconv = nn.ConvTranspose1d(ninp, fmap, kwidth,\n",
    "                                    stride=stride,\n",
    "                                    padding=pad)\n",
    "    if act:\n",
    "      self.act = nn.ReLU(inplace=True)\n",
    "  \n",
    "  def forward(self, x):\n",
    "    h = self.deconv(x)\n",
    "    if self.kwidth % 2 != 0:\n",
    "      # drop last item for shape compatibility with TensorFlow deconvs\n",
    "      h = h[:, :, :-1]\n",
    "    if hasattr(self, 'act'):\n",
    "      y = self.act(h)\n",
    "    else:\n",
    "      y = h\n",
    "    return y\n",
    "\n",
    "block = UpConv1dBlock(1, 1, 31, 4)\n",
    "x = torch.randn(1, 1, 1000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = block(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 8192])\n",
      "Output tensor y size:  torch.Size([1, 1, 8192])\n"
     ]
    }
   ],
   "source": [
    "class Conv1dGenerator(nn.Module):\n",
    "  \n",
    "  def __init__(self, enc_fmaps=[64, 128, 256, 512], kwidth=31,\n",
    "               pooling=4):\n",
    "    super().__init__()\n",
    "    self.enc = nn.ModuleList()\n",
    "    ninp = 1\n",
    "    for enc_fmap in enc_fmaps:\n",
    "      self.enc.append(DownConv1dBlock(ninp, enc_fmap, kwidth, pooling))\n",
    "      ninp = enc_fmap\n",
    "    \n",
    "    self.dec = nn.ModuleList()\n",
    "    # revert encoder feature maps\n",
    "    dec_fmaps = enc_fmaps[::-1][1:] + [1]\n",
    "    act = True\n",
    "    for di, dec_fmap in enumerate(dec_fmaps, start=1):\n",
    "      if di >= len(dec_fmaps):\n",
    "        # last decoder layer has no activation\n",
    "        act = False\n",
    "      self.dec.append(UpConv1dBlock(ninp, dec_fmap, kwidth, pooling, act=act))\n",
    "      ninp = dec_fmap\n",
    "  \n",
    "  def forward(self, x):\n",
    "    skips = []\n",
    "    h = x\n",
    "    for ei, enc_layer in enumerate(self.enc, start=1):\n",
    "      h = enc_layer(h)\n",
    "      if ei < len(self.enc):\n",
    "        skips.append(h)\n",
    "    # now decode\n",
    "    \n",
    "    for di, dec_layer in enumerate(self.dec, start=1):\n",
    "      if di > 1:\n",
    "        # sum skip connection\n",
    "        skip_h = skips.pop(-1)\n",
    "        h = h + skip_h\n",
    "      h = dec_layer(h)\n",
    "    y = h\n",
    "    return y\n",
    "      \n",
    "G = Conv1dGenerator()\n",
    "x = torch.randn(1, 1, 8192)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = G(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor z size:  torch.Size([1, 100, 1, 1])\n",
      "Output tensor x size:  torch.Size([1, 3, 64, 64])\n"
     ]
    }
   ],
   "source": [
    "#DCGAN G and D\n",
    "# from https://github.com/pytorch/examples/blob/master/dcgan/main.py\n",
    "class Generator(nn.Module):\n",
    "    def __init__(self, nc=3):\n",
    "        super().__init__()\n",
    "        nz = 100\n",
    "        ngf = 64\n",
    "        self.main = nn.Sequential(\n",
    "            # input is Z, going into a convolution\n",
    "            nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 8),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*8) x 4 x 4\n",
    "            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 4),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*4) x 8 x 8\n",
    "            nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 2),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*2) x 16 x 16\n",
    "            nn.ConvTranspose2d(ngf * 2,     ngf, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf) x 32 x 32\n",
    "            nn.ConvTranspose2d(    ngf,      nc, 4, 2, 1, bias=False),\n",
    "            nn.Tanh()\n",
    "            # state size. (nc) x 64 x 64\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "      return self.main(input)\n",
    "\n",
    "z = torch.randn(1, 100, 1, 1)\n",
    "print('Input tensor z size: ', z.size())\n",
    "G = Generator()\n",
    "x = G(z)\n",
    "print('Output tensor x size: ', x.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 3, 64, 64])\n",
      "Output tensor y size:  torch.Size([1, 1, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, nc=3):\n",
    "        super(Discriminator, self).__init__()\n",
    "        ndf = 64\n",
    "        self.main = nn.Sequential(\n",
    "            # input is (nc) x 64 x 64\n",
    "            nn.Conv2d(nc, ndf, 4, 2, 1, bias=False),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf) x 32 x 32\n",
    "            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 2),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*2) x 16 x 16\n",
    "            nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 4),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*4) x 8 x 8\n",
    "            nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 8),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*8) x 4 x 4\n",
    "            nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.main(input)\n",
    "      \n",
    "      \n",
    "x = torch.randn(1, 3, 64, 64)\n",
    "print('Input tensor x size: ', x.size())\n",
    "D = Discriminator()\n",
    "y = D(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple training example GPU in Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Method where n: Inputs\n",
    "def sample_points(n):\n",
    "    '''Method to returns (X,Y), where X of shape (n,2) is the numpy array of points and Y is the (n) array of classes'''    \n",
    "    radius = np.random.uniform(low=0,high=2,size=n).reshape(-1,1) # uniform radius between 0 and 2\n",
    "    angle = np.random.uniform(low=0,high=2*np.pi,size=n).reshape(-1,1) # uniform angle\n",
    "    x1 = radius*np.cos(angle)\n",
    "    x2=radius*np.sin(angle)\n",
    "    y = (radius<1).astype(int).reshape(-1)\n",
    "    x = np.concatenate([x1,x2],axis=1)\n",
    "    return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 2) (10000,)\n"
     ]
    }
   ],
   "source": [
    "# Generate the data\n",
    "trainx,trainy = sample_points(10000)\n",
    "valx,valy = sample_points(500)\n",
    "testx,testy = sample_points(500)\n",
    "print(trainx.shape,trainy.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Method to generate single hidden MLP\n",
    "def generate_single_hidden_MLP(n_hidden_neurons):\n",
    "    return nn.Sequential(nn.Linear(2,n_hidden_neurons),nn.ReLU(),nn.Linear(n_hidden_neurons,2))\n",
    "model1 = generate_single_hidden_MLP(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.FloatTensor torch.IntTensor\n"
     ]
    }
   ],
   "source": [
    "#Conversion\n",
    "trainx = torch.from_numpy(trainx).float()\n",
    "valx = torch.from_numpy(valx).float()\n",
    "testx = torch.from_numpy(testx).float()\n",
    "trainy = torch.from_numpy(trainy)\n",
    "valy = torch.from_numpy(valy)\n",
    "testy = torch.from_numpy(testy)\n",
    "print(trainx.type(),trainy.type())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_routine(net,dataset,n_iters,gpu):\n",
    "    # organize the data\n",
    "    train_data,train_labels,val_data,val_labels = dataset\n",
    "    \n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(net.parameters(),lr=0.01)\n",
    "    \n",
    "    # use the flag\n",
    "    train_data,train_labels = train_data,train_labels.long()\n",
    "    val_data,val_labels = val_data,val_labels.long()\n",
    "    if gpu:\n",
    "        train_data,train_labels = train_data.cuda(),train_labels.cuda()\n",
    "        val_data,val_labels = val_data.cuda(),val_labels.cuda()\n",
    "        net = net.cuda() # the network parameters also need to be on the gpu !\n",
    "        print(\"Using GPU\")\n",
    "    else:\n",
    "        train_data,train_labels = train_data.cpu(),train_labels.cpu()\n",
    "        val_data,val_labels = val_data.cpu(),val_labels.cpu()\n",
    "        net = net.cpu() # the network parameters also need to be on the gpu !\n",
    "        print(\"Using CPU\")\n",
    "    for i in range(n_iters):\n",
    "        # forward pass\n",
    "        train_output = net(train_data)\n",
    "        train_loss = criterion(train_output,train_labels)\n",
    "        # backward pass and optimization\n",
    "        train_loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Once every 100 iterations, print statistics\n",
    "        if i%100==0:\n",
    "            print(\"At iteration\",i)\n",
    "            # compute the accuracy of the prediction\n",
    "            train_prediction = train_output.cpu().detach().argmax(dim=1)\n",
    "            train_accuracy = (train_prediction.cpu().numpy()==train_labels.cpu().numpy()).mean() \n",
    "            # Now for the validation set\n",
    "            val_output = net(val_data)\n",
    "            val_loss = criterion(val_output,val_labels)\n",
    "            # compute the accuracy of the prediction\n",
    "            val_prediction = val_output.cpu().detach().argmax(dim=1)\n",
    "            val_accuracy = (val_prediction.cpu().numpy()==val_labels.cpu().numpy()).mean() \n",
    "            print(\"Training loss :\",train_loss.cpu().detach().numpy())\n",
    "            print(\"Training accuracy :\",train_accuracy)\n",
    "            print(\"Validation loss :\",val_loss.cpu().detach().numpy())\n",
    "            print(\"Validation accuracy :\",val_accuracy)\n",
    "    \n",
    "    net = net.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = trainx,trainy,valx,valy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU\n",
      "At iteration 0\n",
      "Training loss : 0.7118081\n",
      "Training accuracy : 0.5684\n",
      "Validation loss : 0.71091545\n",
      "Validation accuracy : 0.564\n",
      "At iteration 100\n",
      "Training loss : 0.66902333\n",
      "Training accuracy : 0.6159\n",
      "Validation loss : 0.66611105\n",
      "Validation accuracy : 0.642\n",
      "At iteration 200\n",
      "Training loss : 0.64932925\n",
      "Training accuracy : 0.7798\n",
      "Validation loss : 0.6450787\n",
      "Validation accuracy : 0.792\n",
      "At iteration 300\n",
      "Training loss : 0.63179314\n",
      "Training accuracy : 0.7632\n",
      "Validation loss : 0.6265697\n",
      "Validation accuracy : 0.778\n",
      "At iteration 400\n",
      "Training loss : 0.6137221\n",
      "Training accuracy : 0.7853\n",
      "Validation loss : 0.6076209\n",
      "Validation accuracy : 0.8\n",
      "At iteration 500\n",
      "Training loss : 0.5947446\n",
      "Training accuracy : 0.81\n",
      "Validation loss : 0.5875939\n",
      "Validation accuracy : 0.832\n",
      "At iteration 600\n",
      "Training loss : 0.57508135\n",
      "Training accuracy : 0.8348\n",
      "Validation loss : 0.56689507\n",
      "Validation accuracy : 0.856\n",
      "At iteration 700\n",
      "Training loss : 0.55492496\n",
      "Training accuracy : 0.8526\n",
      "Validation loss : 0.54558283\n",
      "Validation accuracy : 0.868\n",
      "At iteration 800\n",
      "Training loss : 0.53463894\n",
      "Training accuracy : 0.8712\n",
      "Validation loss : 0.5242612\n",
      "Validation accuracy : 0.878\n",
      "At iteration 900\n",
      "Training loss : 0.51445144\n",
      "Training accuracy : 0.8856\n",
      "Validation loss : 0.5030077\n",
      "Validation accuracy : 0.89\n",
      "At iteration 1000\n",
      "Training loss : 0.4945791\n",
      "Training accuracy : 0.8946\n",
      "Validation loss : 0.48224697\n",
      "Validation accuracy : 0.9\n",
      "At iteration 1100\n",
      "Training loss : 0.47513703\n",
      "Training accuracy : 0.9024\n",
      "Validation loss : 0.46219322\n",
      "Validation accuracy : 0.91\n",
      "At iteration 1200\n",
      "Training loss : 0.4562619\n",
      "Training accuracy : 0.9083\n",
      "Validation loss : 0.44296208\n",
      "Validation accuracy : 0.916\n",
      "At iteration 1300\n",
      "Training loss : 0.43799502\n",
      "Training accuracy : 0.9153\n",
      "Validation loss : 0.42455524\n",
      "Validation accuracy : 0.926\n",
      "At iteration 1400\n",
      "Training loss : 0.42036563\n",
      "Training accuracy : 0.9197\n",
      "Validation loss : 0.40697607\n",
      "Validation accuracy : 0.926\n",
      "At iteration 1500\n",
      "Training loss : 0.40343556\n",
      "Training accuracy : 0.9242\n",
      "Validation loss : 0.3903127\n",
      "Validation accuracy : 0.932\n",
      "At iteration 1600\n",
      "Training loss : 0.387268\n",
      "Training accuracy : 0.9289\n",
      "Validation loss : 0.3745096\n",
      "Validation accuracy : 0.936\n",
      "At iteration 1700\n",
      "Training loss : 0.37187067\n",
      "Training accuracy : 0.9339\n",
      "Validation loss : 0.3594591\n",
      "Validation accuracy : 0.938\n",
      "At iteration 1800\n",
      "Training loss : 0.35724384\n",
      "Training accuracy : 0.9371\n",
      "Validation loss : 0.34529784\n",
      "Validation accuracy : 0.94\n",
      "At iteration 1900\n",
      "Training loss : 0.34342638\n",
      "Training accuracy : 0.9407\n",
      "Validation loss : 0.33199862\n",
      "Validation accuracy : 0.94\n",
      "At iteration 2000\n",
      "Training loss : 0.33045578\n",
      "Training accuracy : 0.9439\n",
      "Validation loss : 0.3195325\n",
      "Validation accuracy : 0.942\n",
      "At iteration 2100\n",
      "Training loss : 0.31833664\n",
      "Training accuracy : 0.9452\n",
      "Validation loss : 0.3079157\n",
      "Validation accuracy : 0.946\n",
      "At iteration 2200\n",
      "Training loss : 0.3070174\n",
      "Training accuracy : 0.9477\n",
      "Validation loss : 0.29709908\n",
      "Validation accuracy : 0.952\n",
      "At iteration 2300\n",
      "Training loss : 0.29646808\n",
      "Training accuracy : 0.9501\n",
      "Validation loss : 0.28707486\n",
      "Validation accuracy : 0.954\n",
      "At iteration 2400\n",
      "Training loss : 0.2866488\n",
      "Training accuracy : 0.9522\n",
      "Validation loss : 0.277806\n",
      "Validation accuracy : 0.954\n",
      "At iteration 2500\n",
      "Training loss : 0.27753302\n",
      "Training accuracy : 0.9538\n",
      "Validation loss : 0.26920643\n",
      "Validation accuracy : 0.964\n",
      "At iteration 2600\n",
      "Training loss : 0.26907468\n",
      "Training accuracy : 0.9552\n",
      "Validation loss : 0.26122385\n",
      "Validation accuracy : 0.964\n",
      "At iteration 2700\n",
      "Training loss : 0.26122853\n",
      "Training accuracy : 0.957\n",
      "Validation loss : 0.25376922\n",
      "Validation accuracy : 0.96\n",
      "At iteration 2800\n",
      "Training loss : 0.25394994\n",
      "Training accuracy : 0.9583\n",
      "Validation loss : 0.24687512\n",
      "Validation accuracy : 0.962\n",
      "At iteration 2900\n",
      "Training loss : 0.2471873\n",
      "Training accuracy : 0.9593\n",
      "Validation loss : 0.24051544\n",
      "Validation accuracy : 0.962\n",
      "At iteration 3000\n",
      "Training loss : 0.24089804\n",
      "Training accuracy : 0.9598\n",
      "Validation loss : 0.23461032\n",
      "Validation accuracy : 0.964\n",
      "At iteration 3100\n",
      "Training loss : 0.23502745\n",
      "Training accuracy : 0.9602\n",
      "Validation loss : 0.22909294\n",
      "Validation accuracy : 0.964\n",
      "At iteration 3200\n",
      "Training loss : 0.22952259\n",
      "Training accuracy : 0.9607\n",
      "Validation loss : 0.22392379\n",
      "Validation accuracy : 0.966\n",
      "At iteration 3300\n",
      "Training loss : 0.22435473\n",
      "Training accuracy : 0.9614\n",
      "Validation loss : 0.21906923\n",
      "Validation accuracy : 0.968\n",
      "At iteration 3400\n",
      "Training loss : 0.21949434\n",
      "Training accuracy : 0.9622\n",
      "Validation loss : 0.2145091\n",
      "Validation accuracy : 0.966\n",
      "At iteration 3500\n",
      "Training loss : 0.21491495\n",
      "Training accuracy : 0.9623\n",
      "Validation loss : 0.21020019\n",
      "Validation accuracy : 0.966\n",
      "At iteration 3600\n",
      "Training loss : 0.21059236\n",
      "Training accuracy : 0.9628\n",
      "Validation loss : 0.20612694\n",
      "Validation accuracy : 0.966\n",
      "At iteration 3700\n",
      "Training loss : 0.20650664\n",
      "Training accuracy : 0.9636\n",
      "Validation loss : 0.20227435\n",
      "Validation accuracy : 0.966\n",
      "At iteration 3800\n",
      "Training loss : 0.2026407\n",
      "Training accuracy : 0.964\n",
      "Validation loss : 0.198617\n",
      "Validation accuracy : 0.964\n",
      "At iteration 3900\n",
      "Training loss : 0.1989775\n",
      "Training accuracy : 0.9643\n",
      "Validation loss : 0.19514658\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4000\n",
      "Training loss : 0.19550143\n",
      "Training accuracy : 0.9647\n",
      "Validation loss : 0.19184619\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4100\n",
      "Training loss : 0.1921972\n",
      "Training accuracy : 0.9647\n",
      "Validation loss : 0.18870373\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4200\n",
      "Training loss : 0.18905178\n",
      "Training accuracy : 0.9649\n",
      "Validation loss : 0.18569607\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4300\n",
      "Training loss : 0.18605176\n",
      "Training accuracy : 0.9651\n",
      "Validation loss : 0.18281807\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4400\n",
      "Training loss : 0.18318605\n",
      "Training accuracy : 0.9654\n",
      "Validation loss : 0.18005896\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4500\n",
      "Training loss : 0.18044439\n",
      "Training accuracy : 0.9655\n",
      "Validation loss : 0.17740764\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4600\n",
      "Training loss : 0.17781685\n",
      "Training accuracy : 0.9656\n",
      "Validation loss : 0.1748722\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4700\n",
      "Training loss : 0.17529394\n",
      "Training accuracy : 0.9658\n",
      "Validation loss : 0.17244564\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4800\n",
      "Training loss : 0.17287111\n",
      "Training accuracy : 0.9661\n",
      "Validation loss : 0.17012142\n",
      "Validation accuracy : 0.966\n",
      "At iteration 4900\n",
      "Training loss : 0.17053913\n",
      "Training accuracy : 0.9663\n",
      "Validation loss : 0.16788584\n",
      "Validation accuracy : 0.966\n",
      "At iteration 5000\n",
      "Training loss : 0.16829854\n",
      "Training accuracy : 0.9669\n",
      "Validation loss : 0.1657299\n",
      "Validation accuracy : 0.966\n",
      "At iteration 5100\n",
      "Training loss : 0.16614076\n",
      "Training accuracy : 0.9668\n",
      "Validation loss : 0.16365573\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5200\n",
      "Training loss : 0.16406125\n",
      "Training accuracy : 0.9669\n",
      "Validation loss : 0.16165243\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5300\n",
      "Training loss : 0.16205607\n",
      "Training accuracy : 0.9672\n",
      "Validation loss : 0.15971616\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5400\n",
      "Training loss : 0.1601202\n",
      "Training accuracy : 0.9674\n",
      "Validation loss : 0.15784413\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5500\n",
      "Training loss : 0.1582505\n",
      "Training accuracy : 0.9682\n",
      "Validation loss : 0.15603238\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5600\n",
      "Training loss : 0.15644476\n",
      "Training accuracy : 0.9682\n",
      "Validation loss : 0.15427677\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5700\n",
      "Training loss : 0.1546972\n",
      "Training accuracy : 0.9683\n",
      "Validation loss : 0.15257747\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5800\n",
      "Training loss : 0.1530049\n",
      "Training accuracy : 0.9682\n",
      "Validation loss : 0.15092951\n",
      "Validation accuracy : 0.968\n",
      "At iteration 5900\n",
      "Training loss : 0.15136448\n",
      "Training accuracy : 0.9684\n",
      "Validation loss : 0.1493346\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6000\n",
      "Training loss : 0.14977424\n",
      "Training accuracy : 0.9688\n",
      "Validation loss : 0.14778078\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6100\n",
      "Training loss : 0.14823256\n",
      "Training accuracy : 0.9685\n",
      "Validation loss : 0.14626728\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6200\n",
      "Training loss : 0.14673655\n",
      "Training accuracy : 0.9686\n",
      "Validation loss : 0.14479254\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6300\n",
      "Training loss : 0.14528437\n",
      "Training accuracy : 0.9687\n",
      "Validation loss : 0.14335787\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training loss : 0.1438717\n",
      "Training accuracy : 0.9689\n",
      "Validation loss : 0.14196372\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6500\n",
      "Training loss : 0.1424968\n",
      "Training accuracy : 0.9695\n",
      "Validation loss : 0.14060557\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6600\n",
      "Training loss : 0.14115818\n",
      "Training accuracy : 0.9697\n",
      "Validation loss : 0.13927749\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6700\n",
      "Training loss : 0.13985379\n",
      "Training accuracy : 0.9699\n",
      "Validation loss : 0.13797294\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6800\n",
      "Training loss : 0.13858303\n",
      "Training accuracy : 0.97\n",
      "Validation loss : 0.13669863\n",
      "Validation accuracy : 0.968\n",
      "At iteration 6900\n",
      "Training loss : 0.13734446\n",
      "Training accuracy : 0.9705\n",
      "Validation loss : 0.13545376\n",
      "Validation accuracy : 0.968\n",
      "At iteration 7000\n",
      "Training loss : 0.13613601\n",
      "Training accuracy : 0.9707\n",
      "Validation loss : 0.13423628\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7100\n",
      "Training loss : 0.1349541\n",
      "Training accuracy : 0.9707\n",
      "Validation loss : 0.13304225\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7200\n",
      "Training loss : 0.1337974\n",
      "Training accuracy : 0.9707\n",
      "Validation loss : 0.13186707\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7300\n",
      "Training loss : 0.1326674\n",
      "Training accuracy : 0.9709\n",
      "Validation loss : 0.13071856\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7400\n",
      "Training loss : 0.1315611\n",
      "Training accuracy : 0.971\n",
      "Validation loss : 0.12959607\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7500\n",
      "Training loss : 0.1304771\n",
      "Training accuracy : 0.9712\n",
      "Validation loss : 0.12850083\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7600\n",
      "Training loss : 0.12941414\n",
      "Training accuracy : 0.9715\n",
      "Validation loss : 0.12742671\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7700\n",
      "Training loss : 0.12837486\n",
      "Training accuracy : 0.9715\n",
      "Validation loss : 0.12637323\n",
      "Validation accuracy : 0.97\n",
      "At iteration 7800\n",
      "Training loss : 0.12736008\n",
      "Training accuracy : 0.9717\n",
      "Validation loss : 0.12534139\n",
      "Validation accuracy : 0.972\n",
      "At iteration 7900\n",
      "Training loss : 0.1263682\n",
      "Training accuracy : 0.9719\n",
      "Validation loss : 0.124339655\n",
      "Validation accuracy : 0.972\n",
      "At iteration 8000\n",
      "Training loss : 0.12539852\n",
      "Training accuracy : 0.972\n",
      "Validation loss : 0.12336038\n",
      "Validation accuracy : 0.972\n",
      "At iteration 8100\n",
      "Training loss : 0.12444831\n",
      "Training accuracy : 0.9723\n",
      "Validation loss : 0.1224013\n",
      "Validation accuracy : 0.972\n",
      "At iteration 8200\n",
      "Training loss : 0.123516515\n",
      "Training accuracy : 0.9724\n",
      "Validation loss : 0.12146191\n",
      "Validation accuracy : 0.974\n",
      "At iteration 8300\n",
      "Training loss : 0.12260431\n",
      "Training accuracy : 0.9727\n",
      "Validation loss : 0.12054178\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8400\n",
      "Training loss : 0.121709034\n",
      "Training accuracy : 0.973\n",
      "Validation loss : 0.11963999\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8500\n",
      "Training loss : 0.12082988\n",
      "Training accuracy : 0.9734\n",
      "Validation loss : 0.11875459\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8600\n",
      "Training loss : 0.11996615\n",
      "Training accuracy : 0.9734\n",
      "Validation loss : 0.11788429\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8700\n",
      "Training loss : 0.119117685\n",
      "Training accuracy : 0.9737\n",
      "Validation loss : 0.117029354\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8800\n",
      "Training loss : 0.11828457\n",
      "Training accuracy : 0.9739\n",
      "Validation loss : 0.11618841\n",
      "Validation accuracy : 0.976\n",
      "At iteration 8900\n",
      "Training loss : 0.117467076\n",
      "Training accuracy : 0.9739\n",
      "Validation loss : 0.11536148\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9000\n",
      "Training loss : 0.11666382\n",
      "Training accuracy : 0.9742\n",
      "Validation loss : 0.11454557\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9100\n",
      "Training loss : 0.11587511\n",
      "Training accuracy : 0.9747\n",
      "Validation loss : 0.11374037\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9200\n",
      "Training loss : 0.11509983\n",
      "Training accuracy : 0.9746\n",
      "Validation loss : 0.112947986\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9300\n",
      "Training loss : 0.114338085\n",
      "Training accuracy : 0.9749\n",
      "Validation loss : 0.112172775\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9400\n",
      "Training loss : 0.113589406\n",
      "Training accuracy : 0.975\n",
      "Validation loss : 0.11141364\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9500\n",
      "Training loss : 0.11285502\n",
      "Training accuracy : 0.9752\n",
      "Validation loss : 0.110663325\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9600\n",
      "Training loss : 0.1121344\n",
      "Training accuracy : 0.9756\n",
      "Validation loss : 0.1099254\n",
      "Validation accuracy : 0.976\n",
      "At iteration 9700\n",
      "Training loss : 0.11142578\n",
      "Training accuracy : 0.9756\n",
      "Validation loss : 0.10919999\n",
      "Validation accuracy : 0.978\n",
      "At iteration 9800\n",
      "Training loss : 0.11072747\n",
      "Training accuracy : 0.9757\n",
      "Validation loss : 0.10848155\n",
      "Validation accuracy : 0.978\n",
      "At iteration 9900\n",
      "Training loss : 0.1100392\n",
      "Training accuracy : 0.9758\n",
      "Validation loss : 0.10776656\n",
      "Validation accuracy : 0.978\n"
     ]
    }
   ],
   "source": [
    "gpu =  True\n",
    "gpu = gpu and torch.cuda.is_available() # to know if you actually can use the GPU\n",
    "\n",
    "training_routine(model1,dataset,10000,gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU\n",
      "At iteration 0\n",
      "Training loss : 0.76269054\n",
      "Training accuracy : 0.4908\n",
      "Validation loss : 0.7724064\n",
      "Validation accuracy : 0.48\n",
      "At iteration 100\n",
      "Training loss : 0.6783634\n",
      "Training accuracy : 0.644\n",
      "Validation loss : 0.6798217\n",
      "Validation accuracy : 0.648\n",
      "At iteration 200\n",
      "Training loss : 0.6511425\n",
      "Training accuracy : 0.6862\n",
      "Validation loss : 0.6489191\n",
      "Validation accuracy : 0.684\n",
      "At iteration 300\n",
      "Training loss : 0.6329879\n",
      "Training accuracy : 0.7417\n",
      "Validation loss : 0.6283912\n",
      "Validation accuracy : 0.738\n",
      "At iteration 400\n",
      "Training loss : 0.6161927\n",
      "Training accuracy : 0.7458\n",
      "Validation loss : 0.6094604\n",
      "Validation accuracy : 0.754\n",
      "At iteration 500\n",
      "Training loss : 0.5995909\n",
      "Training accuracy : 0.7548\n",
      "Validation loss : 0.5907559\n",
      "Validation accuracy : 0.758\n",
      "At iteration 600\n",
      "Training loss : 0.5831825\n",
      "Training accuracy : 0.7652\n",
      "Validation loss : 0.572328\n",
      "Validation accuracy : 0.778\n",
      "At iteration 700\n",
      "Training loss : 0.5671715\n",
      "Training accuracy : 0.7782\n",
      "Validation loss : 0.55437535\n",
      "Validation accuracy : 0.794\n",
      "At iteration 800\n",
      "Training loss : 0.5520917\n",
      "Training accuracy : 0.7881\n",
      "Validation loss : 0.5374959\n",
      "Validation accuracy : 0.804\n",
      "At iteration 900\n",
      "Training loss : 0.5381345\n",
      "Training accuracy : 0.7963\n",
      "Validation loss : 0.52161294\n",
      "Validation accuracy : 0.818\n",
      "At iteration 1000\n",
      "Training loss : 0.52532786\n",
      "Training accuracy : 0.803\n",
      "Validation loss : 0.5068566\n",
      "Validation accuracy : 0.83\n",
      "At iteration 1100\n",
      "Training loss : 0.513652\n",
      "Training accuracy : 0.8063\n",
      "Validation loss : 0.4934073\n",
      "Validation accuracy : 0.832\n",
      "At iteration 1200\n",
      "Training loss : 0.50298125\n",
      "Training accuracy : 0.8052\n",
      "Validation loss : 0.48120517\n",
      "Validation accuracy : 0.83\n",
      "At iteration 1300\n",
      "Training loss : 0.4931935\n",
      "Training accuracy : 0.8045\n",
      "Validation loss : 0.47004968\n",
      "Validation accuracy : 0.824\n",
      "At iteration 1400\n",
      "Training loss : 0.4841694\n",
      "Training accuracy : 0.804\n",
      "Validation loss : 0.4597436\n",
      "Validation accuracy : 0.822\n",
      "At iteration 1500\n",
      "Training loss : 0.47581217\n",
      "Training accuracy : 0.8026\n",
      "Validation loss : 0.45021793\n",
      "Validation accuracy : 0.816\n",
      "At iteration 1600\n",
      "Training loss : 0.46802852\n",
      "Training accuracy : 0.802\n",
      "Validation loss : 0.4413466\n",
      "Validation accuracy : 0.818\n",
      "At iteration 1700\n",
      "Training loss : 0.46074384\n",
      "Training accuracy : 0.8024\n",
      "Validation loss : 0.43306914\n",
      "Validation accuracy : 0.822\n",
      "At iteration 1800\n",
      "Training loss : 0.45389515\n",
      "Training accuracy : 0.8024\n",
      "Validation loss : 0.42532194\n",
      "Validation accuracy : 0.826\n",
      "At iteration 1900\n",
      "Training loss : 0.4474227\n",
      "Training accuracy : 0.8032\n",
      "Validation loss : 0.41803303\n",
      "Validation accuracy : 0.826\n",
      "At iteration 2000\n",
      "Training loss : 0.4412548\n",
      "Training accuracy : 0.8043\n",
      "Validation loss : 0.41114753\n",
      "Validation accuracy : 0.826\n",
      "At iteration 2100\n",
      "Training loss : 0.4353002\n",
      "Training accuracy : 0.8055\n",
      "Validation loss : 0.40462032\n",
      "Validation accuracy : 0.828\n",
      "At iteration 2200\n",
      "Training loss : 0.42948365\n",
      "Training accuracy : 0.8065\n",
      "Validation loss : 0.3984057\n",
      "Validation accuracy : 0.824\n",
      "At iteration 2300\n",
      "Training loss : 0.4237735\n",
      "Training accuracy : 0.8074\n",
      "Validation loss : 0.39227363\n",
      "Validation accuracy : 0.824\n",
      "At iteration 2400\n",
      "Training loss : 0.41813487\n",
      "Training accuracy : 0.8086\n",
      "Validation loss : 0.3863027\n",
      "Validation accuracy : 0.824\n",
      "At iteration 2500\n",
      "Training loss : 0.41256616\n",
      "Training accuracy : 0.8102\n",
      "Validation loss : 0.38049093\n",
      "Validation accuracy : 0.824\n",
      "At iteration 2600\n",
      "Training loss : 0.4070104\n",
      "Training accuracy : 0.8111\n",
      "Validation loss : 0.37477428\n",
      "Validation accuracy : 0.822\n",
      "At iteration 2700\n",
      "Training loss : 0.40142018\n",
      "Training accuracy : 0.8128\n",
      "Validation loss : 0.36914307\n",
      "Validation accuracy : 0.824\n",
      "At iteration 2800\n",
      "Training loss : 0.3956961\n",
      "Training accuracy : 0.815\n",
      "Validation loss : 0.36353767\n",
      "Validation accuracy : 0.828\n",
      "At iteration 2900\n",
      "Training loss : 0.3898586\n",
      "Training accuracy : 0.8168\n",
      "Validation loss : 0.35802647\n",
      "Validation accuracy : 0.83\n",
      "At iteration 3000\n",
      "Training loss : 0.3839619\n",
      "Training accuracy : 0.8193\n",
      "Validation loss : 0.35259816\n",
      "Validation accuracy : 0.826\n",
      "At iteration 3100\n",
      "Training loss : 0.37802932\n",
      "Training accuracy : 0.8196\n",
      "Validation loss : 0.34729293\n",
      "Validation accuracy : 0.83\n",
      "At iteration 3200\n",
      "Training loss : 0.37208742\n",
      "Training accuracy : 0.8209\n",
      "Validation loss : 0.34194076\n",
      "Validation accuracy : 0.83\n",
      "At iteration 3300\n",
      "Training loss : 0.3660486\n",
      "Training accuracy : 0.8223\n",
      "Validation loss : 0.3365376\n",
      "Validation accuracy : 0.832\n",
      "At iteration 3400\n",
      "Training loss : 0.3598807\n",
      "Training accuracy : 0.823\n",
      "Validation loss : 0.33117342\n",
      "Validation accuracy : 0.834\n",
      "At iteration 3500\n",
      "Training loss : 0.35373342\n",
      "Training accuracy : 0.8239\n",
      "Validation loss : 0.32587525\n",
      "Validation accuracy : 0.836\n",
      "At iteration 3600\n",
      "Training loss : 0.34759676\n",
      "Training accuracy : 0.826\n",
      "Validation loss : 0.32045293\n",
      "Validation accuracy : 0.836\n",
      "At iteration 3700\n",
      "Training loss : 0.34146976\n",
      "Training accuracy : 0.8279\n",
      "Validation loss : 0.3149091\n",
      "Validation accuracy : 0.84\n",
      "At iteration 3800\n",
      "Training loss : 0.3353786\n",
      "Training accuracy : 0.8287\n",
      "Validation loss : 0.30947328\n",
      "Validation accuracy : 0.84\n",
      "At iteration 3900\n",
      "Training loss : 0.32934564\n",
      "Training accuracy : 0.8299\n",
      "Validation loss : 0.30391333\n",
      "Validation accuracy : 0.844\n",
      "At iteration 4000\n",
      "Training loss : 0.32334557\n",
      "Training accuracy : 0.8309\n",
      "Validation loss : 0.29842636\n",
      "Validation accuracy : 0.844\n",
      "At iteration 4100\n",
      "Training loss : 0.31740046\n",
      "Training accuracy : 0.8322\n",
      "Validation loss : 0.2929541\n",
      "Validation accuracy : 0.844\n",
      "At iteration 4200\n",
      "Training loss : 0.31149507\n",
      "Training accuracy : 0.8328\n",
      "Validation loss : 0.28759667\n",
      "Validation accuracy : 0.848\n",
      "At iteration 4300\n",
      "Training loss : 0.30566058\n",
      "Training accuracy : 0.8338\n",
      "Validation loss : 0.28225887\n",
      "Validation accuracy : 0.85\n",
      "At iteration 4400\n",
      "Training loss : 0.29991812\n",
      "Training accuracy : 0.8349\n",
      "Validation loss : 0.27694276\n",
      "Validation accuracy : 0.854\n",
      "At iteration 4500\n",
      "Training loss : 0.29428273\n",
      "Training accuracy : 0.8353\n",
      "Validation loss : 0.27173355\n",
      "Validation accuracy : 0.858\n",
      "At iteration 4600\n",
      "Training loss : 0.28874946\n",
      "Training accuracy : 0.8369\n",
      "Validation loss : 0.26669887\n",
      "Validation accuracy : 0.858\n",
      "At iteration 4700\n",
      "Training loss : 0.283337\n",
      "Training accuracy : 0.9027\n",
      "Validation loss : 0.261834\n",
      "Validation accuracy : 0.922\n",
      "At iteration 4800\n",
      "Training loss : 0.27804118\n",
      "Training accuracy : 0.906\n",
      "Validation loss : 0.25711116\n",
      "Validation accuracy : 0.926\n",
      "At iteration 4900\n",
      "Training loss : 0.27288976\n",
      "Training accuracy : 0.91\n",
      "Validation loss : 0.25254232\n",
      "Validation accuracy : 0.928\n",
      "At iteration 5000\n",
      "Training loss : 0.267875\n",
      "Training accuracy : 0.9126\n",
      "Validation loss : 0.24804604\n",
      "Validation accuracy : 0.936\n",
      "At iteration 5100\n",
      "Training loss : 0.26301077\n",
      "Training accuracy : 0.9163\n",
      "Validation loss : 0.2436254\n",
      "Validation accuracy : 0.938\n",
      "At iteration 5200\n",
      "Training loss : 0.25828964\n",
      "Training accuracy : 0.9182\n",
      "Validation loss : 0.23936556\n",
      "Validation accuracy : 0.942\n",
      "At iteration 5300\n",
      "Training loss : 0.25366822\n",
      "Training accuracy : 0.9217\n",
      "Validation loss : 0.23529056\n",
      "Validation accuracy : 0.94\n",
      "At iteration 5400\n",
      "Training loss : 0.24917851\n",
      "Training accuracy : 0.9254\n",
      "Validation loss : 0.23134162\n",
      "Validation accuracy : 0.942\n",
      "At iteration 5500\n",
      "Training loss : 0.24481705\n",
      "Training accuracy : 0.9284\n",
      "Validation loss : 0.22752197\n",
      "Validation accuracy : 0.946\n",
      "At iteration 5600\n",
      "Training loss : 0.24057925\n",
      "Training accuracy : 0.93\n",
      "Validation loss : 0.22388607\n",
      "Validation accuracy : 0.946\n",
      "At iteration 5700\n",
      "Training loss : 0.2364409\n",
      "Training accuracy : 0.9325\n",
      "Validation loss : 0.2203922\n",
      "Validation accuracy : 0.95\n",
      "At iteration 5800\n",
      "Training loss : 0.23244439\n",
      "Training accuracy : 0.9341\n",
      "Validation loss : 0.2170233\n",
      "Validation accuracy : 0.952\n",
      "At iteration 5900\n",
      "Training loss : 0.22859651\n",
      "Training accuracy : 0.9362\n",
      "Validation loss : 0.21381147\n",
      "Validation accuracy : 0.954\n",
      "At iteration 6000\n",
      "Training loss : 0.22487111\n",
      "Training accuracy : 0.9377\n",
      "Validation loss : 0.21075083\n",
      "Validation accuracy : 0.952\n",
      "At iteration 6100\n",
      "Training loss : 0.2212789\n",
      "Training accuracy : 0.9395\n",
      "Validation loss : 0.20780644\n",
      "Validation accuracy : 0.952\n",
      "At iteration 6200\n",
      "Training loss : 0.21781194\n",
      "Training accuracy : 0.9413\n",
      "Validation loss : 0.20493747\n",
      "Validation accuracy : 0.952\n",
      "At iteration 6300\n",
      "Training loss : 0.21448576\n",
      "Training accuracy : 0.9433\n",
      "Validation loss : 0.20220074\n",
      "Validation accuracy : 0.952\n",
      "At iteration 6400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training loss : 0.21129067\n",
      "Training accuracy : 0.9445\n",
      "Validation loss : 0.1995838\n",
      "Validation accuracy : 0.956\n",
      "At iteration 6500\n",
      "Training loss : 0.20821337\n",
      "Training accuracy : 0.9454\n",
      "Validation loss : 0.19708256\n",
      "Validation accuracy : 0.956\n",
      "At iteration 6600\n",
      "Training loss : 0.20523506\n",
      "Training accuracy : 0.9465\n",
      "Validation loss : 0.19465877\n",
      "Validation accuracy : 0.956\n",
      "At iteration 6700\n",
      "Training loss : 0.20236887\n",
      "Training accuracy : 0.9476\n",
      "Validation loss : 0.19233026\n",
      "Validation accuracy : 0.956\n",
      "At iteration 6800\n",
      "Training loss : 0.19960491\n",
      "Training accuracy : 0.9487\n",
      "Validation loss : 0.19010612\n",
      "Validation accuracy : 0.956\n",
      "At iteration 6900\n",
      "Training loss : 0.1969365\n",
      "Training accuracy : 0.9491\n",
      "Validation loss : 0.18798131\n",
      "Validation accuracy : 0.96\n",
      "At iteration 7000\n",
      "Training loss : 0.1943762\n",
      "Training accuracy : 0.9496\n",
      "Validation loss : 0.18591887\n",
      "Validation accuracy : 0.96\n",
      "At iteration 7100\n",
      "Training loss : 0.19191302\n",
      "Training accuracy : 0.9504\n",
      "Validation loss : 0.18392113\n",
      "Validation accuracy : 0.96\n",
      "At iteration 7200\n",
      "Training loss : 0.18954802\n",
      "Training accuracy : 0.9506\n",
      "Validation loss : 0.18199223\n",
      "Validation accuracy : 0.962\n",
      "At iteration 7300\n",
      "Training loss : 0.18726237\n",
      "Training accuracy : 0.9504\n",
      "Validation loss : 0.18014193\n",
      "Validation accuracy : 0.962\n",
      "At iteration 7400\n",
      "Training loss : 0.18506064\n",
      "Training accuracy : 0.9507\n",
      "Validation loss : 0.17835341\n",
      "Validation accuracy : 0.96\n",
      "At iteration 7500\n",
      "Training loss : 0.18293747\n",
      "Training accuracy : 0.9513\n",
      "Validation loss : 0.1766176\n",
      "Validation accuracy : 0.96\n",
      "At iteration 7600\n",
      "Training loss : 0.18088453\n",
      "Training accuracy : 0.952\n",
      "Validation loss : 0.17494924\n",
      "Validation accuracy : 0.958\n",
      "At iteration 7700\n",
      "Training loss : 0.17890099\n",
      "Training accuracy : 0.9524\n",
      "Validation loss : 0.17332347\n",
      "Validation accuracy : 0.958\n",
      "At iteration 7800\n",
      "Training loss : 0.17698227\n",
      "Training accuracy : 0.9533\n",
      "Validation loss : 0.17174731\n",
      "Validation accuracy : 0.956\n",
      "At iteration 7900\n",
      "Training loss : 0.17512928\n",
      "Training accuracy : 0.9532\n",
      "Validation loss : 0.17022309\n",
      "Validation accuracy : 0.954\n",
      "At iteration 8000\n",
      "Training loss : 0.17334075\n",
      "Training accuracy : 0.9534\n",
      "Validation loss : 0.16874667\n",
      "Validation accuracy : 0.954\n",
      "At iteration 8100\n",
      "Training loss : 0.17161438\n",
      "Training accuracy : 0.9536\n",
      "Validation loss : 0.16731356\n",
      "Validation accuracy : 0.954\n",
      "At iteration 8200\n",
      "Training loss : 0.16994311\n",
      "Training accuracy : 0.9538\n",
      "Validation loss : 0.16593246\n",
      "Validation accuracy : 0.954\n",
      "At iteration 8300\n",
      "Training loss : 0.16832718\n",
      "Training accuracy : 0.9539\n",
      "Validation loss : 0.16459943\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8400\n",
      "Training loss : 0.16676056\n",
      "Training accuracy : 0.954\n",
      "Validation loss : 0.16331236\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8500\n",
      "Training loss : 0.16524649\n",
      "Training accuracy : 0.9542\n",
      "Validation loss : 0.16205975\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8600\n",
      "Training loss : 0.16377926\n",
      "Training accuracy : 0.9539\n",
      "Validation loss : 0.16085213\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8700\n",
      "Training loss : 0.16235583\n",
      "Training accuracy : 0.9542\n",
      "Validation loss : 0.15969047\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8800\n",
      "Training loss : 0.16097662\n",
      "Training accuracy : 0.9541\n",
      "Validation loss : 0.15857004\n",
      "Validation accuracy : 0.952\n",
      "At iteration 8900\n",
      "Training loss : 0.159641\n",
      "Training accuracy : 0.9542\n",
      "Validation loss : 0.15748373\n",
      "Validation accuracy : 0.95\n",
      "At iteration 9000\n",
      "Training loss : 0.15834653\n",
      "Training accuracy : 0.9544\n",
      "Validation loss : 0.1564258\n",
      "Validation accuracy : 0.954\n",
      "At iteration 9100\n",
      "Training loss : 0.15708913\n",
      "Training accuracy : 0.9543\n",
      "Validation loss : 0.1554039\n",
      "Validation accuracy : 0.954\n",
      "At iteration 9200\n",
      "Training loss : 0.1558697\n",
      "Training accuracy : 0.9544\n",
      "Validation loss : 0.1544113\n",
      "Validation accuracy : 0.954\n",
      "At iteration 9300\n",
      "Training loss : 0.15468487\n",
      "Training accuracy : 0.9543\n",
      "Validation loss : 0.15345308\n",
      "Validation accuracy : 0.954\n",
      "At iteration 9400\n",
      "Training loss : 0.15353304\n",
      "Training accuracy : 0.9544\n",
      "Validation loss : 0.1525254\n",
      "Validation accuracy : 0.952\n",
      "At iteration 9500\n",
      "Training loss : 0.1524147\n",
      "Training accuracy : 0.9545\n",
      "Validation loss : 0.15162133\n",
      "Validation accuracy : 0.952\n",
      "At iteration 9600\n",
      "Training loss : 0.15132838\n",
      "Training accuracy : 0.9545\n",
      "Validation loss : 0.15074122\n",
      "Validation accuracy : 0.952\n",
      "At iteration 9700\n",
      "Training loss : 0.15027274\n",
      "Training accuracy : 0.9547\n",
      "Validation loss : 0.1498841\n",
      "Validation accuracy : 0.952\n",
      "At iteration 9800\n",
      "Training loss : 0.14924504\n",
      "Training accuracy : 0.9548\n",
      "Validation loss : 0.14905985\n",
      "Validation accuracy : 0.952\n",
      "At iteration 9900\n",
      "Training loss : 0.148246\n",
      "Training accuracy : 0.9551\n",
      "Validation loss : 0.14825596\n",
      "Validation accuracy : 0.95\n"
     ]
    }
   ],
   "source": [
    "# Let's try with 3 hidden neurons.\n",
    "model2 = generate_single_hidden_MLP(3) \n",
    "training_routine(model2,dataset,10000,gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(240, 2) (260, 2)\n"
     ]
    }
   ],
   "source": [
    "out = model2(testx).argmax(dim=1).detach().numpy()\n",
    "green = testx.numpy()[np.where(out==1)]\n",
    "red = testx.numpy()[np.where(out==0)]\n",
    "print(green.shape,red.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Method to print Model with datapoints\n",
    "def print_model(model,datapoints):\n",
    "    out = model(datapoints).argmax(dim=1).detach().numpy()\n",
    "    green = datapoints.numpy()[np.where(out==1)]\n",
    "    red = datapoints.numpy()[np.where(out==0)]\n",
    "\n",
    "    circle1 = plt.Circle((0, 0), 1, color='r')\n",
    "    circle2 = plt.Circle((0, 0), 1, color='y',fill=False)\n",
    "\n",
    "    fig, ax = plt.subplots() # note we must use plt.subplots, not plt.subplot\n",
    "    # (or if you have an existing figure)\n",
    "    # fig = plt.gcf()\n",
    "    # ax = fig.gca()\n",
    "    plt.xlim((-2,2))\n",
    "    plt.ylim((-2,2))\n",
    "\n",
    "    pos_values = plt.scatter(x=green[:,0],y=green[:,1], color='g',)\n",
    "    neg_values = plt.scatter(x=red[:,0],y=red[:,1], color='b',)\n",
    "\n",
    "    ax.add_artist(circle1)\n",
    "    ax.add_artist(circle2)\n",
    "    ax.add_artist(pos_values)\n",
    "    ax.add_artist(neg_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO2deZxdRZX4v+e97qTTSUggCRAInYACEggCiSCijiPjDKBDFNGZGBHXkO5xfjKrCjOy/IyO428WHEmw2QYhRtQBWQRc4u6IGLYEEpBAVhLIRtZOJ9396vfHfS/9+vate+vu93XX9/O5n+733n1Vdevde07VOadOiVIKi8VisVhKeTfAYrFYLMXAKgSLxWKxAFYhWCwWi6WKVQgWi8ViAaxCsFgsFksVqxAsFovFAiSgEETkOBH5mYisEpFnReQzHueIiHxNRFaLyHIROStuvRaLxWJJlqYEyugF/k4p9YSIjAUeF5EfK6VW1p1zIXBi9TgHWFT9a7FYLJaCEHuGoJTarJR6ovr/HmAVcKzrtNnAN5XDo8B4EZkct26LxWKxJEcSM4RDiMg04Ezgd66PjgU21L3eWH1vs0cZ84B5AKNHj575hje8IckmWiyhefxx/WczZ2bXDovFhMcff3ybUmpSlO8mphBEZAzwP8CVSqnd7o89vuKZM0Mp1Ql0AsyaNUstW7YsqSZaLJEQr7u3Sh63Z0cHdHZCXx+UyzBvHixcmH07LMVERNZF/W4iUUYi0oyjDBYrpe7xOGUjcFzd6ynApiTqtljSplwO936adHTAokWOMgDn76JFzvsWS1ySiDIS4FZglVLq3zWn3Q98pBpt9GZgl1JqkLnIYiki8+aFez9NOjvDvW+xhCEJk9F5wGXAChF5qvreVUAbgFLqJuAh4CJgNdAFfCyBei2WTKiZY4pgpqnNDEzft1jCIEVOf219CBbLQJqavIV/uQy9vdm3x1I8RORxpdSsKN+1K5UtlgaiSOYry9DDKgSLpYFYuBCmTx/43vTpNsrIkgxWIVgsDURHB6xcOfC9lSttlJElGaxCsFgaCBtlZEkTqxAslgbCRhlZ0sQqBIulgSjSIjnL0MMqBIulgbBRRpY0STS5ncViSZciLZKzDD3sDMGSOR0dzgIrEedvVhEyedWbNAsXOovQlHL+WmVgSQqrECy+JC1E80rOZpPCWSzBWIUwxEhSgKchRPMKm8yq3qEyC7EMT6xCGEJEFeA6IaYTlosWRRd6SYRNRhG6WYRr2lmIpeFRShX2mDlzprKY41iV9Ue5rFR7+8DvtLd7n6t7X3e4y9VRLuvbZoJfe9Os14Qs6rBYggCWqYgy184QGhCvEbLpKNk9YvUzpYSJbTc1vcQNmwxj+qnvp0olXr0m2EVjloYnqibJ4rAzhMHoRsgi5qP5+hFr0Kg/zCwhzDXURtNesxY/TOsPanvYek2wMwRLEcDOEIYPuhGyCrGtRf2I1W/l68KF0N7ef065rN9fOMxsIk7YpGn9un4ql9ML1zz5ZO/37aIx62xvFKxCaDCSMD/UhGdHh768mhBzC+/58/3PT5OODr3ic9eftfnGKwsp2NTUYJ3tjYTdMa3B0O2YFYb2dufvokWDPxNxhL6fEOvoyGelrN+1u2/jrHcWszuZ6bF9ky2575gmIreJyBYReUbz+TtEZJeIPFU9vpBEvcMR3Ui8vX2weUf33sKFepNKqRQs3PNaKRtGEWad88c6lPXYvmkckspl9N/A14Fv+pzzK6XUexKqb9gSlMvGSzh7vdeID2m5rB9pusk650+Ytg03bN80DonMEJRSvwR2JFGWJZgkRuiNkka53hkZ5O9wk+VMxmYh1WP7pnHI0ql8rog8LSIPi8ipGdZr8SDphzSNKBK3M7JGLdKo3gSWN14RWUVpW9549c306c7szUYdFYyo8aruA5gGPKP57DBgTPX/i4AXfMqZBywDlrW1tSUfpGs5RJz1AO5y4qxe1mHj+ocmad0vFgeKvg5BKbVbKbW3+v9DQLOITNSc26mUmqWUmjVp0qQsmjdsScqkklbiuCz9HH4zHNPZj421N8PuC11cMlEIInK0iDPRF5Gzq/Vuz6LuRqDRBUnWDuqk/Rx+cfKmMfQ21t4ck/ul0Z+JhiXq1KL+AJYAm4EeYCPwCWA+ML/6+aeBZ4GngUeBt5iUOxxSV6Q1fU7KHGRCGqYdv9QTWaacML02a94yJ6ivrEkpHsQwGSWiENI6hoNCSCL7p1vwZ/1ApVGfrl9EBtcdV/GFydeky51kep4l+H6xyjUeViE0MHEESdjkc2k+UEnPSEz6JSlFFHWGUH/NefS5CVnOFMPg1648lWtR+ysMViE0MH4CJWqO/6EwWjUZJSY1kvRTLLrPpk836+/p05Ptl6Suq8jkNUNo1P5yYxVCwfEbdQSN8v1uxrDKQPdAFXFUZPJwJqn4gn4j92dhlHFe/dmoppcsBXMjzPLCYhVCgTG5uf2Ugt/N6GdnN32g4j58aSqToLLTcmabXE8SijhtklSYWZPFICWMybWRsAqhwJgKrSg3Y5Cpw+SBCiNU3WXqzCZZjYjDKrOgPglTXlhzXR4UYYaQlWCPUofpb2hnCAU5hoJCMBUSUR/euA+cafvCOrDTFADudplcv4mwD6scw/RDHuRtE8+i/jh1mP5+RTChhsEqhAJjKmTyenjjxtnHeaCKtlbCVDnq2p/3jMmkjWHaEvf3yWKGYmL/Dzs7znJAkwZWIRSYMII+D+euafuiKgOdAMhaAZoI+yQEWBEd9FFI4vcJq2CjEGdQkvcMKi2sQsgZE9u0eyRZJKGhW9xW/14cheAlALK2b5vUN1QFhClJR9wUYYZQO9wLGmsMFQVej1UIORLFsVl0oRNlwVtYIZLF6NHkmoKUd5F+lzRJI+ImTx9CVvdVEbEKIUfCjoKKEPkRRJgZQdQRdlb94B711kJyh5OwNyGtiJuso4ysQoinELLcIGdIEjbTZyNsXRl172LTTWI6OqBSCS4vLl4b7CjltCnLvaAbAdPfPOzvk8WudfV1+GGzpxoQVZNkcRRhhpD04qhGnyHEHe35TfGTHj0G9XWei+qKRpoRN1n2s2lKkTTut6KANRmlQ5xVxmF9CEUSHGnafrNUiEHCIO416gRdI/iJ3KTV5jT7Qle2O2hDt3K/SIOwJLEKISXCrCEIMwIKsnkWQXCkNarzE9JJ4/f7xVVMfoKuEWaBXqTxm6fZF6ZlZ3nPFYE4CkGc7xeTWbNmqWXLluVWf20zdy+S6LamJm/bbbns2ESHIllec82H4Ka93fv9Gia/rd91+NnjC/y4pUJaz5Dut60vu6PD2ZZT93sM1edMRB5XSs2K8l3rVPZBt1VjUls4NoKDOWl0Tskknck1/JzccX9b/W9XQUbujVX2UCKNZyhIGZTL3gEFbtK45xodqxB8SFt4pa1wGgER70ikpNBFucT9bbW/0bj1qIvmgwweeg5HAZTGM9TZGVyn3zm66DcL1ocQRNJ21frywqSpHgoUzdkaN8/PoGtp3qu4ZI7iWhSzP5Zp/H1RAhK8SLqdQcECQecMdcjbqQzcBmwBntF8LsDXgNXAcuAsk3KLoBCSJGhVZZEf6iRoVGerjvZ2pRi3RkGf87emDKpH6nV79KXI0L6HlDK7j4bavRaGOAohKZPRfwMX+Hx+IXBi9ZgH+FgAhy66aWy5PNikYUJHR/KLbdIos8ZQ85ksXAj8zfFwbdn5e/qSQ5+VJV27n+5eUsqxnQ/lhVcmZqgwpqo07/lGoymJQpRSvxSRaT6nzAa+WdVej4rIeBGZrJTanET9SVEflVAuOzdPknbGJAWi27HW19f/Omqb0yizHl0ETl4+E1Wp0LttDd3rH6Nn1wb+49nF3LhjBa+hmCjwf0YdxvvHjqa3tJ9KqRclFZQoVEkhCn6wp4dmoMdVbhn4p9HHsOk7l9PUMpHm0UfTNK6Nlmlvpnni1ETaHnTPdHYOXRt57br8nlWTcyD9e77RSCzstKoQHlRKnebx2YPAvyilfl19vRT4rFJqUEypiMzDmUXQ1tY2c926dYm0Lwi/EMWkbowkQy7TCN9MOyQ0iz52o3oPsv/5pezf8Bjdrz3L/r0v0N33Mt2jdrF/4kEAWraU+NHWCl/cD9113x3VB//vBfiLNVA6CFIB+qp/gdPeDRtHD67z8F54bIfQO65E7xihd4yi5zBF95EVpA9adoygZf94RpWn0DLmRFomnMao485h1El/jJTNxmi632rAtQ+zENcoDMXQ7zhhp1kphB8AX3YphH9USj3uV2aW6xCyuDGSFIhpxHf7lQnJzJrSnIVVerrpeuYH7H3xEfZs/y17mtawb3IXzbuFUZvKjNqsaNnYR8tmaNkMozZD0x7HwTXtSlg3fnCZU3fC2v/0rq90DSiPPhMFlesGv6+AnsOgezJ0HwP7j4buKWW6jxG6JvfRO0YxZvNoxlZex5gJ5zL2xItonX4B0jRiUFkmoZdZCLS0Z9Vpk/ZaozyIoxASMRkZsBE4ru71FGBTRnUbkYV923Qaa0Ia5pegRVVJTKcXLkxOYPS+tpGdv13EaxvuY/eIF9k3uZuR24SxL5YZs7KXiX+AMS9A8z4F+EvH9ePCvQ/QtstbibTt8j5fgBG7neOw52vv9nd4z1jYc+I+9p60nO3TV7L2QCcHX1aM2TSKw/pO5PC29zHu3Pk0jTv6UB/qlEIWIa5Dwdxi+hw1uuIzJqo32n0A09BHGb0beBjnmXgz8JhJmVlGGaUdlZBG+GpQCGeUlBp+UVB5R2r0de9TO3+5UL1069vU47e2ql88hHryP8tq7YdQr52O6mk1aLzmmHolAyKEasfUK/XfuWsGqvWqgee3XuW8H7Ud7uPgaNSOM1BrPyzqia+V1S8eQj1x82i15vZ3ql3/e4uq9BzILfw0y0ietK4xjXxleUMBwk6XAJtx/GsbgU8A84H51c8FuBF4EVgBzDIpN0uFkEcSriTXNLgfEr/EX6Zl+h1ZcXDrWrXpu59Qy2+aqH75AOr3t5bU6vaS2j4L1TsyOcEbVbjfNcNRGnKN89dEGUT5Tu3oaUFtOxv1wqdL6rHbS+pX96NWLDpSbf6f+arntZez+2FUdvdH2gI56YzGeRNHIdhcRnWkNS009U8kWX+Q09Gk/Lwcbn17trH9pwvY8spiXpuylcOXl5n00z4OXwYjdqZX7+IZcPX5jpmobRcsWApzVyRfx7w/h646t0DrQeh8IFpdB46A194EW99ZZuepfRyx4WiOPPYyjjj/KsqtHvasBMnq/tDVI6LfVyNJGs3PEMeHkMgMIa1jqCxMMxlJJT0KMh0c+5UfpU1Rp/Z9+/eorQ/9s3p20RT1ywdQT/1bWW36M1TPaPPRcyMcUUxTpsfBsaiX34168oay+tX9qJWLpqptP/y/qu/g/nA3jyFZmVKi3r+1NsY1NQ2nGUKkL2V1DBWFYHJDmSiNJOoMe1OHeaCiCIgDG1eoNbe+Xf3mf0Q9fmNZbXwv6sD4+MKxqIdc460Q5Jpk6+k+ArXh/ahlN5XU/363pNbe/i518JU/GN07YX/zKOfW339BQjpowya/+qIokbTKyQqrEAqOyQ2VtELIY/PxMCOp3Y/dpVYumqp+dT/quX8sqz3TkhWIRT3SnCHojt2vR626ypk1PLfodWrPk/f03ycPtqvydWXFtajydWU1/dLvpCL8gu7HKAMN3f0b5AeLMrJvlLxRSikVRyFYH0JGBPkH0rBTnnoqrFzpf06S9t6ga6j0dLPt4at5ees36B7bxbH3weQHFc27k6m/yNT8E+vGOREW9esX4vgQwnBwPGy6WNh0saL1tcP4XvfrWbD9iYEn/fs62N026Lte90kYn5eJT8vvPtTdW15+OL/1GTUKLPZiU4iFaWkwlBRCEKWS900a13EWtElINiuxFZvv/Sxr9vwHTbsrTPl2hYm/glIGDsEoJO1c9nIkiwIFTE3Jee1HpQm2vgPOeDu84v4Nru1DlxW//v4Mu8gyaNGju3w3pvWZrOBu5FXIJtgNcoYA8+eHe9+U+v0AdJvFJIXXYqimUg9f+OhHWbv133jdDb2c2V7hyF/4K4PFM5yVw6VrnL+LZyTXxiBqwnvdeGcUv2688zpOG64+f6AyAKfsqbucVdBZKoPFM+CET8Pk8+AVL8E5br3n99wLtXTJ9fwSOPoR9LnfZkf1mCwkNVm0N1wT3lmFUBBMb/i4dXhtFpNk+f3XoDhq/Hr+af5HuWLtN5n1yQoTHnPMJX6kIZDD4CW8u0Y475vgpcyirIJOA3ffev4Y518FTV2D3nYL0bAr+4OEsImQNrl//RSL6TPl3m2ttgLbrRSGotKwJqMhTtZL7g9sXM6LD76HnZM2MPVOmPwDKIWYnkfJKZQkYfMT1aNbYzCqB7Z7JMHL6ppqbbv8vdAXMBIfVYELNs3hN9/5Klt2H0O5LJ73TJQ1CF7my6TvySTyhZlcWx6JGk2xJqNhjN8oxXSkkwSqr5dNS+aybNkZtDz1Mud8GI69L5wygPxH07o8RLr369HNLsBRDANQVQfzNTDxH9KdAdUUla8yUDBhH9x8L3xnyRJ+/dE2fn1PmY3fvoIbvz7Yvhdla8z6EX7tSHqmGnem3dFhNvsJazJrFKxCaGCCBH5WN23Xc0t56r8P55Vd3+aNf6s44RsVyt3e5wb5B+II5DD16FiwdLDwbj3ovB+ETmntaHWiiKbuxPEkKxxzTfXYPho+Njs9peClqAYhMKbH8Wc07YMTv1ZhxucUG3fdwvJbj6D7pd8OOD0LE2dUoppGTTLI1hhqmz3VsCajBiZoapv2knvV18vLSz7I2rH3MvUuYco96tBeAV6YpG1IIrVD3DKiRhmZmLt054AzQt/21eB6/PBq+2WXeJvB3HiZxSol2PChEhsvrXBC90c4+gO3I6WhOY4MilCqV3hF3kfBmoyGKUGjFJ2DLYkdyg6+8jzLb5vIlt33cdan4bjv+SsDMHPYzl3RP5oW5fwNG6Mf1zE8d4UjwCvXhYsC8ppdSNU0FORgBtjeGm+WoHPIHzHYR+yJ1yysVIGpd1V445Xw8u67eOaWo+jd4R2J1OiYKgMYult0WoXQwOgEe21mEMXOa8Ke3y/miV9OZ8wTezjzryu0bjT7nql/IKpADltP0tQrM1R1rUHVLGQknMVcaXkR1odRT5BZbMxaOKujQsuqHTz+w9exb/n9Az5vJKGnw28A5TY7mZrMsvTjJYFVCA2MTrAr5dxwadh5X/nup1i+6cOccGOF191UCZwV1JOUf6Ao9cBgXwU4SmzqrsFmmppwHuFjUoijtEx8GLVZV/tj4WdhpV448T8rTL2jl6fWzGbL/X8H5Cf0klZCYQdQJr6KRnM+Wx9Cg2O6pD8ulQNdvPjfb+JbT53BrTcvYGNfG22sZwFXMZclRmUknfq5yPXo7Pai4M574CPvc+zzbuKEomYZsrvnRHjm/wpH7p7J6+c9hteihjTt6WmFfSYdpp1H6mzrQ7AMIsloh97XNvL0XUfxvZ+ewb/e1MmGvmkoSqxjGvO4mcXMMSonCf9Akerx81X4zVLmroBv3hs9mklHnAipsIx9AWbOU3z7Fydpzwm6B+OM8NMaeSe9eDNNP14aWIXQ4PjdWElMpXu2ruHp+05i9MoubvrOAroYuMKqi9FczZeMy4vrHwhbz533OK8vuyReGoywK5CDhHMaSisrRVhjxG7oXLwA3fpzv3szrpmpUcI+0/LjpYU1GTU4ptkdo0yle159gacfmcG4J3p4/dcqlOlDeYwhhAoVijfkScp0FHUFchY7sOVNSXNPgP89FzVsMyhZYxHCPt1ENUNF/V7uJiMRuUBEnheR1SLyOY/PPyoiW0XkqerxySTq9WMoRD2Y4HYc6wg7lT646Vme+tFpHP47RxkI0IZ3uKHu/byJG34aVA4EzwKymA3lyRFs137mJ7yijPDdswovijjyjmKGystRH1shiEgZuBG4EJgOzBGR6R6n3q2UOqN63BK3Xj8aLdQrLvU3nI4wU+mDLz/DUz87k4m/7OWERZVDBoEFXEUr+wac28o+FnBVqPZmlc00avipu33rNOdvb3VmCRP2ZWOiaSTKJf8bLopt3W9QU6SV0kmQV3RSEjOEs4HVSqmXlFIHgW8DsxMoNzKNFuqVJHGdWH1dO1nxo7OZ+PNejr+lMsA6PJcldPIpprIWocJU1tLJp4yjjCBaNtOoCiRK+KlX+7SBItW0E/ubHV/FUJ0F+LGDCZ7vVypC5YB+0UUU27rfoCaN7L15kpePJAmFcCywoe71xup7bt4vIstF5HsicpyuMBGZJyLLRGTZ1q1bIzWoURxOaRDHiaUqFf7wzTfSsvEgx9/iPd2YyxLWcjwVyqzl+FDKAMKbceKkw44SdaPbu0B8Zl9RzFBB5LknRBh05sKjx2/gxf9+04D36s24nZ0wfXq4NTKNFrETh7yuNQmF4DWAcj8+DwDTlFKnAz8B7tAVppTqVErNUkrNmjRpUqQGDacbx02cxWgb7nwP+5o28IYFfYH7FkQlrBknjh8gStSNrh2KuuR0Ib4Xhbz3hAiDzoz4la7P81rrKjYtmQt4m3FXrnQGKqa29bgRO43kV8wrOikJhbARqB/xTwE21Z+glNqulDpQfXkzMDOBerU0SqhXWjeo24kFwfVsf+Q6No55mNOuUpQPDP7cjzCj2bBmnLhpKNyOXYiWbbW2u9nUDFZBJ+UMzwKdGfGyg0s47fOKNS3fYucv/isRM26cwY6JX7FICkN3rdDfxtqRZFuTUAi/B04UkeNFZATwl8CARCciMrnu5cXAqgTq1VLk1Lw1snJ8m9TT9dxSnuu+jlOvgZaQVrqwo9mwZpwk01CYtHXBUmh2hS029/a3L4vFX3nvCREWnRmx9WU4ZQGs3P4Z+vq8p1ZhzbhRF44FKaQiBqJ4Dey8oqySbGsi6xBE5CLgP4EycJtSaoGIXA8sU0rdLyJfxlEEvcAOoF0p9VxQuUN5HUJW6XOD6lGVCk/fNp4JP9nLcXeb3Qv18fWlivfGK37pEsLE5yeZhsIktcPiGfDx2XCwqe6EardMqPpIt7dCuQJ9JWfGkPT6grApKIq+3mHNx4Rz713HqzsHuw6zWjcQlEKiyOmsawSl5+5Pex99HYJdmJYTWeU4Capn890fZdOuOzmzveK78X0NLwHtXbgjLJMQUkkJPJPtMf32K3ATpJiittuzj5WjkG54ZGAZHRfCTWcPvC5RMP8xWPiw2XWkTaUZPvDWr3PPzzpwuxyzmrnnvXdIFNwL00xmU0rFUwhNwadY0kD3Ayft+NbeSGM3Itcex1Ej4Es/hZmGWUuNdt/CeexrgrVmmoFognzuimRGvG27vIX9EV2OIlg/Tusz9qRrhLNPca2N9biFepg+qH3+mQuc2Uj9zmr1ZSyeMVgZgPP6prPhvI3x+i0pRbykZw4P//yjuJXB9OnZmXHnzfNe0V/zK2b1PJrizkBgogySaKvNZZQTWTm+Pctr2gfv+kcQeLUH/voc8wgWEzv2oX0A6uga4Qg4U+dz0mGXi2fA3mYGSfzmXtgzst+vEDa8qq/s7TNJYpOeMT2D21NfxtXn63dCUzH3Vkgy0ulqvsR+NTjHx/PPR29fWHQr+js7HeFbtECUKGumkmirVQg5EcXxHSUKYtCDMHYjXPwpOL1//UAYQaVz5pb7+kM7daPs7a1mAibpsMtaedtH0y9gleP/6Cm7/AUR8Oq/JBzDQWUElRXHCZ1kpNN62jzfz3pd0MKFg4VmzSELxQpECfIVuF8n1VbrQ2gQksr/Ltd5Dym99tP1wsTJG8YO7+Uo1X1/wj5n1JzUXse+VB8LIdp+xFH2JnCbaPY2+yfPC7our7pMzUAm/hZTprGGdUwb9H4eDttGcB5DvHbmntzOkj5R4rg7ftBB0/VNyHVC6bqSVhmAeRinyWIvr9BMHV6jWN3I1nSGYVqeHxO6QF3npKSoXeuEfWh3iHP3X9jwVK9Z0Z6Rg0Ng68vw62evusLMvJIM9/VavFaW3lzMMWlnMUhqLUMjL0yzZEDYG7njBx0sWraIPuWcoHzcpWHj6IOyeLr3FvbDS8BohY6PPT1sHUHsHukIyvpr3fZVuNNwY5uwq6S9TDQHm+CwA/oy3P1c7nP+6uoKYwYKo9CC/D3uxWvHtq7lmjlX5mKOSTOLQZJrGfJaS2VNRimQ9DZ8EH4K2XR90yFloEWlE0dvsk4BHAFz+ZPw0MkDTRjgHXbp5fA1MWEYh8q60JlcDkX/4B0KGpbFM+DDlxD5+kwJawYyMS9FWSfS2wqPfgtmnvIzRp30jsB2J/k8pbX1JhTHHDWsTUZFWW5e3440VjyGnUIGKgMc+ZN0hk63WaKvzOBZQtX8cvmTcMeZg00YMHh0XVsU5sZk9O8erU/YN9gU44Xb1DTIOS1OptPaZ9OuBLkGmv7Z+WsSHVUrUxfdlGRKjLBmIJP9HKI4n5u64JiHSmz82V8FtjnpFcRJj7zrn/uhkFSzoWcIaWr7MJjsWpbEKCHMSMlkhpDG5us6R2e5z9lUvn6kGcbxmuSK5Vp5QbOYCfscM1HQtU3Y5ygGrxlIrY3gPdr2cwzHuT4vku5DiO583n8MPPF14S0XH0TK+jCvooy6vTDdrdDOEDKiKPsemNSXxCghTB6XeTP9vU9pbb6uc+BWSoNHmmFCM6NkLvWzbdePfu/4PozweGBrfoSga9veqjdHdY2AK96td+ZqHd4KuprgN1P01xeWNPZcjup8HrUJmncLux+9zfe8Io+6TeVM0ZJq+tHQCqEoN0tWqwi90JnMFr57Ie2z2imLU7EgjCX9nb3CCIg0TBg1wkTUzF0BYz0yvPY09Zs+Fs/AKLWHF/tG6s0qvg70Eiw620lPERadMkx6W884yf4m/kaxbaX/5olFTmUf9NznvZYhCg2tEIpys5jUl8Yowc++2vGDDjof76RP9VGWMvNnXsEjpwl7bk93Z68wAmLB0sEj8xG9ycxcwtq2a05iN+vH9SsXL7NS60EY0RPQGI1/YP246rX6WW0FOkNO/rPcTyHOrGPirxXbW5/0PadoK4jr8ZM/YbOxFnByFhsAACAASURBVIWGVghFuVn86ktzlKCbst50294BIad9qo/OZTfx4y2KVu8NrhIjrIBwy8JecaJ44qasCGOOWjxDn7GibZfTHi+TULkPzl0PB5ujtbFtl9MvOod5jb6QT2nW+ymYzDoWM4dprKFEH9NYw2LmMPY56G3ppWvVj33Ldyeey2rUHRSwUhT5kyQNndyudlMkHeLZKO3QTVnVgcHD3T6Bb7wC16TbJMA8Gd3V5ztmmXoq5f7VuXGS4ukS2XmZaLQ5gRRc9LxjtvGiUoKfH49R/iN3fqf6WdMNj/iHxZZDmqqKtp/CYuYwj5vpwvlh1zGNedwMCs585rvs5ru0nvKuQd8zddqmgVdyudrr2nNdFPmTJA09Q4DoG2Zk2Y4kQ2Pry9Iyznsa8Epa+2JGxERARR3ZhjFd+bVj0ZvwDQk1Gr0reOeLwQvMRnfjGaI7L+RSnCRXGUfB7b/4TPOXDimDGl2M5mq+xKi1vXTtfMaznDyDRkzrLor8SYpCK4THH89/fUFcosRR6xSIuywt77je8+1ju8O3P01MBVSUkW0Y01WQY9cTVc2eaoLA6onBq7v3/gu0P9a/4rjc57wOu69BFru66fDyX2zv8U5ut542WtdBV8+Lnp/nGTRSlICVGlmtt2oIk5HXdK1R8BtpeF2L31Q1aGRUm7Ly7hYWuUaVIwWu/1/jZmfCgqVmK4ijjmxrQrcW/1+babiF8YKl+pXCWqr7E5huoLBunNnK34UPx9/Yxn3dae6i5pWQb9DvOW497Jo26LttrKd1A+wf/Zpn2XnuUVCk/RFMzFdJUeiFaSKzFPRLtiIsRoFwC8TC7sTktxDHb3TiLqs+yqhZynz2hD6uv1yfnC1L6oXIEVWH6o5W5/9dLdBb99CN6IXb7gsOMfUSfmEWYsk1hN4LIRQKRvQNTLWd985mcTfA0e3sNqgfl8+BB26Gnn6zUSv76ORT/GXLEn7zfXjb+QeQpoGaJM+Fp0VZ9ArhF+flvjBNRC4QkedFZLWIfM7j85Eicnf189+JyLQo9RRhMUpYE1DY0Fi/qWqYsha+eyG9X+hFXaN47ZL7eU9JMlUGujh4t0lh+2hnle+d9zjOVXEptqDhil+IZVC0zeIZMPEfqsogbWTwvgu1nc2S3CjI9PwkQlM9d8/zUqqnL2HC+f3J7aaylk4+xVyWUO6G8j44+PLyQV/LK8Fb2LrTNudkab6KrRBEpAzcCFwITAfmiMh012mfAF5TSr0e+A/gK1HqKsJilLCOrrChaX5CP2qYW6V7D6UDyQ1/g4ROVCHtFXVUvzjMC7/y/KJtFs+Aj80emJcoFhEn2n47mwUJbffv0HGhuZBPIjTVb5V1Pa0H4YaXl7CW46lQZi3HM5f+DZp+/NMPMeb40xFxhGqpVAyfoYnD2GSAGFdhZLneKokZwtnAaqXUS0qpg8C3gdmuc2YDd1T//x5wvohvnIwnRYjvDautw45y/IR+1BGT6tlLqScZhWAysowqpKOES/p9xy/axkv55IXuGvz60et3uOlscyGfRGiqrn8ndA105l/+pNMGrwHEYubwxZtupVf1N1wpR6ieemo6iSKTJGiAmERyvizXOyShEI4FNtS93lh9z/McpVQvsAuY4FWYiMwTkWUisgy2AsVaAh5FW4cJTQsS+lHC3JL0E5mMLKMK6SjhkrrPShVnDYEu2sZX8EXprgB929yrL1d3DX796PU76HZ28yrHr99MzVO6aKYbHumPqFqw1JXRdv0cPrx6C0IFocJl3ElfX4tn+StXetebda4yP4IGiEmEzmZpOktCIWiW9IQ+x3lTqU6l1Cyl1KyZMycVLr43C22ddGxzacRoKk2DuzusfRr0QmrduP4y/AS7X0jkRc/jGYd/kc9m7Lpdw/rKjiC6/MmBG8h0NTvCtObM9iQp65ri0IY1t9/nhJC6fSR+4aB+/Rh3JO/Zb8rpN1Ofgklo7wDFtXwO3Hc77J9EzU6nCG/3KIIvsUbQADEp+39W6x2SUAgbgePqXk8BNunOEZEmYBywI4G6MydPR1dUSiNGU3GNJqM6Ff1i9mtl+I3M/YTIQyczWBhL9X0NtfLKHg9Y1wjnuwuWQmtPNRdRtZ27RxLZ7m9KbRvO2rqDhQ8P3JIzyJzipzx1v4OpwnH/DuU+Iu1IF5S2YoDiWvol6BvpX6ABRfAl1ggaIBYl35opSSiE3wMnisjxIjIC+Evgftc59wOXV/+/FPipKnK8awBFWp1o4rAaedxZHJhQGWBSiOpUDNovuSaE/UaOOiES1a49d4WTRkL3Xa9r7Wni0Ag+Lba3Dp6B1V/7IHOKSyn7KU+dspj/mHkeqfq2+PVfHAYorl3eC9R0THeHplQpgi+xRtAAsdHyHcVWCFWfwKeBHwKrgO8opZ4VketF5OLqabcCE0RkNfC3wKDQ1KQoyg5qWWDqsGo6fApN+4UDE/vfiyN8g/ZLXj8uWprlIB+Cn4krknmlRP+oOCXFENX5XkPXj7XfYcI+Dim2UT1w3sZo6a3TSnexYGndrEWTUsWNiCNUn302v9l4GDniN0D0UhjTpzs+hCLKqETWISilHlJKnaSUep1SakH1vS8ope6v/t+tlPqAUur1SqmzlVIvJVGvm6S324vbFr8bKgnFFcZhNWpHK111hr04AqAmpKYmLET8TCRBJq4o5pUBaPwGI3ycwWHLDON8N2V/M4fCZrePhssuibZ/QlrpLuauqOu+86+C8uCNJ8qlA7S3OwJVKahU4gVRxCWNbTtr1zBvnuMsL4KM8qLQuYzCUpQd1IJuqKRuuDAOq9beY9hfN2NPQgBEKcMrdr72+urz+53AbpNH0Gg6yLwSSagrZ5V0ktQL+7ijcl2kUdBiNy/S2E2txqGBw+lLYPbHYNRWatOa8c1b+cIHri+UDy5NOVIUGaWj0KkrgvZUdhM2TURaBC01T2qf2DDlbLhrNt3LHuTEG/qXK8dNXRC2DJNUB6Kct6a6yoq6d2+Nif/Qn1bblNrezk3/7L05ThTq94vuuLCaWrv+upR5Qjtdn7jryRu/FCJnXdzEuNM/xDEfvENfQMakKUeykFG5p64oCkl79KOadYJG7kmFooVxWI2d8k52nTbwPbd9GsKHoYbxFZikOlBV84fbJBR3NH3DI/7OcK/VtbWZTtgNanSIGjh7+s5phI6qqsfv2nVmpyihxnHRzT4+tAJ2T+9j7AkXpN+IEPjJi7g2/ygyKku/6JBSCEl69OOYdYJ+9KQUV5gQ2MPOu4LuSRW6J3mXlcW2i2EjVupNQrq4+b3N5oqrXihN2OccNQHVronOWTwj/AY1OhT9CnPxDP9tO914CfIBDlsXXsrC7zdOW1F4DRy6pkKlWRhz1l+EKqteQNaOJAWln7yIa/MPK6Oy9osOKZMRhMtE6kccs05QpsS4mRSjXuOqm6Zy2A/Xc+z3B3827UrvHcaSND3o6vCj3iS0eIazneX2VgaMrE0yokbB08QVg/q+9DNhufvcz+TymymOz8C9G1vnA87/h/oLKCnv8NIJ+xzntElW2CRZN1c48LZTOOmKZ42/E7SLWtwopPpny484mZfDPL9R5JA1GdWRVFRCHLOOSfqJqOF0piMGr2nmxGM/yLa3eU9Dsth20W9Eq6N+pHtIOLnMLAebHMEXlfqsp3KN839N+fgqgxDrGOrNRX6zA9Rgp7yfQ91rsVtNGbiT9+nWGmxvzXYP5kP1vrXExNddFuo7Qc7XOM5Z4w2oiLda2ktG6cxCWW/UM+RmCEmRlOM3Lu7RhO5GqG+XbhT16U++xgffewTn/gU07Rv4WRYzBNA4UqFfsHqMdOtHqdp9CxTcdU80p/jHZw9OTV3qqwrQgDQWE/YNnrF4opxVy+A/U5qwD7Z91dWWCA71ULMxrz0MAsoPIijY4MDh8Ps74C3v2kOpZYxxuSYpMaOItLD7NycpB/wsBrrZSq1+r9nGokV2hpA4RVhh6DUb0FH/mW6UtOj2wxm/cSJb/njwZ6YhpHFtzQsfdlI6DEIGZ8kMZbKQaD6Pq88frAwAKtU0F0HsaNWvx3A171CI7TqftNE3PDL47SiJ6ELN7Hz2jI6CiT/q1T8VJrx8XChlAMF+tigBJGGVASQjB2qzAl3dnZ3+ckhnLYBp4ZaE12EVgoYi5CwKM/2tfxD8ppnHveELbJgjKNcvbxKHnpTjeYfGXLKjNThiyVOZVIli5kgiNUNQOg/oXx+wbjxaATyhy/uaoySii7vC2B0RFYagNSOVZth4KUyZFX5blCBBHEVQmzxnQXIgbCSQiXmqr89fDunbPUETOhKMVQg+5J2zKIydsP5B8ItiGve2v6K5u4Wtbxv8eVAIaZj8R1HTTARxwyPVVNIatKNvTbt8s54GoEvYp/Mr6NYMgOMY95odQLREdJEX49XaSnSHcpA/6tU/gdGvjWPsrDmhy3YLyBpxBmxBz1l7u78c0I3UTz1VryTCKCGdHCrkjmmW9NAJdhH/EYvfNFNKJdqO/hvWf7gUWl74pb6uJ06aiSDmrnBSSesa7xci6tWu7a3eZZX6qmkr6hjROzBUVZewz8SE5Ka+CV7K1CQRXX0K8rkrnFBa97V5XZcXumswMRn6KXxVgvVzhLbX/3NwIzTUC8jaEWfA5mdmMlEyOuHul6LCRJgHzXaKumOaJSV0N8T8+f4jFq9RlIhzQzY1wRceuJ7KqDKvnRWuPdqUy4RL2ua1JmBUj5OHx8Qv4Tdy9VtEpl0YJ/RHDFXb883vO6Gs9Sa02+5zHL5Bi/C8FF5QdFVtq1ATs1xQCvLLLnGc7w+dPHh9Rf116ZSqzlxkajL0U/jbzoOm3pGM/6Mr/TskQ3TPmemMI8xIvaY8/IS56WxHrzC2bzVv0UCsQigwcfwYtVFUe7vzuhZ50dcHi24q8+D9i1hzRWmQL8EPXdioe19gkxDW2oj3znucGPjto8P5JXQjWL/Rua+/QJzvquscoT93RbSMreDtj5n/WLCfQZeq28sc5FdW/WrvO850zq+/htp16SxYOnORqclQ54+aswrWflyYeuTfIKXiiJ6wz5nbXxCGmvLwU0Kmsx1du2GtWVpZD2zY6RBHHz6r+P03DmPSw/uY8j/m94Au7LM+RDFMCGvUcFe/xVo6wR0UihknzNKEWijmunF49uHUnY5SMAkxDSrLXa5XX4bt+7j5pNZ9pMSu88Yz45NbC6UQwhAlIqked3h4Eoto3diFaRYt+ogj4eQ3f4e1lym6jzQvzyTldRgfQRL7MpiGqQaNrONG5gRRG5l7baVZS8Ohc3K72xaUgrweXV9e9Lz5DmtebfB73+1ruP2PYMMlFU76kwcbVhmA3l/g9uuZbO6Td9CKF437y1iM8Is4aj31Qqbs+GP+8PfmDmYTYe8lrHVbRSaxL4OpSWfQpjI+15AWi2c4ZpxBI+3qfga7Rw52+vq1zSTkVSew3e0Q5fxOYXwjurUqbl9Dxx/DT1ZPo+WEc/0bW3B0A6yaUK/9zXNznzhYhWBIo+7EFrTArm3O/RyYXPZcrOaF6cjcdKvItDZm8Wv/tq86q5qjOrXj4OnYrqOnCcYeCLcNZv0OdqYjft1eCrpMqzUTVVdzNezVp21eZXcDC/Zt0Fx14xAmMWURZwBBNIxCyFMgF2kntrAEOcxKLWM4+XWLWP3X0HWMWZlhR+Z+zsiwpp+ksnLGdWpHxWQhnMkCvXpq16Ku0+c1cvdZGFNd/YgfcRbEtfbo977Qld2rUkrAkyFpZzDIe+AZy6ksIkcAdwPTgLXAB5VSr3mc1wfUbp31SqmL3ed4UXMqx80OGpei5DVKmnqn1hGtW/k/H76Sqxd/a1Ceo7jEdUbWiOJIDsLPsbpgafwNhEzrc9edVP4oXZ+N6vHOtpqE4193flnK9H6hgR+YKmk5g5OSc3k6lT8HLFVKnQgsrb72Yr9S6ozqYaQM6sl727msMw5mgXvWs6NrEl++7Ra++p4PHQpFTWo0ntQG7mFWSpvit9gujf0hgjK+juh1nL1J7U2g6zNIz/Gv82vMm5lhIrAUiWMK8psB5C3nIL5CmA3U9r67A3hvzPI8yVsgJ70TWxHwuskO9I7ihh99mZeuKCW6YU4SfoLFM/RpKcLkIzJNXVGuJJOmw83cFc6aBJ0Xv7lX72+Jgq5vdrSam+rCKvS5K+DaPjiquuCvLGXaZ7Wz8N0NYERPkSDTc95yDuKbjHYqpcbXvX5NKXW4x3m9wFNAL/AvSqnv+5Q5D5gH0NbWNnPdunW5m2zyNlmlgT6NsOK3327m0qf6eLll8KdRzRlx9m8O2qjGK2W0aTnNvU4Yf33G09aDjvPUJCV0VDOWXxpv3RqFKP2eRFrzsNe48X3Cpvc3cdZFL9F0+JTwjR6iZLXXeqomIxH5iYg843HMDlFPW7WBHwL+U0RepztRKdWplJqllJo1aZKTtC/vVNRFyHyaNPpZjzDjlHvY5KEMIHp20KirfiE4Mmf3SLMRtFc5XlE9lz+pX+vlnlFENWOFzXcUtd+TmJ2Fcfy/PFvYMEeY8aYfGSuDvB2pWRE0A8hbzgEELrxWSv2J7jMReVVEJiulNovIZGCLpoxN1b8vicjPgTOBF00bWRO8aThywrShkRWAG2cjDe/3R59+MaPuLdHF4ExxaS/e8iJIGNbyAAUpGV0521thW92of9qV+syke0b2J4/zK9Nvk/tD21q6ZgN+zt6o/V5rZ1zneC3lhR8bLxU2fkA4441LGXXSO4zKdc+++3P6D63nDfQbXNVnNYV85VxcH8L9wOXV/y8H7nOfICKHi8jI6v8TgfOAlWErasSY3iITNOu5fNYVlHJavOXGRBiuGxc8SzBNzuengA42DRz9h129+3HXtpbAoYR6nQ84KbCTXpcRZ3Zmyvo5wsZLS5xx1i+1ysBrJlAER2pWmMwA8pZzcRXCvwDvEpEXgHdVXyMis0Tkluo5pwDLRORp4Gc4PoTQCiFPhuqU1u/mW/juhVzxpnZGUUKAYzPYdF2HyWpckx3TTJPzBSmgeoURxiSj250NgTE9/aPwsCk5aiQVFRaWtR8psfnPy5xxzm9pef15nufoHKpFcKRmRSOYnm1yuwD8klm5p3RpxSfnQf21tDR389lPfZIPyxJOWFSh5LPnQFrUJ3MT9CadIGepSXK+ICe2uw5Th7luPYa7fh1+9aSxRiOIShOs/kyJnWeWeePbH2PkcWdoz9U5THU0+hqfPLHJ7QyIOsr3m7rWh4018mpmN+5r6e5p4fqF32TJ3o+x/N9LHAy5K1kSI1X3alxd2GaQv8EkOV/YnEemJhm/mUfQrCQoDDiNNRp+HJgAT32txMETJ3LWRS/5KgMIP+LP0pGaJUW3NgwLhWAirHU/lMmN3NlZHFtoEjecV5sVJa6/62YOa53F453C7pO8vxt2/UIU5TF3hZlg98LUxOOV8yiM+UZXt9duZc29wT4CncD/cDX/UhJrNEzZeRo8fhNMGPlHnPrJl42iifzW8hTdjJIUjTBoHBYmo6D4Xr91BjWzSRyy6uLklr7rP1MKtj7wj/yh76uc0AmTHx74eZi49zhmjrjfTTolhSkDooyACV2OIzmofj9zEzgKy+vzJNNgKODl9wnrPgJvaL2eCX/6T8bfHYprecKS1XqqOCajYaEQggSc3w+lC890nwf55ztK6oYL6g9HSSqOmrCevz3vKv5m6bdoruY/CpO3KM7mODV/QrnibJs5NWPBnjUmOZDcSiFJH8LBw+CFvyvTdXyZ0875oXFYaT1DyccWhSA5lFw91ofgS1DqCb9IB6/9id3Mmxd9UUmSNsWkIjZ0bT755Popr/Dq9ql84aFbWPDxD7H9Tc45YcIwo2yOEzbz5lDBJNJKkZx5q56tb4Vlt8HIo2dw1iWbIykDyD+kMm8aIQXOsFAIQcI66Ieqv5F19s4oIWVJ2xSTuuF01/L884PPPdA7in/vXMgf/r7EqqtLXPcb8zDMKEnvdLb0z1yg/85QwL3vgRdTdyW73uDAEfDs9WVe+qsypx65kNfPe5Ly6CPiFTqMcA/2TtbsNVEkB/qwMBmB/3Q1L/tm0jbFtK/Db8rbs/MV1t59Aa9Oepqnvgf/eiRsCLDRR/EDaG3pynEAu7+Xp78gLdIOMVUCm/4c1n4cJm89h6lzHrKKICS6Z3H6dGdglabZzPoQEiAP+2YaNsX666jVoVQy12SiwPY8dhfPr/oU0n2QE75R4fAn/csMK7D9bOnlPqiU+suBdAVnUsomSjlpKDoF7DgbXrqiRJkWTjrrLsac8b54hQ5T8kzIaRVCg5LmTZPGbMG0zI72PpbcsYtd+8dz1PgNfLHl83zilSWR6lzMHD7DDWxnIgBjytvYO/szcLp/eWE3gQndLq/Fa2pw1FCQ4M5jQZkXu6bDS/NL9Ewocfxhf8vEi76MlIaFRdmXqAPFrBzI3nVbp3JDkmZ2wzTWRZj4STo6YNFNZXbuPwJFiVd2TuWvdtzMVy6ZS1fITMiLmcPHuI3tTKKW/Gdv3yS473ZYPsf3u10j+kM73YSNzfdaK+GZgVUcBXTZJdBxodmajKwXlLnZNw1WfKnMymtLHH3UR5j1gT1Mes9XrDJA7+MzCQAx8ecVcZGanSHkzKmnwsq6zE7Tp8Ozz0Yry20u0pHmT66b9UwY/SrfX3w0E39XZsrdfYx5KbisaaxhHdM8P5PD1qL+9nj/AjR7C5T74I7vm+cH8hrB6/ZLONQ+5aTKDpqhJLW9aFj2nAQb/qLMa2dVaNv9bo659E7KreOHfWhoPSbpNnQz7qDZdJr+PjtDaFA6OgYqA3Be10YKYUYQ7tGMjrRD3HT1b993FOecu5qWE9/O8n8VnrqhzLbzOLRdpxfradN+pna3HQqxLGvqnNDlHarZVzbfhUw3gi8H5HNSYjZDSWp7URMqJdjyR/DEjWWe+WKJMa97F+e8fS3HfeSBQ8qg6Ctps8Q0S4EXQbPpomQ2cGNnCDkSZUGcbgRhmjysCJFTle69bH34s2zceRs9ow5w7D2KyQ9B076B3/GbIUxlLWtxZgh+dniAy9/rKIFBZRj4Evyimlp7/DfuMZmhZOFD6BkLm/9ceHk2tOxp5dhJVzDxwgWUmgfugpT3zoRFw/SZiiJC0/Qx2BlCg+K3kCzsCMJkZpBFmgATv0ipZQxHve9GZn5sP6e0dbLnncfy6BJ47rNlXjurf9awgKtopntQWSM4wAKuOvTaL2X03BXOyNgLE1+CbqQ+dZc+AV4NkxlKnHTXflRKsONNsOqqMr9bDPve1sZpx9/JmZ/Yy5EX/9sgZQDJLWwsom08Cia+vKgz7qIuUrMzhBzxG5H5PYRKDY5+0J2fx+guih36wLplvPqLa9iilnJw7AEm/aLEkT+u8OCqgVFGE9jGB7mbh3gP62mjjfUs4Crmoo86irOvsMkIvuNCuOls77QREG+GMqg9zOFqvuR57Upg12mw5V0ltr61QsvOFo5s/jOO+uMvMuLY0wLLTmKGMNRyFgX55aJeV1F9CCilCnvMnDlTDWXa25VyxPvAo71dqXLZ+7NyWf89XVmNxr5nH1Frbn27enRxk/rtElEvzhO1+0RURVB3MUe1snfANbayV93FHG0n3DUD1XoVimv7j9arnPdNOvGuGaipV6LkGuev1/f8zpFrBtZdO+Qawx+xVofm2m+ePEet7iip/71b1GPfbFZrbz9fdT3/M6XUwHupdu/o8LsfTfG7b4tCmD5J4ntZlVcDWKYiytzchb7fMdQVglL6m0L3cE6frpcZIuncYElcTxQqfX1q9++/rVZ3vkk9urisfn0vavK4tZ7XPpU1sYV6WsfUK70VwtQrQ5bDGs+PjpqwVr1487lq71PfH/RbhBXwcX8/v0soAkkovaJjFUKGpKXVTeryUwZFeOiSeth0fbz/xf9VUPFWhvQlLsjDHH4KJ+4MpfsI1Ja3o4S+UL+76Wg9yXu66DOEOO3L8tmPQ24KAfgA8CxQAWb5nHcB8DywGvicaflFUwh5jy50N3NRHrokhEFQH+vqOGrSGvWb74pa/i9Nas3lora+2RGkWSmDIIFvMkOpgNo/EbX1LaiXPoZ6+l+b1G++J+pX96GevulwNWH0q6H6N+heAWdWmeQ9nfczEoRfn/hR9OuqJ45CiOVUFpFTqsrgG8DfK6UGeYBFpAz8AXgXsBH4PTBHKbXSfa6bojmV8w7L8wtVg/wdd6ahdH5O58ibGc2v8G9/80v2PPcAe7f8mj2V59hz9G5KvfDz73yYG3/wRTbvPY5jR67nurFX8ZG9S2gaHMAUibBO695W6J4M+492/nYfW2L/cSX2nNALIox9dRxjmk5h7JFvZewpsxl5wrlIqRTaERl2H+N64tzTRV7cVioNvBdriEDFZ21J3s9+GOI4lZviVKyUWlVtgN9pZwOrlVIvVc/9NjAbCFQIRSOpsLyo+EUT5a0MQN8+93L9eqFWW/wETvuD+rh+Yc9AgVMC3sGok97BkdXvqEqFv/v4C/zXd4+nt+KECW08MI0OdTNrroQLzl1Cy5YyLa8KzTsrNO2q0LQHmvdC015o2gP3bJrDdVu/xMZKG8eV1nP9iKv4UFM1qqcElP33dXjpE9A7vkzPeKH7qAr7j6xQGQkt25oYte8wWphMy5gTOfzw0xgz/WJGts3Upo3QX7t3/SabO+mIc0/X0sEXEd34N2hcnPeznxWJhJ2KyM/RzxAuBS5QSn2y+voy4Byl1Kc1Zc0D5gG0tbXNXLduXez2JUXeo4Sih/SZtC+oD5PuY91YpVxW7FvzNN3rfkf31uX07N9C78EdztG3k17Zx72/ew8LvvVfHDjYn39i5Mh9/OOnP8Wf/tESUCAV4f0rFa/2DK5jEvDYtPNpHnEETS1H0XLUGxk19c00T56eWa4g0/BkN0Uc+SZB1AVhQbOtIs2EUg07BX4CPONxzK475+dofAg4DTzRfwAADW9JREFUfoZb6l5fBvyXiT3L+hC821Bkx1ZQ+4JsuH7RVWGvOyg8N6gcY6fsg+2eUUTtDxbsx1HBfqgi28aTICicW3ePmYZ6F6HfyDvKKEAhnAv8sO7154HPm5RbNIWgVPEFctExEbKm0VVBfR9X+IVxQLY/2K7K15UV16LK15ULqQyUiq8kG52w4dwiA0PBix7YoVTxFUIT8BJwPDACeBo41aTcIioESzyizLKiRi+ZKAO/ctIOocxrcDHcBzVe1x8k6MPOdPMkN4UAvA8ncugA8GptJgAcAzxUd95FOJFGLwJXm5ZvFUL65CEcwtYZ9eEznSHoyknTRJi2+XE4C/0o1x520FDk9Ra5zxDSOoaqQijKw1oEn4gJUR8+U7uvXzlp/VZpCpRG+V3TIOq1mwwekqgnC6xCaCCKdCOFEUp5KrE4fVbf7qQXYcUhTZNDkUevOpK6v9IaPBTtmfDDKoQGwsQpldWN1UijoaQevqI8xGkK7TSVTRhM+zrJ+yvOtfsphaIIexOsQmgggqalWd6ApkKpEUecRSdNJVuE3yvM9SXZ3qTSpxRh0BCVOArBbpCTEKabgphugJHFVnomm9l0dAyfVZpJ43dPLFzo7J9dz/TpySxsMvld0ybMBk9J3l9JXPvChc6iPKWcv0VYbJYZUTVJFkejzBDCjIZMHZ1JT+/90mxHXYxjZwh6gu6JoR5lFOa+TnpGk/e15w3WZJQvYW/orBe4pBV5MdwetDAE3RNFMOukSdiABXt/JUcchWBNRgkQdspbPyVtb/c+J8npfdj9mWv4TdmLkj+pqATdE0PdDBfGdLNwoXM/1cypWe3/bRmM3VM5AeImZEs7XXDSCb2GauKzJMk6iV8RKXIa7KFMnOR2doaQAHEdWWk7sXSO7CAHdxGck41KUN8Nh74d1s7ZRiWqrSmLo1F8CEoV25GV1MIuU2e0xSGojxqpDxuprcMdrFPZEkSSD3ReTkArlPLBOn0bC6sQLJkSJz1AVIEeVigNd+WR5PUP9YiooYZVCJZMCRNjXiPuKDPJMMahriySHtFH+b0t+RFHIdgoI0tookTIxI2qCRMp5VeXbp/hoRTmmHQE03CIiBpK2CgjS6ZEiZCJG3cfJlLKr66oazIaiaTXOAyHiCiLg1UIltBEWUgUNfS1Rhih5FfXUF8QBvH72o1dODZ8sAphCGGaYC8JwsaYJ7FWw1Qo+dWVtLAsImmM6O2agmFCVOdDFod1KpvTCKGBWTpz/dZPFL2fkiDpvh7qjvihBHk5lUXkA8C1wCnA2UopTw+wiKwF9gB9QK8ydHhYp7I51vFnjk2pEI6OjqHviB9KxHEqx1UIpwAV4BvA3wcohFlKqW1hyrcKwZyo+YosliDsYKOxiKMQmuJUrJRaVW1AnGIsCaBzmA4l27glH4aDI97ikJVTWQE/EpHHRcQGq6WADQ0sPkk6/bMMIBgOjniLQ6BCEJGfiMgzHsfsEPWcp5Q6C7gQ+CsRebtPffNEZJmILNu6dWuIKoY3RQ0NzFJwFblNNTt8/X4IixZFqzvJskywg41hRFRvdP0B/BzHR2By7rU4/gYbZTTEKWJET15tKtpG8mFob1dKpDi/ocUfirxjmoiMFpGxtf+BPwWeSbteS/4UcVVwXm2KYofXzWSytOnXZiPuwIRf/KJ4Mz9LfGI5lUXkfcB/AZOAH4jIU0qpPxORY4BblFIXAUcB91Ydz03At5RSj8Rst6UBKKIzMq82hXX6u0M9a2ahKGXFQacoV670blveJkpLPGLNEJRS9yqlpiilRiqljlJK/Vn1/U1VZYBS6iWl1Burx6lKqQVJNNxiTl52/CI6I/NqU1g7vN9MJkubfhhFOZTyQQ1XbOqKIU7WDsh6iuiMzKtNYZ3+fjOZLAMIwihKG4Y6BIjqfMjisE5lM/zSCuS9uUkRUx4UsU1u8v7dauic8EVom8UbiuxUtqRL0Awgbzt+EZOiZd2mKCa7osyuvGYj06d7n2vDUIcAUTVJFoedIQQTNJIsykizRiOMzpMkTphrkfuqyG0b7mC30By+BG1vWKS1AMNxa0urkC1ZE0chxAo7teRPUAhizRxShOyeQWsAdGGWRTAzRSVvk109fqGsjdzHluSwPoQGx8TWHNdmnlTYapG3tkwrNLdIobd597Gl+FiF0OCkHYKYZNhqUbe2TDM0tyjOYSjWbMVSUKLamrI4rA8hf0xs4KZ2aT8fQp629rTrLordvmj+DEs6YMNOLWkRNKoMM7r2m83kOZJOe+QcZLLLaiV5kWYrloISVZNkcdgZQv5kGdbqtw+yu54kR9p5jpyzjgKr78v6DKY24mjogA07taRFkMAKCntNq/4kBWeeobl5KaMihSNbkiWOQoi1p3La2D2Vi4HfpvRp77erKz/pevyuMU3y2gvb7pM8dImzp7JVCJZYuGPbayQV6WSyXXeBb+FA8hLMeSkiS/rEUQjWqWyJRdphr0Hx+n77CaTlqE2y7LwcvUVaH2EpEFFtTVkc1odgieJDSNM+nkbZeYSlWh/C0AXrQ7AMZert+zX87PxpmmGGku09L7+JJV2sD8FiqSNN+7i1vVuKjvUhWCx1pGkft7Z3y1AmlkIQka+KyHMislxE7hWR8ZrzLhCR50VktYh8Lk6dFksQaTpq7Wpfy1Am7gzhx8BpSqnTgT8An3efICJl4EbgQmA6MEdENHsuWSzxSTPyKcv9jC2WrEnMhyAi7wMuVUrNdb1/LnCtUurPqq8/D6CU+nJQmdaHYLFYLOGI40NIcoOcjwN3e7x/LLCh7vVG4BxdISIyD6hNwA+IyDOJtTAdJgLb8m6EAbadyWLbmSy2nclxctQvBioEEfkJcLTHR1crpe6rnnM10Ass9irC4z3ttEQp1Ql0VstdFlXTZUUjtBFsO5PGtjNZbDuTQ0Qim1UCFYJS6k8CKr8ceA9wvvK2P20Ejqt7PQXYFKaRFovFYkmfuFFGFwCfBS5WSnVpTvs9cKKIHC8iI4C/BO6PU6/FYrFYkidulNHXgbHAj0XkKRG5CUBEjhGRhwCUUr3Ap4EfAquA7yilnjUsvxF2e22ENoJtZ9LYdiaLbWdyRG5joVcqWywWiyU77Epli8VisQBWIVgsFoulSqEUQiOkwhCRD4jIsyJSERFt+JmIrBWRFVXfSuar60K0M9e0IiJyhIj8WEReqP49XHNeX7UvnxKRzIISgvpHREaKyN3Vz38nItOyapurHUHt/KiIbK3rw0/m0MbbRGSLbm2ROHyteg3LReSsrNtYbUdQO98hIrvq+vILObTxOBH5mYisqj7nn/E4J3x/Rs2bncYB/CnQVP3/K8BXPM4pAy8CJwAjgKeB6Rm28RSchR8/B2b5nLcWmJhjXwa2M+++rLbhX4HPVf//nNdvXv1sbw59GNg/QAdwU/X/vwTuLmg7Pwp8Peu2udrwduAs4BnN5xcBD+OsXXoz8LuCtvMdwIM59+Vk4Kzq/2NxUge5f/PQ/VmoGYJS6kfKiUoCeBRnzYKbs4HVSqmXlFIHgW8DszNs4yql1PNZ1RcVw3bm2pdVZgN3VP+/A3hvxvX7YdI/9e3/HnC+iMnGn4lShN8xEKXUL4EdPqfMBr6pHB4FxovI5Gxa149BO3NHKbVZKfVE9f89OBGcx7pOC92fhVIILj6Oo93ceKXCcHdEEVDAj0Tk8Wo6jiJShL48Sim1GZybHDhSc16LiCwTkUdFJCulYdI/h86pDmZ2ARMyaZ1HG6rofsf3V00H3xOR4zw+z5si3I+mnCsiT4vIwyJyap4NqZopzwR+5/oodH8mmcvIiKxTYUTBpI0GnKeU2iQiR+Ks03iuOvJIjATamXpfgn87QxTTVu3PE4CfisgKpdSLybRQi0n/ZNKHAZi04QFgiVLqgIjMx5nVvDP1loWjCH1pwhPAVKXUXhG5CPg+cGIeDRGRMcD/AFcqpXa7P/b4im9/Zq4QVAOkwghqo2EZm6p/t4jIvTjT+kQVQgLtzCStiF87ReRVEZmslNpcnc5u0ZRR68+XROTnOCOitBWCSf/UztkoIk3AOLI3NwS2Uym1ve7lzTg+uqLREGlu6gWvUuohEVkoIhOVUpkmvRORZhxlsFgpdY/HKaH7s1AmIxkiqTBEZLSIjK39j+MsL2LW1iL05f3A5dX/LwcGzWxE5HARGVn9fyJwHrAyg7aZ9E99+y8FfqoZyKRJYDtdtuOLcWzOReN+4CPV6Jg3A7tq5sQiISJH1/xEInI2jhzd7v+txNsgwK3AKqXUv2tOC9+feXrKPTznq3FsXk9Vj1r0xjHAQy7v+R9wRohXZ9zG9+Fo3gPAq8AP3W3EifZ4uno8m3UbTduZd19W658ALAVeqP49ovr+LOCW6v9vAVZU+3MF8IkM2zeof4DrcQYtAC3Ad6v37mPACVn3oWE7v1y9F58Gfga8IYc2LgE2Az3Ve/MTwHxgfvVzwdlM68Xq76yN4su5nZ+u68tHgbfk0Ma34ph/ltfJy4vi9qdNXWGxWCwWoGAmI4vFYrHkh1UIFovFYgGsQrBYLBZLFasQLBaLxQJYhWCxWCyWKlYhWCwWiwWwCsFisVgsVf4/iRS0FGZY2ygAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_model(model1,testx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO2deZgcZZ34P9/umWQySUggCRACMwEEJBAREkEWz2VdAV0iiEeMXCIxM8v+ZE8Vdrl2o+u6h6gkMBwuR4zXAnKjsqLrgZAgEAg3OUmEHOSaySQz0+/vj+pOeqrrrXrr7OqZ9/M89cx0d/Vbb1VXvd/3e76ilMJisVgslkK9O2CxWCyWfGAFgsVisVgAKxAsFovFUsYKBIvFYrEAViBYLBaLpYwVCBaLxWIBEhAIInKIiPxCRJ4XkedE5Ise+4iIfEtEXhGRZ0TkhLjHtVgsFkuyNCXQRj/wt0qpJ0VkLLBURH6mlFpetc/pwBHl7SRgYfmvxWKxWHJCbA1BKbVeKfVk+f/twPPAFNdus4DblMNjwHgRmRz32BaLxWJJjiQ0hD2IyFTgeOD3ro+mAGuqXq8tv7feo425wFyA0aNHz3j729+eZBctltAsXar/bMaM7PphsZiwdOnSjUqpSVG+m5hAEJExwP8Alyqltrk/9viKZ80MpVQX0AUwc+ZMtWTJkqS6aLFEQrzu3jL1uD07O6GrCwYGoFiEuXNhwYLs+2HJJyKyKup3E4kyEpFmHGGwSCl1p8cua4FDql4fDKxL4tgWS9oUi+HeT5POTli40BEG4PxduNB532KJSxJRRgLcDDyvlPpPzW73AOeVo43eDWxVStWYiyyWPDJ3brj306SrK9z7FksYkjAZnQKcCywTkafK710GtAEopa4HHgDOAF4BeoALEziuxZIJFXNMHsw0Fc3A9H2LJQyS5/LX1odgsQymqcl78C8Wob8/+/5Y8oeILFVKzYzyXZupbLE0EHkyX1mGHlYgWCwNxIIFMG3a4PemTbNRRpZksALBYmkgOjth+fLB7y1fbqOMLMlgBYLF0kDYKCNLmliBYLE0EDbKyJImViBYLA1EnpLkLEMPKxAslgbCRhlZ0iTR4nYWiyVd8pQkZxl6WA3BkjmdnU6ClYjzN6sImXodN2kWLHCS0JRy/lphYEkKKxAsviQ9iNarOJstCmexBGMFwhAjyQE8jUG0XmGTWR13qGghluGJFQhDiKgDuG4Q0w2WCxdGH/SSCJuMMuhmEa5ptRBLw6OUyu02Y8YMZTHHsSrrt2JRqY6Owd/p6PDeV/e+bnO3q6NY1PfNBL/+pnlcE7I4hsUSBLBERRxzrYbQgHjNkE1nye4Zq58pJUxsu6npJW7YZBjTT/V1KpXiHdcEmzRmaXiiSpIsNqsh1KKbIYuYz+arZ6xBs/4wWkKYc6jMpr20Fj9Mjx/U97DHNcFqCJY8gNUQhg+6GbIKsaxF9YzVL/N1wQLo6Ni7T7GoX184jDYRJ2zS9Pi661QspheuedRR3u/bpDHrbG8UrEBoMJIwP1QGz85OfXuVQcw9eM+b579/mnR26gWf+/hZm2+8qpCCLU0N1tneSNgV0xoM3YpZYejocP4uXFj7mYgz6PsNYp2d9cmU9Tt3922c9cpidiUzPfbaZEvdV0wTkVtE5E0ReVbz+QdEZKuIPFXerkjiuMMR3Uy8o6PWvKN7b8ECvUmlUAge3OuVKRtGEGZd88c6lPXYa9M4JFXL6L+B7wC3+ezzf0qpjyZ0vGFLUC0br8HZ671GfEiLRf1M003WNX/C9G24Ya9N45CIhqCU+hWwOYm2LMEkMUNvlDLK1c7IIH+Hmyw1GVuFVI+9No1Dlk7lk0XkaRF5UESOyfC4Fg+SfkjTiCJxOyMrVCKNqk1g9cYrIisvfas3Xtdm2jRHe7NRRzkjaryqewOmAs9qPtsHGFP+/wzgZZ925gJLgCVtbW3JB+la9hAnH8DdTpzsZR02rn9oktb9YnEg73kISqltSqkd5f8fAJpFZKJm3y6l1Eyl1MxJkyZl0b1hS1ImlbQKx2Xp5/DTcEy1Hxtrb4ZdFzq/ZCIQRORAEUfRF5ETy8fdlMWxG4FGH0iydlAn7efwi5M3jaG3sfbmmNwvjf5MNCxRVYvqDVgMrAf6gLXARcA8YF7580uA54CngceAPzFpdziUrkhLfU7KHGRCGqYdv9ITWZacMD03a94yJ+haWZNSPIhhMkpEIKS1DQeBkET1T/fAn/UDlcbxdNdFpPbYcQVfmHpNutpJpvtZgu8XK1zjYQVCAxNnIAlbfC7NByppjcTkuiQliKJqCNXnXI9rbkKWmmIY/PpVT+Ga1+sVBisQGhi/ASVqjf+hMFs1mSUmNZP0Eyy6z6ZNM7ve06Yle12SOq88Uy8NoVGvlxsrEHKO36wjaJbvdzOGFQa6ByqPsyKThzNJwRf0G7k/CyOM63U9G9X0kuXA3AhaXlisQMgxJje3n1Dwuxn97OymD1Tchy9NYRLUdlrObJPzSUIQp02SAjNrspikhDG5NhJWIOQY00Erys0YZOoweaDCDKruNnVmk6xmxGGFWdA1CdNeWHNdPciDhpDVwB7lGKa/odUQcrINBYFgOkhEfXjjPnCm/QvrwE5zAHD3y+T8TQb7sMIxzHWoB/W2iWdx/DjHMP398mBCDYMVCDnGdJCp18MbN84+zgOVt1wJU+Go63+9NSaTPobpS9zfJwsNxcT+H1Y7znJCkwZWIOSYMAN9PZy7pv2LKgx0A0DWAtBksE9iAMujgz4KSfw+YQVsFOJMSuqtQaWFFQh1xsQ27Z5J5mnQ0CW3Vb8XRyB4DQBZ27dNjjdUBwhTko64yYOGUNncCY0VhooAr8YKhDoSxbGZ90EnSsJb2EEki9mjyTkFCe88/S5pkkbETT19CFndV3nECoQ6EnYWlIfIjyDCaARRZ9hZXQf3rLcSkjucBnsT0oq4yTrKyAqEeAIhywVyhiRhK302wtKVUdcuNl0kprMTSqXg9uLitcCOUk6fslwLuhEw/c3D/j5ZrFpXfQw/bPVUA6JKkiy2PGgISSdHNbqGEHe256fiJz17DLrW9UyqyxtpRtxkeZ1NS4qkcb/lBazJKB3iZBmH9SHkaeBI0/abpUAMGgzinqNuoGsEP5GbtPqc5rXQte0O2tBl7udpEpYkViCkRJgcgjAzoCCbZx4GjrRmdX6DdNL4/X5xBZPfQNcIWqAXafzmaV4L07azvOfyQByBIM7388nMmTPVkiVL6nb8ymLuXiRx2ZqavG23xaJjEx2KZHnOFR+Cm44O7/crmPy2fufhZ4/P8eOWCmk9Q7rftrrtzk5nWU7d7zFUnzMRWaqUmhnlu9ap7INuqcaklnBsBAdz0uickkk6kyv4Obnj/rZ+v51uEEx66c9GII1nKEgYFIveAQVu0rjnGh0rEHxIe/BKW+A0AiLekUhJoYtyifvb+v1GupnvcByA0niGurqCj+m3jy76zYL1IQSRtF21ur0wZaqHAnlztsat8+Nnm660mbQPZlAf7utQxauLiqtQxauLquO+fN44ST9Dfte80vZw8xtUQ719CCJyC/BR4E2l1LEenwtwLXAG0ANcoJR6MqjdevsQksZE1Z07d+jOXIaaz8TEjp3ase/vZOGS2oMLwryZ81jwkSF6E2F2Hw21ey0MefAh/Ddwms/npwNHlLe5gM9jNHTRqbHFYq1Jw4TOzuSTbdJos8JQ85n4/VZpm/26lnrfTArFwiUL6bx/6GZemZihwpiq0rznG42mJBpRSv1KRKb67DILuK2szjwmIuNFZLJSan0Sx0+K6qiENGbrSQ6I7tnpwMDe11H7nEab1egicOrlM1GlEv0bV9C7+nH6tq7hv248iOvuPY23evZj4pg3+H8f+Xc+/v7F9Bd2Uir0o6SEEoUqKETB/b/5FM3FG+gbGDn4fGQ3//jxb7Luh8/R1DKR5tEH0jSujZap76Z5YnsifR9Q/jdN19KuIaslVO5Fv2fVZB9I/55vNBILOy0LhPs0JqP7gH9VSv26/PoR4EtKqRp7kIjMxdEiaGtrm7Fq1apE+heEX4hiUjdGkmpsGipx2mp2FtfYjerfzc4XH2Hnmsfpfes5du54md6B1+kdtZWdE3cD0PJmgZ8+Mpt/+eEN9PaN3vPdUcVu/v2oi/nU2MUUdoOUgIHyX+DYl1ewtm9qzTH3HbmBxy86gP5xBfrHCP1jFH37KHr3LyED0LJ5BC07xzOqeDAtY46gZcKxjDrkJEYd+UGkaDZHa7qmKVAoqCuHWYxrBIaiaSmOySgrgXA/8DWXQPgHpdRSvzaz9CFkcWMkOSCmEd/t1yYkozWlqYWV+nrpefZ+drz6ENs3/Y7tTSvontxD8zZh1Loio9YrWtYO0LIeWtbDqPXQtB0EmMoKVjG1ps12VrKSQz2PV2AAr3JgQokStWqPAvr2gd7J0HsQ7DwQeg8u0nuQ0DN5gP4xijHrRzO2dDhjJpzM2CPOoHXaaUjTiJq2dD6ECkUp0n9F+iNa2lp12qSda1QP4giERExGBqwFDql6fTCwLqNjG5GFfdtUjTUhDfNLUFJVEur0ggXJDRj9b61ly+8W8taan7BtxKt0T+5l5EZh7KtFxizvZ+JLMOZlaO5WgP/guJq2UO8DtLHaU4i0sdpzfwFGbHO2fV6svLv3gveNhe1HdLPjyGfYNG05K3d1sft1xZh1o9hn4Aj2bTuLcSfPo2ncgXvMQTqhMHdG+jGuQ8HcYvocNbrgMyZqeJJ7A6YCz2o++wjwIM4z8W7gcZM2sww7TbvcQBrhq0EhnFFKagSFUtazBMNAb7fa8qsF6rWb36uW3tyqfvkA6g/fLKqVn0G99Q5UX6tB5zVbOys8P2pnhfY7dzBbtbJj0Nut7FB3MDtyP9zb7tGoze9ErfysqCe/VVS/fAD15I2j1Yrv/qna+tubVKlvV93CT7Ms0ZFWKZU06pXVG+pdywhYDKwH+nC0gYuAecC88ucCXAe8CiwDZpq0m6VAqEcRriRzGtwPiV/hL9M28xDLvXvDSrXuRxepZ66fqH51L+qJmwvqlY6C2jQT1T8ymUE3zuB+B7NVOyuUMKDaWWEkDO6Yjmq/FCVXOn/vmG7ez74W1MYTUS9fUlCPf7eg/u8e1LKF+6v1/zNP9b31enY/jMru/kh7QE66onG9qbtASGvLOjEtrVlIWkXyohyz+thB7dfrQejftkG9cfelatn1k9Sv7kMt+2pR/fHPULvGRxvswwiFsIN76GNMR7VehuKqvVvrZeGEQvXWux9q/YdRz3y9qH51H+rZhQeqN+/5e9Xf/Va6P5LK7v7QHUe3LGbSZCX4ksIKhJxjckMlPQsyHVP82o/Sp6hCbWDndrXhgX9Szy08WP3qXtRT/1FU6z6M6hud3GCch6390sHCoLK1Xxq/7d1jUa9/BPWHa4vq/+5BLV/YrjY+/M9qYPfOcDePIVmZUqLev5U+xp1kWQ0hJ9tQEQgmN5SJ0EjimGFv6jAPVJQBYtfaZWrFze9Tv/kfUUuvK6q1H0tfE6jnJld6CwS5Mtnj9O6HWvNx1JLrC+q3Pyqold/9kNr9x5eM7p0wPomw94f7voyjqQbdv0kJrOHkQ7DlrzPAJNw06fC3oLIKcdv3Ikzo7vYnFrF26eVsmrKKSb8uMuWHA4xZmUw/8szUS2HV+Nr327fAym+mc8ztb4PXP1lk47sHmPT64Ux59zcY886zACd8tWtpFwNqgKIUOWrCUSzfuLymjY6ZHbES3YLuR7/Q67AlQtIoe91IUUa5yENIg6EiECD4hkojHvqYY2B57bM9iCTzLILOodTXy8YHL+f1DTfQO7aHKT+Byfcpmrclc/w8s2g6XH4qrBrnRFioqmvVuhu67oU5y9Ltw+7xsO5MYd2Zita39uHHvW9j/qbAkmKAd15DmEFSN1nY037AfehXUrz6e6YToRwPe7HJQy0jSwBBi43rbvigZDE/nntu8HoAXiRZkllfzlux4d5/YMmPxrJ23TeZcns3J81WtH0vn8Jg0XRnJl+40vm7aHr89ub+RVkzEEcYiAKUoxlkIQwARmyBqbcp3v1pmPyDbdz8lpkwgNpSGe71Bio5CLo6QEH5PEGfd3R4v+++f4NKY8PwKi8fFisQcsK8eeHeN6VaEOkWi0kKL+HSVOjjigsuYOWG/+Dwa/s5vqPE/r+EQknfTtIDchiqB28lzt+5fxGvD5efCj2uZGMl0L7VMRNlIQwqLJoOh10Ck0+BP4ZIuizK4FFUN/D6FXD0bT/gc7/FjqoxSSQ1mQQN14J3ViDkBNMbPu4x/LSUJNrfew6KA8av5h/nXcAXVt7GzM+XmPC4Yy7xI40BOQxeg3fPCOd9E7yE2epx3vvq3k8L97XV/hge5hR35nPYzP6gQdhkkDa5f/0Ei+kzZar9DEWhYX0IQ5ysnWG71j7Dq/d9lC2T1tB+O0y+HwohfBT1cLpWU7hysH2/gigoXe3/3cqAWy1QWnfDqD7YNLp2/6zOqdK38z8GAwEz8VElOG0i/GYzvKmgWCgyd8bcGodylNpfXs7epO/JJOqFmZxbPQo1mmJ9CMMYv1lKWDtvHNRAP+sWz2HJknfS8tTrnPRZmPKTcMIA6j+bbtsa7v1qdNoFOIJhEKrsYL4SJv59uhpQRVD5CgMFE7rhxrvgh1+BX28u8OvjhLXTLuK6079Ts3uUpTGrZ/iVLWlNNa6m3dlppv2ENZk1ClYgNDBBA35WN23PC4/w1H/vyx+3fp/j/kZx2A0lir3e+wb5B+IMyGGOo2P+I7WDd+tu5/0gdEJrc6vjOG7fgmOOUTjmmvK2aTRcOCs9oeAlqGoQGNPn+DOauuGIb5WY/mXF2q038czN+9H72u8G7Z6FiTMqUU2jJisaVhhqiz1VsCajBiZItU27tK8a6Of1xZ9k5di7aL9DOPhOtWetAC90JpXqKBuTfYKI20YlRHT1OEcQzX/E7Hsm5i7dPuDM0Dd+I/g4fnj1/dyzvc1gbrzMYqUCrPlMgbXnlDis9zwO/MR3kcLQnEcGhcZWC7w8r6NgTUbDlKBZij4MNP6xd//xRZ65ZSJvbvsJJ1wCh/zYXxiAmcN2zrK9s2mJGJYZ1zE8Z5kzgJeuDhcF5KVdSNk0FORgBtjUGk9L0Dnk9+sx+76XFlYoQfsdJY67FF7fdgfP3nQA/Zu9y3s3OqbCAIbuEp1WIDQwuoFdRu4Aotl5Tdj+xCKe/NU0xjy5neP/qkTrWrPvmfoHog7IYY+TNNXCDOUIg0o0j9HgLOZCy4uwPoxqgsxiY1bCCZ0lWp7fzNKHD6f7mXsGfd5Ig54OvwmU2+xkajLL0o+XBFYgNDBz5wLi0k+bu1FnzKPz/s5U7Lx//NHFPLPusxx2XYnDry8FagXVJOUfyMtxoNZXAY4Qa99aa6apDM4jfEwKcYSWiQ+jonV1PB5eCyv0wxHfLNF+az9PrZjFm/f8LVC/QS9pIRR2AmXiq2g057P1ITQ48rHPwaNXwNY2GLcaTr0M3rE48SUUS7t6ePW/38X3Ssu5+TVYOzqcfR2S8Q80ynF0dntRcPudcN5Zjn3eTZxQ1CxDdrcfAc/+s7D/thm8be7jeCU1pGlPTyvsM+kw7Xos0Wl9CMOZ478Lf30oXFV0/r5jMVBbaiAO/W+t5ek7DuDHG5fzb6/DmjHRksaS8A/k6Th+vgo/LWXOMrjtrujRTDriREiFZezLMGOu4vu/PFK7T1DETZwZfloz76STN9P046WBFQgNjrukQDVytdB0TROd90fXpfs2rODpnxzJ6OU9XL8RepoHfx7GWQvx/QNhj3P7nc7rc8+OVwYjbAZy0OCchtDKShBWGLENuhbNR5fy7DfoxTUzNUrYZ1p+vLSwJqMGp/P+Tu1C69VEKV/c98bLPP3QdMY92cfbvlWiGCOLtx4kZTqKmoEcNXy1kSgwgNLMK/3MN1HDNtMobZ02Uc1QUb9Xd5ORiJwmIi+KyCsi8mWPzy8QkQ0i8lR5+3wSx/VjKEQ9mLDgIwvomNnhqykAdC0Np0vvXvccT/30WPb9vSMMhGydtUkQN/w0qB0I1gKy0IbqyX5s0n7mN3hFmeG7tQov8jjzjmKGqpejPraGICJF4CXgQ8Ba4AlgtlJqedU+FwAzlVKXhGk7qoaQ5zojaSNX671Y6kqz33r368/y1KMnMOnRAabeVNpjEEhyxp3FzDlqXSJ3/1aNw9sqomBCOYx0c+vQ1QL8mMibbGJSzfvFwgD9PrUyomgIfoljeV+0JixxEt/qrSGcCLyilHpNKbUb+D4wK4F2I9NooV5JotMUgjSICgM9W1j20xOZ+Gg/h1YJA0jGRh2lmmnUMhRRNBqv/mlFbLnsxM5mx1cxVLUAPzYzwfP9Ukko7dInXUSxrftpBmlU760n9fKRJCEQpgBrql6vLb/n5uMi8oyI/FhEDtE1JiJzRWSJiCzZsGFDpA41isMpDdxlioPer0aVSrx023G0rN3NoTd5axNxzSBhzThxymFHibrRrV0gPspVFDNUEPVcEyIMbXhnLR84fg2v/ve7Br1Xbcbt6oJp08LlyDRaxE4c6nWuSQgEjTI9iHuBqUqpdwA/B27VNaaU6lJKzVRKzZw0qVYVNWE43Thu3D6FohSNHcprbv8o3U1rePv8gcB1C6ISNos4jh8gikaj64eiqjhdiO9Fod5rQoRhPpfRSveg91rp5us9X+Gt1udZt3gO4G0TX77c0QhMbetxI3Yaya9Yr+ikJATCWqB6xn8wsK56B6XUJqXUrvLLG4EZCRxXS6OEeqV1gy74yAL6r+hHXan2JKc1XdPkG4a66aGrWTvmQY69TFHcVfOxL2Fms2HNOHHLULg1GohWbbWyull7Bo71pJzhWTCHxXRxMe2sRCjRzkq6uJhzdy/m2K8oVrR8jy2//HYiZtw4mfcmTto8CQzducLePla2JPuahEB4AjhCRA4VkRHAp4FBhU5EZHLVyzOB5xM4rpY8l+atkFUUQSUstZKoNqAGWLhk4SCh0PPCI7zQezXHXAktIa10YWezYc04SUY2mfR1/iPQ7K4G0r+3f1kkf9V7TYiwzGExKzmUEkVWcihzcJIjW1+Ho+fD8k1fZGDAW7UKa8aNmjgWJJDyWHPIfa7gHWWVZF8TyUMQkTOAbwJF4Bal1HwRuQZYopS6R0S+hiMI+oHNQIdS6oWgdodyHkJW5XObrmnyzFqulLZQpRJP3zKeCT/fwSE/MLsXqqNwCiXvhVf8yiWEiTJKsgyFSWmHRdPhc7Ngd1PVDuXLUoko2tQKxRIMFByNIenIorAlKPKe77DiQuHku1bxxpZa12FWeQNBJSTyXM66QlB57r1l7+uch6CUekApdaRS6nCl1Pzye1cope4p//8VpdQxSqnjlFIfNBEGQ52sHN+6EhaV9//4o88xUOpmyo/MhUH1LFsXWVgp+exlmgnjmE4y+9Zk5n35qS5hAIMWstk02vl/oAitfcHCLKlFelCwo7m2jc7TnSzsaq3n3LOd9/NC+x2KU46/By8HTFZm3CC/Yh4DUdwmrKC+JNFX961vyYhiUT8jSfQ4UtQKBblKOGAEfPV/YYZh1VKj1bdwxtDKLLdimoFoA/mcZcnMeNu2es+89+txBuzV47Q+Y096RjjrFFf6WI1bswlzDSqff/E0RxupFkjVbSyaDtefWJtrocR5/5S18a5bUprH4r7ZPPjoBbjjT6ZNy86MO3eud25SRSBl9Tya4s6lMhnsk+irrWVUJ7JyfPuGmwq80Qd/dZL57NXEjr1nHYAqekY4A5zpjDnpsMtF050ZtnvEb+6H7SP3zrDDhlcNFL19Jkks0jOmr7Y/1W1cfqp+JTQVc22FJCOdLuer7FS1NT5efDF6/8Li9itW6OpyBt+8BaJEyZlKoq9WINSJKI7vKFEQJqUtwgxUOmducWCvSUc3y97UajbAJB12WWmvYu4BQDn+j76ih4koJF7XLwnHcFAbQW3FcUInGem0mjbP97M2xyxYUDtoVhyykK9AlCBfgft1Un21xe0ahKTKcehKW5gWqDNx8vqtG+zGy1Gq+/6EbmfWnNRax76UHwsh2nrEUdYmcJtodjT7F88LOi+vY5magaKW/fBiKitYxdSa9+vhsG0E5zE0dukKSwZEiePuvL9zT/5B4eqCb50j0zBOEyevp1NUg9csVjezNdUwTNvzY0IPqKudkhSVc53QjXaFOPf1Cxue6qUVbR9ZGwJb3YbfdfY6VhjNK8lwX6/ktaL018Uck7bzOKlchkZOTLNkQNgb2Z1/oHzcpWHj6IOihNxrC/vhNcBoBx0fe3rYYwSxbaQzUFaf68ZvwO2GC9uEjY7yMtHsboJ9dunbcF/n4oDzV3esMGagMAItyN/jTl6b0rqSK2dfWhdzTJpVDJLMZahXLpU1GaVA0svwQXgVUpd/MAiVThy9SZ4COAPM+X+AB44abMKAWrMUCk+Hr4kJw8vMZYLO5LIn+gdHk7j2ofjRPJ89m8jnZ0pYM5CJeSlKnkh/Kzz2PZhx9C8YdeQHAvud5POUZiXkvJijhrXJKC/p5tX9SCPjMawKabKEppB8hU7PPAX3nKNsfjn/D3Dr8bUmDKidXVeSwtyYzP7ds/UJ3bWmGC/cpqYa57Q4lU4rn029FORKaPon569JdFSlTV10U5IlMcKagUzyRaI4n5t64KAHCqz9xV8G9jnpDOKkZ97Vz30ecxnC0tAaQl7WPdD1o5okZglhZkomGkIai6/rHJ3FAWdR+eqZZhjHa5IZy5X2grSYCd2OmSjo3CZ0O4LBSwOp9BG8Z9t+juE45+dF0tcQojufdx4ET35H+JMzdyNFfZhXXmbdXpg892A1hMzIy7oHJsdLYpYQpo5LULnrtBZf1zlwS4XamWaY0MwoGct+tu3q2e+td8MIjwe24kcIOrdNrXpzVM8I+MJH9M5crcNbQU8T/OZg/fmFJY01l6M6n0etg+ZtwrbHbvHdL8+zbtNxJm9FNf1oaIGQl5slqyxCL3QmM3f+gSCMJf3F18MMEGmYMCqEiaiZswzGelR47Wvaa/pYNN3RJKLQPVJvVvF1oBdg4YnRylDohL3IEYoAACAASURBVGHSy3rGKfY38TeKjctv8t0nz6Xsg577eucyRKGhBUJebhaT46UxS/Czr3Z2QtesBQxc2U/xnxXz/tjPQ8cK27+b7speYQaI+Y/UzsxH9CejuYS1bVecxG5Wj9srXLzMSq27YURfQGc0/oHV48rn6me1FegKqfxnuZ5CHK1j4q8Vm1r/4LtP3jKIq/Ebf8JWY80LDS0Q8nKz+B0vzVmCTmVduLBWUHTdUOJnP59Nq/cCV4kRdoBwj4X94kTxxC1ZEcYctWi6vmJF21anP14moeIAnLwadjdH62PbVue66BzmFQZCPqVZr6dgonUsYjZTWUGBAaaygkXMZuwL0N/ST8/zP/Nt312pNKtZd1DASl7GnyRpaIGQl3UPdP1Ie5YQxjQ2oJq4YdFXU1sJrRpTs8TlpzpmmWpKRSeKJ+7MNow5SlsTSMEZL+q1h1IBHj0Uo/pH7iU4q7Wmax/yT+QrhjRV5W09hUXMZi43soqpKAqsYipzuZHvqdmMf7aJbct/5Pm9igZcj7gXk+imvIw/SdLQUUaNQtJx1JW2wiKUKJED42sZXYSKmyjRUGEiarT9UOVNM21q3+KU+Q4UCApOfRVemaiP6V803XE+d490taeg43FY8GDAMaqIUjYjSWpKcFy3gk19U2v7w0p+ccGhqPeczGEX/bbm83pGGOU5uimIIRtltHRp/fML4hIljlqnqrrb0qFbDGTKiJTtRSExjbGPMrMNY7oKcux6Ul6fwAhxhEFQdveOf3UG/0rGcXEgvDCAbFZ10+Hlv9jU513cbjVttK6Cnr5XPT+vZ9BIXgJWKmSVb5VrDUFkpoK9GkIjqmNhZxp+uRVBmkFF+4DaNkaO6GZh68VcuGWxeedTxjSDOM7M1jTbVpcpHIgmg9prvzvuzG5ls6xWUTMqyPdfK2Dr1JrvtrOSZW87lBeuaOZd59bazKyG4BA23yqOhtBQAiEv6loYE1DQ0n1u/G5EP2Hgbqu6j81NA3zpb87lmn9frC3OliXVg8h+ZYfq5lbn/60t0F9l1RrRD7f8JDjE1GvwC2M2kiuJJhBMUTBiYHCpbVEwL4IGkBRxhYanQPcSkM/MhntvhL69kqKVbrq4mE+3LOY3d8N7T92FNA2eGdQz8TQvSa8QXjjV3WQkIqeJyIsi8oqIfNnj85Ei8oPy578XkalRjpOHZJSwJqCwobF+qmqYtqqT2N76w8N89LjvZyoMdHHwbpPCptFOlu/tdzrOVbfzNWi64hdiGRRts2g6TPz7sjBIG6ldd6GyslmSCwWZ7p9EaKrn6nleQvUdi5lw6t7idu2spIuLmcNiir1Q7Ibdrz9T87V6Om3DHDttc06W5qvYAkFEisB1wOnANGC2iExz7XYR8JZS6m3AfwFfj3KsPCSjhM2ODhua5jfoRw1zK/Vup7Aruelv0KATdZD2ijqqTg7zwq89v2ibRdPhwlmD6xLFIqKi7beyWdCg7f4dOk83H+STCE31y7KupnU3XPv6YlZyKCWKrORQ5rDXdPmz//0MYw59ByLOoFoo5MNnaFIZwGSCGFdgZJlvlYSGcCLwilLqNaXUbuD7wCzXPrOAW8v//xg4VcTPmOJNHuJ7w0rrsLMcv0E/6oxJ9e2g0JeMQDCZWUYdpKOES/p9xy/01Ev41AvdOfhdR6/f4foTzQf5JEJTddd3Qs9gZ/75t83m8mWDcxAqLGI2/3L9zfSrvR1XyhlUjzkmnUKRSRI0QUyiOF+W+Q5JCIQpwJqq12vL73nuo5TqB7YCE7waE5G5IrJERJbABiBf8b1RpHWYGkRBg36Ytiok6ScymVlGHaSj1MXRfVYoOTkEumgb34EvyuUKkLfN/fp2defgdx29fgddCK9XO37XzdQ8pYtmuvahvRFV8785m1vXDs5B+Cx3IJQQSpzL7QwMtHi2v3y593GzrlXmR9AEMYl6a1mazpIQCLoI7rD7OG8q1aWUmqmUmjljxqTcpYBnIa2jDPp+FEaMptRUe7m9skeD0A1Sq8btHUT8Bna/kMgzXsSzVPYZPoux61YNGyg6pbXP/8PgBWR6mp3BtOLM9iQp61o5j6F9C3z3J04IqV+Cmhu/6xh3Ju953ZRz3Ux9CiahvZfzVXpwhx0VqNjpVIS8mDz4EisETRCTsv8nPSboSEIgrAUOqXp9MLBOt4+INAHjgM0JHDtzGjE7sTBiNCXXbFKXPRokFPxi9iuDiN/M3G8QeeAoagdjKb+vodJe0eMB6xnhfHf+I9DaV65FVO7ntpFEtvubUlmGs5J3sODBwUtytm9xBNblp3rPyv2Ep+53MBU47t+hOECkFemCstJX452DEIc8+BIrBE0Q81JvzZQkBMITwBEicqiIjAA+Ddzj2uce4Pzy/+cA/6vyHO8aQFbS2gQTh9XIQ05g14TSIJOC18yth9Fczld9jxe0XnJlEPabOeoGkah27TnLnDISuu96mVf6mtibiZwSm1prne/V5z7/Ee8Fgqork+quo05YzHvcvI5UdV/8rl8c2oieDDnNHZpSJg++xApBE8RGq3cUWyCUfQKXAA8DzwM/VEo9JyLXiMiZ5d1uBiaIyCvA3wA1oalJkZcV1LLA1GHVtO/BNO0Udk3c+55u5hY0ozNZL3n1uGhlloN8CH7RTZHMKxXLBaQmGKI63yvormPld5jQzR7BNqoPTlkbrbx11HUNgpjPZQjh4p1FnEH1uefqp42HGUf8JoheAmPaNMeHkMcxKpE8BKXUA0qpI5VShyul5pffu0IpdU/5/16l1CeUUm9TSp2olHotieO6SXq5vbh98buhkhBcYRxWoza30lNl2NPN3ExmdJVBqj3hQcTPRBIU3RTFvDIIjd9ghI8zOGybYZzvpuxsZk/Y7KbRcO7Z0dZPSKvcxRwWB16+YmHXnmKQSkGpFC+IIi5pLNtZOYe5cx1neR7GKC9yXcsoLHlZQS3ohkrqhgvjsGrtP4idVZP/+VxGK92D96Gb+VxmfPwog4hX7Hzl9eWn7nUCu00eQbPpIPNKpEFdOVnSSVI92MedlesijYKS3bxIYzW1Cu3aSYZifPMGrvjENbnywaU5juRljNKR69IVYaudhi0TkRZBqeZJ1UkJ086aO2bRu+Q+jrh2r/q+iNlczldZTRttrGY+lw1KGDIhTPkDk1IHopy32l1tRV27t8LEv/eosRNApYZS0z95L44Theq6TJ2nOyuiRa1u6lctNqvKpiZUAhiqfVaV0hUnXP4jxr3jMxz0yVt9WsiWNMeRLMaoupeuyAtJe/SjmnWCZu5JhaKFcViNPfhP2Xrs4PfmMDh7FAgdhhrGV2BS6kCVzR9uk1Dc2XTQmgNe2bUVTSfsAjU6RA3Wnn54LKGjqqrxO3ed2SlsKYwkmMNiuqgtXfEZFrNt2gBjDzst/U6EwG+8iGvzjzJGZekXHVICIUmPfhyzTtCPnpTgChMCu88pX6B3UoneSd5tRQ1DDUPYiJVqk5Aubn5Hs9mg5jaJTOh2top5pEMTnbNoevgFanQo9grMRdP9l+104zWQz3+kNsy0gpew8PPDpC0o3JOPOSympx1KzcKYEz4Vqq3qAbKyJTlQ+o0XcW3+YceorP2iQ8pkBMktRhPHrBNUKTFuJcWo5/j89e3s8/Bqptxd+9lUVrCKqTXvt7Nyj/YQF93CLX5Um4QWTXeWs9zUyqCZtUlF1CiYluc2pdqM42fCcpt7/Kq2/uZgx2dQbTqqfAZV1wsoKO/w0gndjnPapCpskqyaI+x679Ec+YXnjL+je3YqxI1CMl2AKk7l5TDPb5RxyJqMqkgqKiGOWcek/ETUcDrTGYOXmjlxyifZ+F5vNSRqGGoY/Ga0OqpnunsGJ5eZZXeTM/BFpbrqqVzp/F8RPr7CIEQeQ7W5yE87QNU65f0c6l7JbhVh4C7ep8s12NSa7RrMe477ngITDz831HeCnK9xnLOmC1BBvGxprzFKZxbKeqGeIachJEVeFshwzyZ0N0J1v3SzqEs+/xaf/Nh+nPwpaBocYJSJhgAaRyrsHVg9ZrrVs1TtugXlBWjCzmgXTYfPzaotTV0YKA+gAWUsJnTXaiyeKCdrGfw1pQndsPEbrr5EcKiH0sY0i/yYOuy9CAo22LUvPHEr/MmHtlNoGWPcrklJzChDWpDm4SbJcSDKoliV43tpGwsXWg0hcfKQYeilDeio/kw3S1r43X0Zv3Yib36w9jPTMNS4tuYFDzolHWqQ2iqZoUwWEr6ePziDllsYAJTKZS6C2Nyqz8dwdW9PiO0qn7LR1z5U+3aUQnSh/DWa84yaT2JSEfeNPxcmvH5IKGEAwX62KAEkYYUBJDMOVLQC3bG7uvzHIZ21AKZGVuutQNCQh5pFYdTf6gfBT8085O1XsGa2oFy/vC4SpDoMNYlFVcAZRHXvB0UseQqTMlHMHLFLM2gK9rmp5AesGo92AJ7Q433OUQrRxc0wdkdEhSEoZ6TUDGvPgYNnhl8WJWggjjJQmzxnQeNA2EggE/PUwID/OKTv9wRN6EgwViD4UO+aRWHshNUPgl8U07j3/iXNvS1seG/t516RINWEWVQlapmJIK59qFxKWoN29q3pl2/V0wB0Bft0fgVdzgA4jnEv7QCiFaKLnIxX6SvRHcpBGdhv/BmMfmscY2eGj2BzD5AV4kzYgp6zjg7/cUA3Uz/mGL2QCCOEdONQLldMs6SHbmAX8Z+x+KmZUijQduBfs/qzhdDjhV/p62rilJkIYs4yp5S0rvN+IaJe/drU6t1WYaBctqKKEf2DQ1V1BftMTEhuqrvgJUxNCtFVlyCfs8wJpXWfm9d5eaE7BxOToZ/AVwVYPVtoe9s/BXdCQ/UAWdniTNj8zEwmQkY3uPuVqDAZzIO0nbyumGZJCd0NMW+e/4zFaxYl4tyQTU1wxb3XUBpV5K0TwvVHW3KZcEXbvHICRvU5dXhM/BJ+M1e/JDJtYpywN2Ko3J/b7nZCWat9Grf8xHH4BiXheQm8oOiqylKhJma5oBLk557tON8fOKo2v6L6vHRCVWcuMjUZ+gn8jadAU/9Ixr//Uv8LkiG658xU4wgzU68ID7/B3FTb0QuMTRvMezQYKxByTBw/RmUW1dHhvK5EXgwMwMLri9x3z0JWfKFQ40vwQxc26l4X2KRoW2XGe/udTgz8ptHh/BK6Gazf7NzXXyDOd9XVzqA/Z1m0iq3gXRdo3uPBfgZdqW4vc5BfW9XZ3rce7+xffQ6V89JZsHTmIlOToa4u0uznYeXnhPb9/xop5GfoCfucuf0FYagIDz8hZKrt6PoNKyPXHLdhp0Mcffis4okb9mHSg90c/D/m94Au7LM6RFEX8uhVXyfMvtX4JWvpBu6gUMw4YZYmVEIxV43D8xq2b3GEgkmIaVBb7na9rmXYax+3ntSq8wpsPWU80z+/IVcCIQxRIpKqcYeHJ5FE68Ymplm06COOhKPe/UNWnqvo3d+8PZOS12F8BHEWxQlbnTNoZh03MieIyszcaynNShkOnZPb3begEuTV6K7lGS8mt6SnG7ev4bvvhzVnlzjyz+5rWGEAen+B269nsrhPvYNWvGjcX8ZihF/EUesxp3Pw5g/y0t+ZO5hNBnuvwVq3VGSciKOwJp2aRWV8ziEtFk13zDg1M+3yegbbRtY6ff36ZhLyqhuw3f0Q5fxOYXwjXn3z8jV0fhB+/spUWg472b+zOUc3waoM6pW/9VzcJw5WIBjSqCuxBSXYtc2+h12Ti57Jal6YzsxNl4pMa2EWv/5v/IaT1RzVqR0HT8d2FX1NMHZXuGUwq1ewM53x69ZS0FVarZioeprLYa8+ffNquxeY371Gc9aNQ5jClHnUAIJoGIFQzwE5TyuxhSXIYVZoGcNRhy/klb+CnoPM2gw7M/dzRoY1/SRVlTOuUzsqJolwJgl61VTORV2tr2vkvmZhTHXVM37ESYhr7dOvfaFru1+lVIAnQ9KuYFDviWcsp7KI7Af8AJgKrAQ+qZR6y2O/AaBy66xWSp3p3seLilM5bnXQuOSlrlHSVDu19mvdwP/77KVcvuh7NXWO4hLXGVkhiiM5CD/H6vxHzBf/iXs897GTWtxGd81G9XlXW03C8a/bvyhF+q9o4AemTFrO4KTGuXo6lb8MPKKUOgJ4pPzai51KqXeWNyNhUE29l53LuuJgFri1ns09k/jaLTfxjY9+Zk8o6iJmh14wx4ukFnAPkyltil+yXRJlOtwEVXwd0e84e5Nam0B3zSA9x7/OrzF3RoaFwFIkjinITwOo9zgH8QXCLKCy9t2twMditudJvQfkpFdiywNeN9mu/lFc+9Ov8doXCokumJOEn2DRdH1ZijD1iExLVxRLyZTpcDNnmZOToPPiN/fr/S1R0F2bza3mprqwAn3OMrhqAA4oJ/wVpUjHzA4WfKQBjOgpEmR6rvc4B/FNRluUUuOrXr+llNrXY79+4CmgH/hXpdTdPm3OBeYCtLW1zVi1alXdTTb1Nlmlgb6MsOJ332/mnAte4fXeqTWfRi2HHWbtZa/v+i1U41Uy2rSd5n4njL+64mnrbsd5alISOqoZy6+Mty5HIYoZKWqeRzVhz3HtWcK6jzdxwhmv0bTvweE7PUTJaq31VE1GIvJzEXnWY5sV4jht5Q5+BvimiByu21Ep1aWUmqmUmjlpklO0r96lqPNQ+TRp9FqPMP3oO1m3K9kFc6Jm/UJwZM62kWYzaK92vKJ6zv+DPtfLrVFENWOFrXcUtSprEtpZGMf/67OENbOF6e/6qbEwqLcjNSuCNIB6j3MAgYnXSqk/030mIm+IyGSl1HoRmQy8qWljXfnvayLyKHA88KppJysDbxqOnDB9aGQB4MZZSMP7/dHvOJNRzT307K6tU91G5Kz4yAQNhpU6QEFCRtfOplbYWDXrn3qpvjLp9pF7i8f5tem3yP2eZS1d2oCfszdq0lyln3Gd45WSF36sPUdY+wnhncc9wqgjP2DUrlv73lvTf2g9b6Bf4Kq6qinUd5yL60O4Bzi//P/5wE/cO4jIviIysvz/ROAUYHnYAzViTG+eCdJ6zr+olYIMvnu9FszJApPBcNW4YC3BtDifnwDa3TR49h82e/dzrmUtgT0F9brudUpgJ52XEUc7M2X1bGHtOQXeecKvtMKg8/5Omq5pQq4Wmq5povP+zlw4UrPCRAOo9zgXVyD8K/AhEXkZ+FD5NSIyU0RuKu9zNLBERJ4GfoHjQwgtEOrJUFVp/W6+BQvgC/OKjGreiUiJKaNqF8zJCpNsXJMV00yL8wUJoGqBEcYko1udDYExfXtn4WFLclRIKkcjLCvPK7D+L4q886Tf0fK2Uzz36by/k4VLFjJQzkUYUAPO6wFvH2YjR/DpaATTsy1uF4BfMSu3SpdWfHI9qD6XluZevnTx5/msLOawhSUKPmsOpEV1MTdBb9IJcpaaFOcLcmK7j2HqMNflY7iPr8PvOGnkaARRaoJXvlhgy/FFjnvf44w85J3afZuuadojDAbxH2tge62vodFzfOqJLW5nQNRZvp/qWh021sjZzG7c59Lb18I1C25j8Y4LeeY/C+wOWpUsofyFatzZuLqwzSB/g0lxvrA1j0xNMn6aR5BWErQWQRo5Gn7smgBPfavA7iMmcsIZr/kKA8BbGAB86B88387SkZolebc2DAuBYDJY634oE9W1qysfSSWQzA3n1WdFgWvuuJF9WmeytEvYdqT3d8PmL0Qxc8xZZjawe2Fq4vGqeRTGfKM7ttdqZc39wT4C3YD/2XL9pSRyNEzZciwsvR4mjHw/x3z+daNooqJ4h7UVj/th7s0oSdEIk8ZhYTIKiu/1yzOomE3ikNUlTi71Xf+ZUrDh3n/gpYFvcFgXTH5w8OdTWcEqptZ8zyt/IY6ZI+53ky5JYcqgKCNgQo/jSA46vp+5CRyB5fV5kmUwFPD6WcKq8+Dtrdcw4c//0fi7FR+Cm+GUsJZVPlUck9GwEAhBA5zfD6ULz3TvB/Wvd5TUDRd0PRwhqThgwmr+5pTL+OtHvkdzuf5RgQGUh+IplCgxeJYYZ3Gcij+hWHKWzWzPeGDPGpMaSG6hkKQPYfc+8PLfFuk5tMixJz1sHFZaTef9nXQt7WJADVCUInNnzB02wgCCx6HkjmN9CL4ElZ7wSxjxWp/Yzdy50ZNKkrQpJpX6ruvzUUdVq7zCG5vaueKBm5j/uc+w6V3OPro8Ba/3oyyOE7by5lDBJNJKkZx5q5oN74Elt8DIA6dzwtnrIwkDgAUfWUD/Ff2oKxX9V/QPK2EAjVECZ1gIhKDBOuiHqg7P1Nk7o4SUJW1TTOqG053Liy/W7rurfxT/2bWAl/6uwPOXF7i65TJaGVwuVZe/EKXonc6W/sXT/M6o8XGve+BF+9Zk8w127QfPXVPktb8scsz+C3jb3D9QHL1fvEaHEe7J3lGatSZy5UBXSuV2mzFjhkqKjg6likWlwPnb0TH4M2e4H7xV75MGlf64t2IxWntpn4dX25Wtb8t69fINx6lf34n6zvTZqo0VShhQ7axQdzDb80t3TEe1Xobiqr1b62XO+7oDyZWD99+zXen9vTumo9ovdb7Xfql/242yRbluYbaSoNaeifr13ahXbzxJ9e/YlMwNNIzQPYvTpunHoaQAlqiIY+6w8CGYUI8cgjRsitXnUTmGUsmck4mPYvvjd/Di8xcjvbs57IYS+/7Bv82wDl4/W3pxAEqFve1AurH5STmno7SThmNcAZtPhNe+UKBIC0eecAdj3nlWvEaHKfUsyGmdyg1KmjdNGhVaTdvs7Bhg8a1b2bpzPAeMX8O/tHyFi/4YLcPZHZUzZjfsGIG++lyZsIvAROlXTfKaqo0aChq465FQ5sXWafDavAJ9Ewocus/fMPGMryGFYWFR9iXqRDErB7L3sa1TuSFJs7phGnkRJn6Szk5YeH2RLTv3Q1Hgj1va+cvNN/L1s+fQE7IS8qLpcKGr9s+OkWbf7RmxV4i4CRub75Ur4VmBVZy+nns2dJ4enEwG2SeUuemeCsu+WmT5VQUOPOA8Zn5iO5M++nUrDND7+EwCQEz8eXlMUrMaQp055hhYXlXZado0eO65aG25zUU60vzJdVrPhNFvcPeiA5n4+yIH/2CAMa8Ft+VnHtLF3Q9C4alJFAfg1rvN6wN5zeB16yVU92+/nmANJanlRcOy/UhY86kib51Qom3bRzjonNspto4fUuVX4qK7l6vRadxB2nSaa6xYDaFB6ewcLAzAeV2ZKYSZQbhnMzrSDnHTHX9T9wGcdPIrtBzxPp75N+Gpa4tsPIU9y3V64TeTV+wNsSxqjjmhxztUc6BovgqZbgZfDKjnpMRMQ0lqeVETSgV48/3w5HVFnv2XAmMO/xAnvW8lh5x37x5hkPdM2iwxrVLgRZA2nZfKBm6shlBHoiTE6WYQJrMZv+8nhYlfpNS7gw0Pfom1W26hb9QuptypmPwANA2OVvXVEKpn2X52eIDzP+YIAb82dGgzhJWT/+C3cI+JhpKFD6FvLKz/C+H1WdCyvZUpk77AxNPnU2huGbRfvVcmzBumz1SUITRNH4PVEBoUv0SysDMIE80gixoxJn6RQssYDjjrOmZcuJOj27rY/qdTeGwxvPClIm+dsFdrmP+IU+fHzQhX7R+/ktFzljkzYy9MfAm6mXr7Vn0BvAomGkqcctd+lAqw+V3w/GVFfr8Iut/bxrGH3s7xF+1g/zP/o0YYQHKJjXm0jUfBxJcXVePOa5Ka1RDqiN+MzO8hVKo2+kG3fz1md1Hs0LtWLeGNX17Jm+oRdo/dxaRfFtj/ZyXua6qt/fPJZ+GBo+KHqppoCCYz+M7T4foT8SwbAfE0lJr+MJvL+SqraaON1cznsj1rVCiBrcfCmx8qsOE9JVq2tLB/84c54IP/wogpxwa2nYSGMNTWHw/yy0U9r7z6ECIlL2S1JZmYlkf8Esn8ktZ039O11Wh0P/eQWnHz+9Rji5rU7xaLenWuqG1HOAlTUZKy4iZymSS3+e2jS6aTK8MljN3BbNXKjkFvt7JD3Th5tnqls6B++wNRj9/WrFZ+91TV8+IvlFL+CZlukkhsTDrZMg3CXJMkvpdVexWIkZhW90HfbxvqAkEp/U3hl+moGzNE0s+CjHo+USgNDKhtT3xfvdL1LvXYoqL69V2oyZrBtf3S+IN6Wlv7pdH6XNMOKzw/OmDCSvXqjSerHU/dXfNbhB3g4/5+fqeQB+pVlSBLrEDIkLSkusmx/IRBHh66pB62jvs6VPHqouIqVPHqouq4z2lg56u/VSQ02056u4PZql1TriOuhtK7H+rN96GEgVC/u+lsPcl7Ou8aQpz+ZfnsxyGOQIjlQxCRTwBX4aybfKJSytPgLyKnAdcCReAmpdS/mrSfNx9Cve2jQVEP9Y4GScQGHVA3X7cU4wHNcOeRwthXi4x9foAxLyrGvgQjN4c9i/BUFgXqYW/SQSvdg9agNik1oYBdE2HHkbD9KNh+dBM7Dhug1KzYZ/2+/OnfvsCm7v1rjq+7vn6RLBW/U6W0iZs82saTIGp0T97Pq5q6la4QkaOBEnAD8HdeAkFEisBLwIeAtcATwGyl1HL3vm7yJhDqHZbndzND/W9O04fNz+msG/CLUqT/in69wJgxj/848lNsf+Fedrz5a7aXXmD7gdso9MOjL8B1vbC+AFN64eon4LzfQlNv3DN2CLMoEEB/K/ROhp0HOn97pxTYeUiB7Yf1gwhj3xjHmKajGbv/exh79CxGHnYyUiiEHpRMwya9iHNP5zm5rVDwHvhFoOSTW1LvZz8McQRCU5wDK6WeL3fAb7cTgVeUUq+V9/0+MAsIFAh5I6mwvKj4RRPVWxiAvn/udP3qQa2S/ARO/3Vr71ber9TQ1y20MurID1CZQ6tSib/93jl8e/dd9Jf7sHYUdL4fVlwIp42BljeLtLwhNG8p0bS1RNN2aN4BTTug20S4PwAADuBJREFUaTvcuW42V2/4KmtLbRxSWM01Iy7jM03lqJ4CUITVb7V59nk1bbx2EfSPL9I3Xug9oMTO/UuURkLLxiZGde9DC5NpGXME++57LGOmncnIthnashHVSU0mg63J4k464tzTlXLweUQ3/w2aF9f72c+KRMJOReRR9BrCOcBpSqnPl1+fC5yklLpE09ZcYC5AW1vbjFWrVsXuX1LUe5aQd7XVpH9B1zBIQwiLXO09WSlKke4Ll9C76vf0bniGvp1v0r97s7MNbKFfurnr9x9l/ve+za7de01BI0d28w+XXMyfv38xKJCS8PGLX+ONjVNrjjFp7Doe//Z5NI/Yj6aWA2g54DhGtb+b5snTMqsVZBqe7CaPM98kiGoyMjHX5kUTSjXsFPg58KzHNqtqn0eBmZrvfwLHb1B5fS7wbRMHR96cynmIUMi7Yyuof0EO8Y77OjydxtO+M83T0ezbF6+2zp6tGLdCwUDg9QvjlK33fWGK7pwaoe9JEBTOHXbNlDxeN+odZRQgEE4GHq56/RXgKybt5k0gKJX/ATnvmAyy7iijad+Z5ikkgoRCpY1BwqB5h/FDHCS8qmmU+yJoYMtz35MgbDi3yOBQ8CCBmodoqrwLhCbgNeBQYATwNHCMSbt5FAiWeESZTdcM7OWteLX/01fznXErQj3EaYdQ1kuINIrwSguv8w8a6MNquvUkjkCIZcgUkbNEZG1ZC7hfRB4uv3+QiDxQNkn1A5cADwPPAz9USkUs8GxJmqzrzkRZezrI0ayjKK56EVu9nb8623Ca61WkXVnU73etXiO8vz8fdu8kCbqnvc4/yLfiriGW11pEsYkqSbLYhqqGkJcZWqPYvqNqCDU+hJAaglLp/VZpah+N8rumQdRzN/GtJHGcLKDeJqO0tqEoEPJ0I4UZlOopxHSOZlPH8h6BMutChfTl4tqnaXLIe7awF0ndX1HP3cS3klafk8YKhAbCxCmV1Y3VSLMhXTmL0O3k5CFOc9BOU9iEwfRaJ3l/xTl3P6GQl8HeBCsQGoggtTTLG9B0UGrEGWfeSVPI5uH3CnN+SfY3ibbyMmmIShyBYBfISQhT56yp0ymLpfRMnKadncMnSzNpghy706YN3n/atGQcvGk6w00Js8BTkvdXEuc+1J3uvkSVJFlsjaIhhJkNhVnLIOk+6spsR03GsRqCnqB7Im0zXL1nuWHu66Q1mnqfe73BmozqS9gbOusEl7QiL4bbgxaGoHsiD2adNAkbsGDvr+SIIxCsySgBwqq81SppR4f3Pkmq92HXZ67gp7LnpX5SXgm6J4a6GS6M6SZKboolHeyaygkQt+hd2uWCky7oNVQLnyVJYBG/YXBt81wGeygTp7id1RASIK4jK20nVtSsyjw4JxuVoGs3HK7tsHbONipRbU1ZbI3iQ1Aq346sODbaKM5oi0PQNWqka9hIfR3uYJ3KliCSfKDr5QS0g1J9sE7fxsIKBEumxCkPEHVADzsoDXfhkeT5D/WIqKGGFQiWTAkTY14h7iwzyTDGoS4skp7RR/m9LfUjjkCwUUaW0ESJkIkbVRMmUsrvWLp1hodSmGPSEUzDISJqKGGjjCyZEiVCJm7cfZhIKb9jRc3JaCSSznEYDhFRFgcrECyhiZJIFHdBkTCDkt+xhnpCGCS/eItNHBs+WIEwhMhy9bOwMeZJ5GqYDkp+xxqyK11VkcaM3uYUDBOiOh+y2KxT2ZxGCA3M0pnrlz+R9+uUBElf66HuiB9KUC+nsoh8ArgKOBo4USnl6QEWkZXAdmAA6FeGDg/rVDbHOv7MsSUVwlFZ/9mNNRvlkzhO5bgC4WigBNwA/F2AQJiplNoYpn0rEMyJWq/IYgnCTjYaizgCoSnOgZVSz5c7EKcZSwLoHKZDyTZuqQ/DwRFvccjKqayAn4rIUhGxwWopYEMD80+STv8sAwiGgyPe4hAoEETk5yLyrMc2K8RxTlFKnQCcDvyliLzP53hzRWSJiCzZsGFDiEMMb/IaGpjlwJXnPlXs8NXrISxcGO3YSbZlgp1sDCOieqOrN+BRHB+Byb5X4fgbbJTRECePET316lPeFpIPQ0eHUiL5+Q0t/pDnFdNEZLSIjK38D/w58Gzax7XUnzxmBderT1Hs8DpNJkubfkUbcQcm/PKX+dP8LPGJ5VQWkbOAbwOTgPtF5Cml1IdF5CDgJqXUGcABwF1lx3MT8D2l1EMx+21pAPLojKxXn8I6/d2hnhWzUJS24qATlMuXe/et3iZKSzxiaQhKqbuUUgcrpUYqpQ5QSn24/P66sjBAKfWaUuq48naMUmp+Eh23mFMvO34enZH16lNYO7yfJpOlTT+MoBxK9aCGK7Z0xRAnawdkNXl0RtarT2Gd/n6aTJYBBGEEpQ1DHQJEdT5ksVmnshl+ZQXqvbhJHkse5LFPbur9u1XQOeHz0DeLN+TZqWxJlyANoN52/DwWRcu6T1FMdnnRrry0kWnTvPe1YahDgKiSJIvNagjBBM0k8zLTrNAIs/MkiRPmmudrlee+DXewS2gOX4KWN8xTLsBwXNrSCmRL1sQRCLHCTi31JygEsWIOyUN1z6AcAF2YZR7MTFGpt8muGr9Q1ka+xpbksD6EBsfE1hzXZp5U2Gqel7ZMKzQ3T6G39b7GlvxjBUKDk3YIYpJhq3ld2jLN0Ny8OIchX9qKJadEtTVlsVkfQv0xsYGb2qX9fAj1tLWnfey82O3z5s+wpAM27NSSFkGzyjCzaz9tpp4z6bRnzkEmu6wyyfOkrVhySlRJksVmNYT6k2VYq986yO7jJDnTrufMOesosOprWV3B1EYcDR2wYaeWtAgasILCXtM6fpIDZz1Dc+sljPIUjmxJljgCIdaaymlj11TOB36L0qe93q6u/aSP43eOaVKvtbDtOslDlzhrKluBYImFO7a9QlKRTibLdef4Fg6kXgNzvQSRJX3iCATrVLbEIu2w16B4fb/1BNJy1CbZdr0cvXnKj7DkiKi2piw260OwRPEhpGkfT6PteoSlWh/C0AXrQ7AMZart+xX87PxpmmGGku29Xn4TS7pYH4LFUkWa9nFre7fkHetDsFiqSNM+bm3vlqFMLIEgIt8QkRdE5BkRuUtExmv2O01EXhSRV0Tky3GOabEEkaaj1mb7WoYycTWEnwHHKqXeAbwEfMW9g4gUgeuA04FpwGwR0ay5ZLHEJ83IpyzXM7ZYsiYxH4KInAWco5Sa43r/ZOAqpdSHy6+/AqCU+lpQm9aHYLFYLOGI40NIcoGczwE/8Hh/CrCm6vVa4CRdIyIyF6go4LtE5NnEepgOE4GN9e6EAbafyWL7mSy2n8lxVNQvBgoEEfk5cKDHR5crpX5S3udyoB9Y5NWEx3tatUQp1QV0ldtdElXSZUUj9BFsP5PG9jNZbD+TQ0Qim1UCBYJS6s8CDn4+8FHgVOVtf1oLHFL1+mBgXZhOWiwWiyV94kYZnQZ8CThTKdWj2e0J4AgROVRERgCfBu6Jc1yLxWKxJE/cKKPvAGOBn4nIUyJyPYCIHCQiDwAopfqBS4CHgeeBHyqlnjNsvxFWe22EPoLtZ9LYfiaL7WdyRO5jrjOVLRaLxZIdNlPZYrFYLIAVCBaLxWIpkyuB0AilMETkEyLynIiUREQbfiYiK0VkWdm3knl2XYh+1rWsiIjsJyI/E5GXy3/31ew3UL6WT4lIZkEJQddHREaKyA/Kn/9eRKZm1TdXP4L6eYGIbKi6hp+vQx9vEZE3dblF4vCt8jk8IyInZN3Hcj+C+vkBEdladS2vqEMfDxGRX4jI8+Xn/Ise+4S/nlHrZqexAX8ONJX//zrwdY99isCrwGHACOBpYFqGfTwaJ/HjUWCmz34rgYl1vJaB/az3tSz34d+AL5f//7LXb17+bEcdrmHg9QE6gevL/38a+EFO+3kB8J2s++bqw/uAE4BnNZ+fATyIk7v0buD3Oe3nB4D76nwtJwMnlP8fi1M6yP2bh76eudIQlFI/VU5UEsBjODkLbk4EXlFKvaaU2g18H5iVYR+fV0q9mNXxomLYz7peyzKzgFvL/98KfCzj4/thcn2q+/9j4FQRk4U/EyUPv2MgSqlfAZt9dpkF3KYcHgPGi8jkbHq3F4N+1h2l1Hql1JPl/7fjRHBOce0W+nrmSiC4+ByOdHPjVQrDfSHygAJ+KiJLy+U48kgeruUBSqn14NzkwP6a/VpEZImIPCYiWQkNk+uzZ5/yZGYrMCGT3nn0oYzud/x42XTwYxE5xOPzepOH+9GUk0XkaRF5UESOqWdHymbK44Hfuz4KfT2TrGVkRNalMKJg0kcDTlFKrROR/XHyNF4ozzwSI4F+pn4twb+fIZppK1/Pw4D/FZFlSqlXk+mhFpPrk8k1DMCkD/cCi5VSu0RkHo5W86ep9ywcebiWJjwJtCuldojIGcDdwBH16IiIjAH+B7hUKbXN/bHHV3yvZ+YCQTVAKYygPhq2sa78900RuQtHrU9UICTQz0zKivj1U0TeEJHJSqn1ZXX2TU0blev5mog8ijMjSlsgmFyfyj5rRaQJGEf25obAfiqlNlW9vBHHR5c3GqLMTfXAq5R6QEQWiMhEpVSmRe9EpBlHGCxSSt3psUvo65krk5EMkVIYIjJaRMZW/sdxluexamseruU9wPnl/88HajQbEdlXREaW/58InAIsz6BvJtenuv/nAP+rmcikSWA/XbbjM3FsznnjHuC8cnTMu4GtFXNinhCRAyt+IhE5EWcc3eT/rcT7IMDNwPNKqf/U7Bb+etbTU+7hOX8Fx+b1VHmrRG8cBDzg8p6/hDNDvDzjPp6FI3l3AW8AD7v7iBPt8XR5ey7rPpr2s97Xsnz8CcAjwMvlv/uV358J3FT+/0+AZeXruQy4KMP+1Vwf4BqcSQtAC/Cj8r37OHBY1tfQsJ9fK9+LTwO/AN5ehz4uBtYDfeV78yJgHjCv/LngLKb1avl31kbx1bmfl1Rdy8eAP6lDH9+DY/55pmq8PCPu9bSlKywWi8UC5MxkZLFYLJb6YQWCxWKxWAArECwWi8VSxgoEi8VisQBWIFgsFouljBUIFovFYgGsQLBYLBZLmf8PrkbxzDI9MhQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_model(model2,testx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU\n",
      "At iteration 0\n",
      "Training loss : 0.74870116\n",
      "Training accuracy : 0.4908\n",
      "Validation loss : 0.75464624\n",
      "Validation accuracy : 0.48\n",
      "At iteration 100\n",
      "Training loss : 0.6967418\n",
      "Training accuracy : 0.5925\n",
      "Validation loss : 0.6992114\n",
      "Validation accuracy : 0.578\n",
      "At iteration 200\n",
      "Training loss : 0.68586004\n",
      "Training accuracy : 0.6163\n",
      "Validation loss : 0.68691874\n",
      "Validation accuracy : 0.61\n",
      "At iteration 300\n",
      "Training loss : 0.6822022\n",
      "Training accuracy : 0.5765\n",
      "Validation loss : 0.6827547\n",
      "Validation accuracy : 0.578\n",
      "At iteration 400\n",
      "Training loss : 0.6795778\n",
      "Training accuracy : 0.5576\n",
      "Validation loss : 0.67998815\n",
      "Validation accuracy : 0.546\n",
      "At iteration 500\n",
      "Training loss : 0.67694056\n",
      "Training accuracy : 0.5494\n",
      "Validation loss : 0.67730606\n",
      "Validation accuracy : 0.55\n",
      "At iteration 600\n",
      "Training loss : 0.6741164\n",
      "Training accuracy : 0.5495\n",
      "Validation loss : 0.674454\n",
      "Validation accuracy : 0.55\n",
      "At iteration 700\n",
      "Training loss : 0.6710631\n",
      "Training accuracy : 0.5537\n",
      "Validation loss : 0.6712429\n",
      "Validation accuracy : 0.55\n",
      "At iteration 800\n",
      "Training loss : 0.6678069\n",
      "Training accuracy : 0.5591\n",
      "Validation loss : 0.66765296\n",
      "Validation accuracy : 0.552\n",
      "At iteration 900\n",
      "Training loss : 0.6642503\n",
      "Training accuracy : 0.5654\n",
      "Validation loss : 0.6638306\n",
      "Validation accuracy : 0.568\n",
      "At iteration 1000\n",
      "Training loss : 0.6605078\n",
      "Training accuracy : 0.5706\n",
      "Validation loss : 0.6598623\n",
      "Validation accuracy : 0.572\n",
      "At iteration 1100\n",
      "Training loss : 0.6566744\n",
      "Training accuracy : 0.5749\n",
      "Validation loss : 0.65581185\n",
      "Validation accuracy : 0.582\n",
      "At iteration 1200\n",
      "Training loss : 0.6526896\n",
      "Training accuracy : 0.5819\n",
      "Validation loss : 0.65176183\n",
      "Validation accuracy : 0.594\n",
      "At iteration 1300\n",
      "Training loss : 0.64853036\n",
      "Training accuracy : 0.5872\n",
      "Validation loss : 0.6475601\n",
      "Validation accuracy : 0.59\n",
      "At iteration 1400\n",
      "Training loss : 0.64429563\n",
      "Training accuracy : 0.5932\n",
      "Validation loss : 0.64306253\n",
      "Validation accuracy : 0.6\n",
      "At iteration 1500\n",
      "Training loss : 0.6402161\n",
      "Training accuracy : 0.5969\n",
      "Validation loss : 0.6385985\n",
      "Validation accuracy : 0.604\n",
      "At iteration 1600\n",
      "Training loss : 0.6364606\n",
      "Training accuracy : 0.6008\n",
      "Validation loss : 0.63438994\n",
      "Validation accuracy : 0.606\n",
      "At iteration 1700\n",
      "Training loss : 0.6332714\n",
      "Training accuracy : 0.6039\n",
      "Validation loss : 0.6307836\n",
      "Validation accuracy : 0.604\n",
      "At iteration 1800\n",
      "Training loss : 0.63066775\n",
      "Training accuracy : 0.6062\n",
      "Validation loss : 0.6278636\n",
      "Validation accuracy : 0.608\n",
      "At iteration 1900\n",
      "Training loss : 0.62838775\n",
      "Training accuracy : 0.6076\n",
      "Validation loss : 0.62541264\n",
      "Validation accuracy : 0.606\n",
      "At iteration 2000\n",
      "Training loss : 0.626356\n",
      "Training accuracy : 0.6102\n",
      "Validation loss : 0.6232222\n",
      "Validation accuracy : 0.608\n",
      "At iteration 2100\n",
      "Training loss : 0.62451565\n",
      "Training accuracy : 0.612\n",
      "Validation loss : 0.62118816\n",
      "Validation accuracy : 0.606\n",
      "At iteration 2200\n",
      "Training loss : 0.62284714\n",
      "Training accuracy : 0.6149\n",
      "Validation loss : 0.61932915\n",
      "Validation accuracy : 0.608\n",
      "At iteration 2300\n",
      "Training loss : 0.62133527\n",
      "Training accuracy : 0.6157\n",
      "Validation loss : 0.6176781\n",
      "Validation accuracy : 0.606\n",
      "At iteration 2400\n",
      "Training loss : 0.61994433\n",
      "Training accuracy : 0.6172\n",
      "Validation loss : 0.61617345\n",
      "Validation accuracy : 0.606\n",
      "At iteration 2500\n",
      "Training loss : 0.6186179\n",
      "Training accuracy : 0.6182\n",
      "Validation loss : 0.6148445\n",
      "Validation accuracy : 0.606\n",
      "At iteration 2600\n",
      "Training loss : 0.6173928\n",
      "Training accuracy : 0.6186\n",
      "Validation loss : 0.61359\n",
      "Validation accuracy : 0.612\n",
      "At iteration 2700\n",
      "Training loss : 0.61628085\n",
      "Training accuracy : 0.6197\n",
      "Validation loss : 0.61249965\n",
      "Validation accuracy : 0.612\n",
      "At iteration 2800\n",
      "Training loss : 0.61523724\n",
      "Training accuracy : 0.6212\n",
      "Validation loss : 0.61150557\n",
      "Validation accuracy : 0.614\n",
      "At iteration 2900\n",
      "Training loss : 0.61427104\n",
      "Training accuracy : 0.6227\n",
      "Validation loss : 0.610541\n",
      "Validation accuracy : 0.616\n",
      "At iteration 3000\n",
      "Training loss : 0.6133871\n",
      "Training accuracy : 0.6238\n",
      "Validation loss : 0.60957426\n",
      "Validation accuracy : 0.616\n",
      "At iteration 3100\n",
      "Training loss : 0.6125686\n",
      "Training accuracy : 0.625\n",
      "Validation loss : 0.6086842\n",
      "Validation accuracy : 0.62\n",
      "At iteration 3200\n",
      "Training loss : 0.61180925\n",
      "Training accuracy : 0.626\n",
      "Validation loss : 0.6078606\n",
      "Validation accuracy : 0.624\n",
      "At iteration 3300\n",
      "Training loss : 0.6111031\n",
      "Training accuracy : 0.6265\n",
      "Validation loss : 0.6070657\n",
      "Validation accuracy : 0.622\n",
      "At iteration 3400\n",
      "Training loss : 0.61041594\n",
      "Training accuracy : 0.6273\n",
      "Validation loss : 0.6062788\n",
      "Validation accuracy : 0.624\n",
      "At iteration 3500\n",
      "Training loss : 0.6097545\n",
      "Training accuracy : 0.6282\n",
      "Validation loss : 0.605525\n",
      "Validation accuracy : 0.626\n",
      "At iteration 3600\n",
      "Training loss : 0.60912377\n",
      "Training accuracy : 0.6287\n",
      "Validation loss : 0.6048121\n",
      "Validation accuracy : 0.626\n",
      "At iteration 3700\n",
      "Training loss : 0.6085109\n",
      "Training accuracy : 0.6291\n",
      "Validation loss : 0.6040632\n",
      "Validation accuracy : 0.626\n",
      "At iteration 3800\n",
      "Training loss : 0.6079264\n",
      "Training accuracy : 0.6292\n",
      "Validation loss : 0.60334474\n",
      "Validation accuracy : 0.628\n",
      "At iteration 3900\n",
      "Training loss : 0.60737866\n",
      "Training accuracy : 0.6295\n",
      "Validation loss : 0.60265684\n",
      "Validation accuracy : 0.628\n",
      "At iteration 4000\n",
      "Training loss : 0.6068559\n",
      "Training accuracy : 0.6306\n",
      "Validation loss : 0.6019817\n",
      "Validation accuracy : 0.628\n",
      "At iteration 4100\n",
      "Training loss : 0.60634834\n",
      "Training accuracy : 0.6311\n",
      "Validation loss : 0.6013208\n",
      "Validation accuracy : 0.626\n",
      "At iteration 4200\n",
      "Training loss : 0.605859\n",
      "Training accuracy : 0.6314\n",
      "Validation loss : 0.6006744\n",
      "Validation accuracy : 0.626\n",
      "At iteration 4300\n",
      "Training loss : 0.60540324\n",
      "Training accuracy : 0.6321\n",
      "Validation loss : 0.6000681\n",
      "Validation accuracy : 0.628\n",
      "At iteration 4400\n",
      "Training loss : 0.6049666\n",
      "Training accuracy : 0.6328\n",
      "Validation loss : 0.59948415\n",
      "Validation accuracy : 0.63\n",
      "At iteration 4500\n",
      "Training loss : 0.6045548\n",
      "Training accuracy : 0.6337\n",
      "Validation loss : 0.59891367\n",
      "Validation accuracy : 0.632\n",
      "At iteration 4600\n",
      "Training loss : 0.6041487\n",
      "Training accuracy : 0.6336\n",
      "Validation loss : 0.5983533\n",
      "Validation accuracy : 0.632\n",
      "At iteration 4700\n",
      "Training loss : 0.6037652\n",
      "Training accuracy : 0.6337\n",
      "Validation loss : 0.59781206\n",
      "Validation accuracy : 0.632\n",
      "At iteration 4800\n",
      "Training loss : 0.6033995\n",
      "Training accuracy : 0.6339\n",
      "Validation loss : 0.59730816\n",
      "Validation accuracy : 0.636\n",
      "At iteration 4900\n",
      "Training loss : 0.6030476\n",
      "Training accuracy : 0.6343\n",
      "Validation loss : 0.5968159\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5000\n",
      "Training loss : 0.6027023\n",
      "Training accuracy : 0.6345\n",
      "Validation loss : 0.5963397\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5100\n",
      "Training loss : 0.6023683\n",
      "Training accuracy : 0.6348\n",
      "Validation loss : 0.5958915\n",
      "Validation accuracy : 0.64\n",
      "At iteration 5200\n",
      "Training loss : 0.6020482\n",
      "Training accuracy : 0.6349\n",
      "Validation loss : 0.59547436\n",
      "Validation accuracy : 0.64\n",
      "At iteration 5300\n",
      "Training loss : 0.6017388\n",
      "Training accuracy : 0.6348\n",
      "Validation loss : 0.59508467\n",
      "Validation accuracy : 0.64\n",
      "At iteration 5400\n",
      "Training loss : 0.60143846\n",
      "Training accuracy : 0.6357\n",
      "Validation loss : 0.5947003\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5500\n",
      "Training loss : 0.60115\n",
      "Training accuracy : 0.6358\n",
      "Validation loss : 0.5943162\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5600\n",
      "Training loss : 0.6008767\n",
      "Training accuracy : 0.6362\n",
      "Validation loss : 0.59394795\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5700\n",
      "Training loss : 0.6006126\n",
      "Training accuracy : 0.6365\n",
      "Validation loss : 0.5935939\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5800\n",
      "Training loss : 0.6003592\n",
      "Training accuracy : 0.637\n",
      "Validation loss : 0.5932527\n",
      "Validation accuracy : 0.638\n",
      "At iteration 5900\n",
      "Training loss : 0.60011053\n",
      "Training accuracy : 0.638\n",
      "Validation loss : 0.5929371\n",
      "Validation accuracy : 0.64\n",
      "At iteration 6000\n",
      "Training loss : 0.5998734\n",
      "Training accuracy : 0.6384\n",
      "Validation loss : 0.59264034\n",
      "Validation accuracy : 0.64\n",
      "At iteration 6100\n",
      "Training loss : 0.59963924\n",
      "Training accuracy : 0.6388\n",
      "Validation loss : 0.59235674\n",
      "Validation accuracy : 0.642\n",
      "At iteration 6200\n",
      "Training loss : 0.5994149\n",
      "Training accuracy : 0.6391\n",
      "Validation loss : 0.5920938\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6300\n",
      "Training loss : 0.59920406\n",
      "Training accuracy : 0.6393\n",
      "Validation loss : 0.5918561\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6400\n",
      "Training loss : 0.5989958\n",
      "Training accuracy : 0.64\n",
      "Validation loss : 0.59163666\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training loss : 0.5987796\n",
      "Training accuracy : 0.6402\n",
      "Validation loss : 0.5914394\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6600\n",
      "Training loss : 0.5985569\n",
      "Training accuracy : 0.6403\n",
      "Validation loss : 0.59125906\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6700\n",
      "Training loss : 0.59834313\n",
      "Training accuracy : 0.6407\n",
      "Validation loss : 0.5911017\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6800\n",
      "Training loss : 0.59813416\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.5909582\n",
      "Validation accuracy : 0.644\n",
      "At iteration 6900\n",
      "Training loss : 0.59792256\n",
      "Training accuracy : 0.6409\n",
      "Validation loss : 0.59083486\n",
      "Validation accuracy : 0.644\n",
      "At iteration 7000\n",
      "Training loss : 0.5977238\n",
      "Training accuracy : 0.6411\n",
      "Validation loss : 0.59070665\n",
      "Validation accuracy : 0.646\n",
      "At iteration 7100\n",
      "Training loss : 0.5975359\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.5905603\n",
      "Validation accuracy : 0.646\n",
      "At iteration 7200\n",
      "Training loss : 0.5973651\n",
      "Training accuracy : 0.6415\n",
      "Validation loss : 0.5904152\n",
      "Validation accuracy : 0.644\n",
      "At iteration 7300\n",
      "Training loss : 0.5972014\n",
      "Training accuracy : 0.6415\n",
      "Validation loss : 0.5902777\n",
      "Validation accuracy : 0.642\n",
      "At iteration 7400\n",
      "Training loss : 0.5970432\n",
      "Training accuracy : 0.6416\n",
      "Validation loss : 0.59014314\n",
      "Validation accuracy : 0.64\n",
      "At iteration 7500\n",
      "Training loss : 0.5968952\n",
      "Training accuracy : 0.6411\n",
      "Validation loss : 0.59002787\n",
      "Validation accuracy : 0.64\n",
      "At iteration 7600\n",
      "Training loss : 0.5967458\n",
      "Training accuracy : 0.6414\n",
      "Validation loss : 0.58993196\n",
      "Validation accuracy : 0.64\n",
      "At iteration 7700\n",
      "Training loss : 0.596599\n",
      "Training accuracy : 0.6416\n",
      "Validation loss : 0.5898546\n",
      "Validation accuracy : 0.64\n",
      "At iteration 7800\n",
      "Training loss : 0.59645957\n",
      "Training accuracy : 0.6416\n",
      "Validation loss : 0.58975583\n",
      "Validation accuracy : 0.642\n",
      "At iteration 7900\n",
      "Training loss : 0.59632844\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.58965117\n",
      "Validation accuracy : 0.642\n",
      "At iteration 8000\n",
      "Training loss : 0.59620434\n",
      "Training accuracy : 0.6411\n",
      "Validation loss : 0.5895451\n",
      "Validation accuracy : 0.64\n",
      "At iteration 8100\n",
      "Training loss : 0.59608394\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.58945227\n",
      "Validation accuracy : 0.638\n",
      "At iteration 8200\n",
      "Training loss : 0.5959639\n",
      "Training accuracy : 0.6408\n",
      "Validation loss : 0.5893753\n",
      "Validation accuracy : 0.638\n",
      "At iteration 8300\n",
      "Training loss : 0.5958434\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.58930564\n",
      "Validation accuracy : 0.638\n",
      "At iteration 8400\n",
      "Training loss : 0.5957322\n",
      "Training accuracy : 0.6416\n",
      "Validation loss : 0.58923256\n",
      "Validation accuracy : 0.638\n",
      "At iteration 8500\n",
      "Training loss : 0.59562534\n",
      "Training accuracy : 0.6411\n",
      "Validation loss : 0.5891578\n",
      "Validation accuracy : 0.64\n",
      "At iteration 8600\n",
      "Training loss : 0.5955196\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.5890928\n",
      "Validation accuracy : 0.64\n",
      "At iteration 8700\n",
      "Training loss : 0.59541637\n",
      "Training accuracy : 0.6412\n",
      "Validation loss : 0.58902323\n",
      "Validation accuracy : 0.64\n",
      "At iteration 8800\n",
      "Training loss : 0.5953104\n",
      "Training accuracy : 0.6409\n",
      "Validation loss : 0.588963\n",
      "Validation accuracy : 0.64\n",
      "At iteration 8900\n",
      "Training loss : 0.5952109\n",
      "Training accuracy : 0.6413\n",
      "Validation loss : 0.5889072\n",
      "Validation accuracy : 0.644\n",
      "At iteration 9000\n",
      "Training loss : 0.5951185\n",
      "Training accuracy : 0.6415\n",
      "Validation loss : 0.5888296\n",
      "Validation accuracy : 0.644\n",
      "At iteration 9100\n",
      "Training loss : 0.5950269\n",
      "Training accuracy : 0.6414\n",
      "Validation loss : 0.5887582\n",
      "Validation accuracy : 0.644\n",
      "At iteration 9200\n",
      "Training loss : 0.5949388\n",
      "Training accuracy : 0.6418\n",
      "Validation loss : 0.58869064\n",
      "Validation accuracy : 0.644\n",
      "At iteration 9300\n",
      "Training loss : 0.59485066\n",
      "Training accuracy : 0.6413\n",
      "Validation loss : 0.58860385\n",
      "Validation accuracy : 0.644\n",
      "At iteration 9400\n",
      "Training loss : 0.5947628\n",
      "Training accuracy : 0.6419\n",
      "Validation loss : 0.5885083\n",
      "Validation accuracy : 0.642\n",
      "At iteration 9500\n",
      "Training loss : 0.59467983\n",
      "Training accuracy : 0.6419\n",
      "Validation loss : 0.58842236\n",
      "Validation accuracy : 0.642\n",
      "At iteration 9600\n",
      "Training loss : 0.5945968\n",
      "Training accuracy : 0.642\n",
      "Validation loss : 0.58834404\n",
      "Validation accuracy : 0.642\n",
      "At iteration 9700\n",
      "Training loss : 0.5945141\n",
      "Training accuracy : 0.642\n",
      "Validation loss : 0.5882697\n",
      "Validation accuracy : 0.642\n",
      "At iteration 9800\n",
      "Training loss : 0.5944358\n",
      "Training accuracy : 0.6421\n",
      "Validation loss : 0.5881953\n",
      "Validation accuracy : 0.642\n",
      "At iteration 9900\n",
      "Training loss : 0.59435517\n",
      "Training accuracy : 0.6423\n",
      "Validation loss : 0.5881299\n",
      "Validation accuracy : 0.64\n"
     ]
    }
   ],
   "source": [
    "model3 = generate_single_hidden_MLP(2) \n",
    "training_routine(model3,dataset,10000,gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO2deZgcZZ34P9/umRyThAQyIYRAEkBABrIciRyL67GsLocr4qprREA8xszo/mRPFXa51qi77iGuTnBEFCEi6qIghxde6ypiwhUSQI4chARykXNyzEy/vz+qK6mpqbfOt6qrJ+/nefqZme7qt96q7nm/7/cWpRQWi8VisVQaPQGLxWKxlAMrECwWi8UCWIFgsVgsljpWIFgsFosFsALBYrFYLHWsQLBYLBYLYEAgiMiRIvJzEXlSRJaJyMcCjhER+YKIPCsij4vIaVnPa7FYLBaztBgYYwD4O6XUwyIyAVgiIj9RSi33HHMecGz9cQawsP7TYrFYLCUhs4aglFqnlHq4/vt24Elguu+wC4FvKIcHgUkiMi3ruS0Wi8ViDhMawj5EZBZwKvA730vTgRc8f6+pP7cuYIxOoBNg3Lhxc1796lebnKLFkpgl65ZAUEK/wJxpcwqfj8USxpIlSzYqpaakea8xgSAi44H/Aa5QSm3zvxzwlsCaGUqpXqAXYO7cuWrx4sWmpmixpEKuC/r6Oiy+pvjvZ/e93fQu6WVQDVKVKp1zOum5oKfweVjKiYisSvteI1FGItKKIwwWKaXuDDhkDXCk5+8jgLUmzm2x5E1Vqomez5Pue7tZuHghg2oQgEE1yMLFC+m+t7vwuVhGHiaijAT4KvCkUuo/NYfdDVxajzY6E9iqlBpmLrJYykjnnM5Ez+dJ75LeRM9bLEkwYTI6G7gEWCoij9afuxKYAaCUuhG4DzgfeBboAy43cF6LpRBcc0wZzDSuZhD3eYslCVLm8tfWh2CxDKXl+pbAxb8qVQauHmjAjCxlQ0SWKKXmpnmvzVS2WJqIMpmvLCMPKxAsliai54IeOto7hjzX0d5ho4wsRrACwWJpIrrv7Wb5xuVDnlu+cbmNMrIYwQoEi6WJsFFGljyxAsFiaSJslJElT6xAsFiaiDIlyVlGHlYgWCxNhI0ysuSJ0eJ2FoslX8qUJGcZedjENEvhNKo4my0KZzkQsIlpltzovreblutbkOuElutbMoc3Nqo4my0KZ7FEYwXCCMPkAp7HItqosMmizmtagFosRWIFwggi7QKuW8R0i+XCxQtTL3omwibTLLpFhGtaLcTS7FiBMIJYuHih9nnd4hm2iIUtlmkXvaxhk2kX3SLCNW3SmKXZsQKhCQnaIcfdJfsXz7BFLMliGXfRyxo2mWTR9d6nmqplOm8cbNKYpdmxAqHJ0O2Qb1x8Y+wxvItn2CKWZLGMu+j1XNBD19yufcKmKlW65nbFjvaJu+j675PydWxNet442KQxS7Nj8xCaDN0O2b/gheFdPKtS1dbXD4p5r6la4LmSLHo9F/SkXogFiXV+3X3Ks2/A8ZOPH1Z4DmzSGNiQ32bBaghNhgnzg7t4hvkJ3EWs54IeBq4eQF2jGLh6gPlz54cenyfd93ZrBZ///EWbb4KqkIItTQ3W2d5M2MS0JkPXMSsJXXO7gGAntCDMnzs/dBFr1G4v7NrVNSrWsXlpCLaTmR57b4ql4YlpInKziKwXkSc0r79BRLaKyKP1x9UmznsgotuJd83tCrTN6+z1OpNKRSqRi7tfayhqB5xEEBZd88c6lPXYe9M8mPIhfB34IvCNkGP+Vyn1FkPnO2CJqmUTtDgHPdeM/6Rh/g4/Rdf8STK3Aw17b5oHIwJBKfUrEZllYixLNFmcsi7N8k/qNU/p0O36TdynuHTO6Qw0wVmHsr03zUSRTuWzROQxEblfRE4s8LyWAEybVPIo2eB3RroIAuQTOpqWrOG0I5mge9PR3kHvkl5b4qNkGHMq1zWEe5RSJwW8dhBQU0rtEJHzgRuUUsdqxukEOgFmzJgxZ9WqVUbmZxmOKeewu3D7ybogWmfkyCSv74vFIYtTuRCBEHDsSmCuUmpj2HE2yqg5yGvhlutE+5o/qigrYcIxruC0sfbxsII+X7IIhEIS00TkMOBlpZQSkdNxTFWbijh3M9DsC0nRDmrTfg7/jtWNk3fRveb9jMLGaKbPsgjifF+a/X+iWTEVdno78FvgeBFZIyIfEJH5IuJmMb0DeEJEHgO+ALxblTkBokDyStopsgxzHiUbwuZr2hkZVh8pbu0kW9guPlHfF5vI1jiMCASl1Dyl1DSlVKtS6gil1FeVUjcqpW6sv/5FpdSJSqmTlVJnKqV+Y+K8I4GsC4mu0F2R/1B5xPzrrl+QYTvzrIIvbMcaV/tpxjDeRhH1fbHCtXHY0hUNJstColv4dWWw8/qHyiPCRpuR7CldYUrwhe1Yw17zCqOkYxdFGRv2RH1fGilcy3i/isSWrmgwYeUYohbVNGUsTDtj8yKO49GUczIs6gWCS3x0tHcE1i4KOm7ZR5bFnotJmjWap1FO52a9X34aXrrCEk7YriPMrBK1200qDHS71TLuiuKYoUztJMN2rLrXnt70dKyxl29c3rD72aymlyLLjni/+0Vr1mXECoSciTJruAuOjrAvo26BdxO3/AT9Q2U1u+QlTOKYoUw6s936TF6twL2eoNpNSYROoxaUZvVrFJXkp0t89FP2+2USazLKmbjqb5qY+yhTR5ywvSTquT8UUFf/vygVO6mKHxXKmGS8pOa6RpjqyhDvX0T4aNpzxP0Mmy0/wpqMSkzcXVqa3W6UqSNORdK0HcgG1aDWhh7Ww9kkSXaScTShJCaWJOaLRjmWi6746qeIaLcs54gr0A+kmktWQ8iZuLu0Rjm04s4vSx+GtDt2k8S5zqRaWtk0piCy3OOsn08RGkrU9zKNdhznvWXGagglJu4urVHF0eLOL4sdNWiHXXSuRBxNKKmW5tfCln1kWekK3KXtXWHi8ynChxHH/q+bd1hvkaJ7fZQFqyEYII5t2r+TfHrT06VJyw+aPwz1QWT9J26GjmYjJewwLXFKjSf5fMqgIbgIQu2a2rDnR2KJDKshNJA4OynvLq1zTifLNy4vVVq+fxcJDLumMKJs5EGvFx0BE0cTOpBLWOcRcVOEDyPuWLpe3I3q/ldWrIaQkaS7oDJEfkSRxF+Qdodd1H3w73oFQaFGzG7QFHlF3BQdZRRGsyRlZqX01U5HMkl3us0QG552Fxi3bWX3vd3U1HD13T9eVoIElEIdMLv+JOQVcVNE1zrvOcICA1yhZzcDeqzJKILubmhpARHnZ7fPspPUEZlHZVDThM0lypwSpYK7i3SQCm96oY4KI80zQ7uM2d9hxDH7pf18irzPHe0d2mPLZKYtK1YghNDdDQsXwmB98zQ46PztFQpJ7aS65wfVYGkWjrDoi6z2Vt0iXZWq8R1bmDZmIopGt9A1Y/nmvCJu8rwXutyYjvaOIRsXXeb+gVSSIi7WhxBCS8t+YeClWoUBjxk1qZ00yuZZBpNGXrbfIrughfkpIFhgxLWRh/lLdJ9tmfxEQeTxmefpK8qzCkAzU4oWmnnQaIEg+u8RJm5bMziYTVPkNYct2rpCZhBvkQi7jjB7/EhcgMLIazHWfbbesaM2XiP1/8yGneZEVWNW1T2flGZwMJumyHIKYWGkWX05YZ+dzkRRJj9RUeTVTS9MGLi9KqLCaA+kkhRxsQIhhE7N90X3fFKawcGcN4LkaiLTObmzCqawz0gX834gLkBFdtPzjh1VJbgMZtkyYgVCCD090NW1XyOoVp2/ezJ8j7yOyCJCL8uELgy0EWRNQovzGRVRvrnskUxFdtOD/f63sGNsApoeIz4EEbkZeAuwXil1UsDrAtwAnA/0Ae9TSj0cNW6jfQimiaPqjuT46JHmM4ljxy763IIwf+78EfsdgmK76TUjZfAhfB04N+T184Bj649OQP9fNIIJC7lME9qXxw4xz13nSPOZhH1WeZv9dN8lhSp9iGtW4pihkpiqmkHTKgojmcpKqV+JyKyQQy4EvqEcdeRBEZkkItOUUutMnN8U3d3Q2+uEmlarjq8gi3nIj8kF0b9DdOO7IXyhKnpML7oInEb5TFStxsDGFexe/RD9W1/gv5Yt4kubl/IKinaB/zf2IP5ywjgGKruoVQZQUkOJQlUUouDe7f20Av2+cavAP407nLXfvoyWMe20jjuMlokzGDPrTFrbZxqZe9R3pndJ74jVEuJkxCfJms/zO99sGAs7rQuEezQmo3uAzyqlfl3/+wHg40qpYfYgEenE0SKYMWPGnFWrVhmZXxRuEpqfrD4DLybV2DxU4rzV7EZUE1UDe9n19APseuEhdr+yjF07nmH34IvsHruVXe17ARizvsKPN9T41C7Y7Xnv2EH492fgr1ZAZS9IDRis/wROugDWjBt+zoMH4KHNwsDECgPjhYHxiv6DFLsPrSGDMGbzKMbsmsTY6hGMGX8sYyafxNgjz2DscW9EqvH2aHFqDx1oIa5pGImmpWaoZRQUhxf4bVVK9QK94PgQ8pyUl15NUEJvrzmB0DmnM3BBTONEzsP8EhpKeZ1k9nHE3bWlpda/m74n7mXHcz9k+6bfsr1lBTun9dG6TRi7tsrYdYoxawY5aB2MWQdj10HLdhBqvPMK2D1p6Hi7qvBvh0H3t4LP9+I7g5/fUoVX9Shg6P1UQP9BsHvaXnYfvp5dh61nxxGPsfHw/6HvlUEGnleMXzeOCbVjGD/5LCYcez5tHeciLaOGnUP3XXIpSutq9vLRI82MmZWiBMIa4EjP30cAaws6dyyCMpLDnk+DyQUxD/NLVFKVCXXaZLGzgVfWsOW3C3nlhbvYNuo5dk7bzeiNwoTnqoxfPkD7H2D8M9C6UwHhu73VE5M9DzBjK6yaFPx8EAKM2uY8DnrafXb//e6fANuP3cmO4x5nU8dyVu7pZe+LivFrx3LQ4LEcPOMiJp41n5aJh+27hzqhUESk2kgwt8T9P2p2wReXosJO7wYuFYczga1l8x/knYTmFslb+JYeuH6Arpez1V+P4zRL6iyLu4g0qgZMbU8fW/93IStufh0P3zyO3zx4JGue+VdG/WoZx3x+N3/8l3DGJYqOaweY8W04+FFo3RlvbN0irnseYMED0LZ36HNte53n09C6HQ55GGZ8C068eoAzL1ac9U446ou7aP3fpax+5lP85jfTeOSm8az8+jl89pA51K7a07AeDkl6UGclL8dv3P+jZqtNlRZTYae3A28A2oGXgWuAVgCl1I31sNMv4kQi9QGXB/kP/BQZdpqnDyGvscN2LTp7fUd7B8s+sizWmGEUZZ/u37iKjb/4FzZuuost0zcydn2FgxfDwb+vMXEpVPeYOc+i2dD5F9Dnsc607YXeH8DFS8Pfd9U5jiYxY6sjDMKOT/sel4ExsPWP4JXTK7wyB/ZMrjHpxUNpP/TttP/pP9My6fB4AxmgqBpBefueonb/zeZnsLWMDJFXlFFeRfJCz2mggXij/hEGt29k088WsP6lRbxyxAYOfrzKlJ8NcvBiGLUlt9NmWqiTnCON4NGx5xB45TWw4U+rbDlxkENeOIxDp1/CIedcSbUtwJ5lkKK+H7rz6NpimqbZiuOVIQ9hRNDT4yzQSjk/TTmT4/gnTKulWZqPu6QpO5BWta/t3sHG+69m+Y1H8ptfTmHdc//N5O9t4Ky/gpOuHGTqT/MVBuAsyCs/D7XrnJ+mhQE4AqfP5yPuG+U8n4bRm+GwH8Hsjw9y5jw4+J6XeHHFf/LbBw7myRtnsenHn6LWvzt6oBQUVZdK911WqMjvlwlT04FUYsZqCAUQR0MwvQsx1RIxidaSRrXf++ITrP3RR1g76X8Z81KFqT8ZZMov8l/8G0XlGlABH7UoRxCZYs8hsOGN8PKbKuydDIfvOIfDz/sSrVOPjXxv0s88zbEuWTRV9/26768pU1MjwqWzYE1GJSeOD8G0QIgqq5B1/CCSmBC2/34Ra5Zcxabpq5jy6yrTvz3I+JVGplFqZl0RHJk0c4ujleTB9lfBi++qsvHMQaa8eAzTz/wc40+5CBi+oB8/+XiWb1w+bIysi1/U9zFs/KQlQvIoe91MUUZWIDQBUf6JPOyUJ37pxMB/bi8m7b1R11Dr383G+6/ixQ1fZveEPqbfBdPuUbRuM3L6UuP6J1ZNdMJPvVpCFh9CEvZOgrVvFda+VdH2ykF8d/erWLApsqQYEPw9SbJIxvFphX0Pdd8t//viboTKaPs3hfUhNAFR/gldDX3d83FY9pFlQ0ISgzBp7w2ztW74wT+y+DsTWLP280y/dSdnzFPM+GY5hcGi2c5OvnKN83PR7Ozjdf5FXTMQRxiIApSjGRQhDMAxw836huLMd8O0O7bx1VfiCQMYbsdP6vOK49MKo2tuV+Dz/u9vnJDXkWj7N4UVCCVh/tz5iZ6Pi7cfQN7x6kHCpUXB1QcrVm74D465YYBTu2oc+kuohASHmF6Qk+BdvJU4Pzv/ItscghzJSmDm1vyc1zoWzYajPwrTzoaXEiRd+hfRpDkIUYtw1Otxy2jH8ZvF2QQdqAXvispUtkSQd1kH9xx52j2HXENtkKlV6DoUPvydGof+rL4rjsAflukuyFDMwhkWBRTn/EGhq2myoPMgKOQ1EMWwYjP+RTRpyYeoUhtxFuk439+wbPu4/1NxM7Cbya8QF+tDGOEU/aXds+ZxnrvnLWyZ8gIzb4Vp90IlgYuiEU5XL1migHQ5BmP7YVNAEbyirsmd22Vvg8EIa8nYGpzbDv+3GdYrqFaCvzNpchDSRhklwUREUJxrK3PkUTMUt7PkRJJs5TxrzajBAdZ9+zJWjL2daY8KZ9wC1RTh743eTSetT+RFp12M7XcEw5DXVN3BfA1M7oMbfpifBuQKqlBhoPbP46+ehxUfqLD+DYpjBj/A1PO+OOzwNIUa89ZQ3XNAek27+97uWNpPmMms0QIhC9aH0MREOfaKqjXT99QDPPr1g3lp67c4+W8VR3+5phUGUf6BNDWF0pxHR5b6RDqhtbnNcRzP3IJjjnFNMvXHpnFw+YX5+UqCBNUwBMb3O0KpZScc+4Uasz+hWLP1Jh7/6iHsfv63Qw7PozWmKXR9tKOI09HQZaRWSbUCoYmJWvDz/tKqwQHW3PZ2Hn76z2j/4U5O/UiN8Sv0x8dx2JooGJfFMXzx0v2LtySMAgoTZm4W9MytBBaD72+Bj4X1HIxJkCCMq135jzvoD3BaZ41Jv9rOkqV/zLo7LkPV9kcDpF14y0rURsmr/YzU7GUrEJqYqAU/zy/t3pee5vGb21m/7S5O+ygc+V21r3GMjjhlG7IsyEnOE0baEhZBwkzqpqE4i/Omtmxagk4QHtIX7/1BAq1Sg5m31Tj5Cnhx2208cdNUBjavTj/JEhO2UfJrPyO1RacVCE2MbmF3cxfyqjWz/feLePhXHYx/eDun/nWNtjXx3hfXP5C1plCj/BBeYYZyhIGqm4ViLc6SvqYR6AUhDBdU/vZUUVrY+JVwWneNMU9uZsmPjmHn43cPeb2ZFj0dYRsov/YT12TWbKWzrUBoYnQLu1v0Kw8770vf+RCPr30vR3+pxjE31iK1Ai+m/ANlOQ8MN9HAftOQP1rJXZxHhURdZRFacXwYrtbV9VByLawyAMd+vsbMWwZ4dMWFrL/774DGLXqmhVDSDVQck1mRPSNMYMNOm5y4Kf1Zqe3p47mvv4Zv1pbz1eedXsJJS0SbLv1c5vNc8nZ9+Oqtd8KlF0EtYDuWJRS1yJDd7cfCE/8iHLptDq9aG/w/mmeZ9LzCPk2HaTeidLYtXWEZhsloh4FX1vDYbVP57sbl/NuL8ML4dFm8JvwDZTpPmK8iysH8je+Z7bYG5ju4hTHhGZjTqfjWZv2GLeo7mGWHn9fO27SjvNmcz1YgNDlhXywTqnT/hhU8dtdxjFvex40boa916OtJa/kX0XPAe55b73T+vuTt2cpgJIneWT0xenHOQ2gVJQhdRm2D3pf0r4d9N7OamZol7LOonhGmsCajJidudcc0qnT/y8/w2A9nM/Hhfl71hRrVgmr5m8KU6ShtBnIRHdgajS6zG8K/c2m7reVR2jpv0pqh0r6v4SYjETlXRJ4WkWdF5BMBr79PRDaIyKP1xwdNnDcMt6m9iPOzu5xO/cz4Hcc6kqrSe9cu49Efn8TBv3OEgVCss9YEprqTJYne8WsBRWhDjSQsaips8Uqzw/drFUGUceedxgzVKEd9ZoEgIlXgS8B5QAcwT0Q6Ag69Qyl1Sv1xU9bzhuE2pHG7lA0OOn+PZKHgfuF0JFGl9774BI/+/FTafzXA0Qtr+/KoTNmoi6pmmjb81D+/VZrjN7U5WsLkncWYaJqJakTZ9jS29bBNTZkypU3QqOgkExrC6cCzSqnnlVJ7gW8BFxoYNzW9mnume34kkdWJNdi3haU/Pp32Xwxw1E21If/WJmzUabKI0wqQNBpN0Py0S1u97MSuVsdXMVK1gDA2twU/X0NR26NXH9LY1sM2NSMhU9pLo3wkJgTCdOAFz99r6s/5+UsReVxEvisiR+oGE5FOEVksIos3bNiQakJxmtqPVLI4sVStxh++cTJj1uzlqJuCtY2sZpCkZpwsZSjSaDS63gVhpbvTmKGiaGRPiCTohOthAs99/TVDnvNGFfUu6aWjvSNRjkyzRexkoVHXakIgBG2g/P8+PwBmKaX+CPgpcItuMKVUr1JqrlJq7pQpU1JNqKq5Z7rnRxJZktFeuPUt7Gx5gVcvGMzQpy2cpGacLH6ANBqNbh4KT3G6BO9LQx5NevJCJ3T/9W54pe1J1t5+MRBsE1++cTmdczpj29azRuw0UzZ1o6KTTAiENYB3x38EsNZ7gFJqk1JqT/3PrwBzDJxXS6fmnumebxR5Ob79Tiwg8h9h0w+vY834+znpSkV1z7CXQ0mym01qxslahsKv0UC6aqtud7OZBTjWTTnDi0AndC9ZAid9UrFizDfZ8sv/NmITz7LZieOkLZPA0F0r7P9fdh8m55o57FREWoA/AOcALwK/B96jlFrmOWaaUmpd/feLgI8rpc6MGjtL2GlUU/tG4zq+/XR1mZ1nnIzOvqce4JGn3sRJ/6SYuGzYoaEkDe1MerzJ7Ns451402ylF3e/pFNI6AF+7yzmmiCzoLE16ysbmOfDUlcLZjyutM6aIhvdRYa5lbnjjEhVi7s41S9ipkTwEETkf+DxQBW5WSi0QkeuBxUqpu0XkM8BbgQFgM9CllHoqatyRnIfQ0hLs06hWYcBgGHXUP4Kq1Xjs5klM/ukOjrwj3nfBG19fqQU3XglbsJPE55tcgOMIl0Wz4f0Xwl5v66j6bZlc95FuaoNqDQYrjsZgOr8gqRAse77DisuFs2YpXg74ehWVNxBVQiJtXkSR6Obo4s614R3TlFL3Aff5nrva8/sngU+aONdIoSjHd1S0wkvfeT+DtZ1M/058YeBdoHVduNySz0GL1MVL4y9Y7nEmFrw45qerzvEJA9i3s/UmoQ1W9zuow4RZmnkveCCg97GCHa3OmN4xus+DG0/fr1GsmuRkZf/fEdBzf/S5imDmbYqzPwN3bieyV3Ne6HotuyaZMmY++xPTouZiYq62hWaDqFb1GoLR84R8keRaYeoo+PTPYE7MqqWxum/h/N+7u1zXKQrpFvIkAiQMXXvMQ/r2C68k+nLfKKdPsTtHL37BmeQeuK9/7FxHG/F2VvOOsWj2UGHgosR5/uw12e6bKc3j9lfD/VsYttp0tHcUZo6JavkZJTCKJqj9bRQm5mprGTWIohzfoTswgZf74a/PiB/BEseZu68PgIe+Uc4CF9f5bDrsctFsZ4ftX/FbB2D76P0RPUnDqwarwRFAJpr0jO8fPh/vGFedoy8boTL2VjAZ6XTVObArYOv59Kan008wIbqM/t4lvXTf2126mkNpEtBMzNUKhAbR0+M4kF2NoFqNdiiniUqKU9oiyUKli6apDu6PMtHtsje1xVtgTIdduuNtGsf+BVY5/o/+aoCJKCFB989Ek56oMaLGyhIKazLSSTePos0xPRf0DFs03WgjoFQ9oqN8Bf6/Tc3VFrdrEkxFJemca3EjWOI4eXVO0SCCHKW690/e6eyak5owksxnH/V/C0G/C/fiv39poqP8JpodreHF86KuK+hccc1AJiOddPNshMO2GZzHkG2eDS9uZ8mfNOU4vBpFpeL85NpB+K8V8Pi8IcfGjaOPk+wVlKykI2j3qNtRxtUw4o4XxuQ+UNc5JSnca528E22HOP/9S5olHaQVbR/tmLR0Y4Td56BzJdG8TBYyDJpnVTXGHJO389hULkMzJ6ZZCiBpVJK/wN9+RbACW2fBD76yTygkLVAXVb7C31s4jKAFRrvohNjTk54jim2j90f0uNe68XNwa8zGNkmzpINMNHtb4KA9+jH897k66PzUnSuJGSiJQIvy9/jvxfR+uGb8uIaYY/IsCWGyQmke7W/jYE1GOZBHUlzSvAXd8UOYuJKZlx9lPG49Tp4COAvMZY/AfccPNWFAcNhlkMM3jgkjyMwVB53JZV/0D44mccMPs0fzvPftpL6+uCQ1A8UxL6XJExlogwe/CXNO+Dljj3tD5LxNtrXMMwGtLOaoA9pkVJa+B9555FF6O2lUUpx8Btk6w3iFTr9ZYrDKcC2hbn657BG45dThJgwYvruerCmcGWf379+hTt453BQThN/UNMw5LU6lU/e1WVeAXAMt/+z8jBMd5Y6pi24yWRIjqRkoTiHDNM7nlj44/L4Ka37+kcg5m+4LYHrn7TURlTGXISlNrSEUVf4hCt08vJjIQE6iecTREGaykpUclW1SPrQOxEGnqbx3p5nE8Wq6ZEQcLWbyTsdMFHVtk3c6giFIA3HnCMG77TDHsOmSGHmU3UjrfN51ODz8ReGP37oXqerDvMqy6w4ibrdCqyEURFn6HsQ5n4kM5J4eR6go5fwME3pR+Qxt7GQBV2aflA+dA7dWGb7TTBKamaZyaZht27v7veX7MCrg/9X1I0Rd26Y2vTmqbxR8+AK9M1fr8FbQ1+JkHJsij57LaZ3PY9dC6zZh22SPnDQAACAASURBVIM3hx5X5l133FyBMnZx09HUAqEsfQ/inC+v0ts6k5k/z0EEJozejFBjJivp5UNczO3G55NkgcjDhOGSJKLm4qUwIaDCa3/LftPHotmOJpGGnaP1ZpVQB3oFFp7ulKdIik4Ymm7rmaWLXvv/KTYuD2+eWOYeCFFCqdG5DGloaoFQlr4Hcc6XR+ntsFahfvPS/A/X+OGidrbPqLKSo3IRBpBsgVjwwPCd+aiB5C05g0hq23adxH5WT9wvXILMSm17YVR/xGQ0/oHVE+vXGma1FehNqPwX2U8hi9bR/mvFprZHQo8pWwaxlzBhlaR/cploaoFQlr4HYeeLk4GcFp2pauHC4YKi98s1fvLTebStNj8PL0kXCP9aOCBOFE/WkhVJzFGLZusrVszY6swnyCRUHYSzVsPe1nRznLHVuS86h7nLYML/0qL7KcTROoI0lglPwcCYAfqe/Eno+OL7dIradUflFJRZWKWlqQVCmvIPRc4jjq0/C0lMY4OqhS8v+nRundC8xDVLXHXO0L4DALWqE8WTdWebxBylrQmk4Pyn9dpDrQK/OIpY9Y/8LTi9WtMNPwxP5KsmNFWZKJthEp3G8s2TYNITLWxb/p3A97lOW5Wo5KAZ4kQ3NSpXIE+aOsqoWTCZl+AdKylCjRqNt7266CJU/OTVDCdyHqr+0GybZm5xynxHCgQF5zwHz7brY/oXzXaczztH+8ZT0PVQslLWJpsKpSFJCY6fbwH12rM4+gO/GfZ6IyOMyhzdFMWIjTJasqTx+QVZCbPzh70nyFHsH0uHaBao6aNythclJG6MfZqdbRLTVZRjN5B6f4JYiCMMorK7d3zWWfzdjOPqYHJhANkcvVkJ0gbC/DNtq6Cv/7nA1xsZYVS26Kai2ns2RT8EdxGFcrXBjENYaGzQtfhzGrzXHhXe6mofMDwvYvSonVzfdiXErDFUBIGNYAJIm5zlb67j2tD9i/GCB/SZwlrq/QniWjNWTYyX+dtzf/bGNiabCkURpA0M+zxDku7aXoBd414JfL2RPQrK1B8hqDeC+7dp81SpNQQ/RecX6EiSHZ00NDZMgIRpBl5/hd+n0doyyD9c8SHety2fyKKkuA7GS94OY/vrRePqGcQtvmuME3WkC7GMG22TaaFMIETef+HQuVzy9nQhpXFI6+hNQhJtwC84XY2lbQ3sOrQfNTB8p9JIp22ZHMa6fIc0PROiMCIQRORcEXlaRJ4VkU8EvD5aRO6ov/47EZmV5jxF5xcEkdQElDQ0NkyAJBnLm8T2yiM/4i0nf0tbqTMP4i7Sm8Y5Wb633uk4V/3O16gNeNiiHxVts2g2tP+DU2Yid2R43wW3s5nJRkFxjzcRmhrYPU8jICf3BZvvqruhuhP2vvj4sPc00mmb5Nx5m3OKNF9lFggiUgW+BJwHdADzRKTDd9gHgFeUUq8C/gv41zTnKjq/IIik2dFJQ2PDFv20Yba13dup7DEXX7SIecxiBRUGmcUKFjG0lHbaRToo6sibHBZE2Hhh0TaLZsPlFw6tS5SJlLEZYZ3NohZt/+LffV78Rd5EaGpYlrWXtr2OsNdpLD/ZAOO/9hrkOkGuEyrXVXKzkSeh54IeBq4eCM0piBONlFVgFJmcZ0JDOB14Vin1vFJqL/At4ELfMRcCt9R//y5wjojO9amn6PyCIJKagJKGxoYt+mnDbFX/Dir9ZgTCIubRyVdYxSwUFVYxi06+MkQopF2k04RLhr0nLPQ0SPg0Ct01hN3HIGFx4+nxF3kToam6++vXBi57xJlDkNayaDZ86iUn/8RFoVi4eCEnfulEo4Xt8iDKnGOiOF+R5isTAmE68ILn7zX15wKPUUoNAFuByUGDiUiniCwWkcWwAWhcfkEQabKjk9Qgilr0k4zlYjK0+Co+TR9DYwj7GMdVfHrf32kX6TR1cXSvVWpODoEu2iZ04UtzuyLkbeuAflzdNYTdxyBhoQvhDRon7L7FNU/popm82sCCB4ZXtH3v2x0znVwDl1wEOsPH8o3LA5/Pw3aelihzjgn7f5GmMxMCQRfBnfQY50mlepVSc5VSc+fMmZJ7cldSisiOTrPoh1EZNY5ay/DbHWX6CWI1MwKfX8XMfWOELexhIZHnP01gqezzQ3qx67qGDVadheiyR4Y2kOlrdRbTQ8Kyg01Z1+p5DDO3wNfuckJIwxLU/ITdx6w7+cD7ppz7FtenECe0V+tnqD9UihWoDIXtXKLMOabs/3HMVyYwIRDWAEd6/j4CWKs7RkRagInAZgPnLpyyZEcnoTJqHDXfP2Uc008QM9DlMsi+Mc7/7jztoh+2iNx3PMMXY6k/r8Edrxrw/9U3ynnvggegrb9ei6i+2G0bTWq7f1zcNpyuzbzn/qEtOaPMKWHCUycs4goc/+dQHSRVR7qoaKY8sqPLUNjOJcqcU+bifEGYEAi/B44VkaNEZBTwbuBu3zF3A5fVf38H8DNV5hTpCEzv4LMQJwR29JGnsWdybYhJIY7pJ4gFXEkbO7Wv9zGO+9Z8OnTnqFtE0tq1L17qlJHQvTdol9rfwv5M5JzY1DY84sd77UHmFO+uPEx46oTF/Ifi15HyziXs/mUhS4OfjnZ/bIpDmWoFRZlzyhS+GgcjpStE5Hzg80AVuFkptUBErgcWK6XuFpExwK3AqTiawbuVUs9HjZumdEUe7SvLSpIGQb+5s8Jp8xVjHLcMFQZRAfuBOOUtFjGPq/g0q5hJkH0lbYmMqJILYYldYe9dPTFGiQxNi85M+Mb0l87IWmLCZDvPvMpdLJrt5FvEKVHiIgjz586n54Ieo+0zk2C6bad3rOMnH8/Tm57O7ZqylK4YUbWMytJBzZ1LmGAyIbiS9Fl+5KbxzFy4k0Medv6exQpWMWvYe5N0UTMxhpew+kMQXpso7L1XnaPvShbFqAHYWzc1mcC7wKbtNuYSdM2iHC0habZzHt3U9s3pGkLvXxXoLFFRuDz7Luc5tsuIrWWUlLJ0UItKXktT3yiIJCGwbQOHs8vjDw4y/STtopZmjKDYeffvq87Z7wT2mzyi4uajzCupTEMKbr4rxftC8Jpg0nYbc9FFGkUluwWRRzc1l5m661EwaQCubmsvjTCAfDODi8w6TsOIEghl6aAWJZhMCa4kIbBt40+gb+b+j/tibqeXDzGTlam7qCUdIyh2fuHpw2PpVwWYhOL4F3S+iTg9B4KYWe9XkLT8dBjexT5NVJUX3T0JS3YLw3Q3NRedv+O2O+E3qoUPzjzfzIkMkWdmcNmK5vkZUQLBdAe1JDWLvEQJJlOCK0kI7IQj/pStJw197mJuZyVHUaO6z8STNAzVP0aYQIlT6kDVwxH9Dtasu+mongO6WjuQvEGNDlFDI36+fRKJo6q8hF27TlhkrV+UBp328Z6lsK1jkAlHn5v/JBIQFgGUtTRFmqijoiqdwggTCCZzBLKYdaIEkynBlSQE9qCzP8zuKTV2TwkeK20YahKSRqx4TUK6uPkdrfEWNf+iNHnn/qJ6M7c4OQJB5pJFs81pCIr9u+5Fs8PLQvsJWsgXPDA8zNQlSFiElcLIW1AEaR99M6HWKow/7a8SjeVdIN2HyYUyLAIoa7Z00qgjE5nOSRhRTmUwF2WUxGEbNIcw53ZW53faa3zyxpkc9KPVTP/+8NdMO4iD0EWyhOF1sA6JqvHsrEcNOLZ+0+WdgxytWfA6lNv/IbhpjP843Txch+//HeGY2bzOaa8j3huFVFHB4aWTdzoFBvNwKIex6mJhz5+cwHEfXhb7PTqnrEtW56w3IiiMLI1ykkQwpWnUY53KHkzlCGQx68QpP5E2uS2u5hJk7mqf/i42/kmwGqLLQNY9n4awHa0O70533+LkM7PsbXEWvrR4q57KNc7vrvAJFQYJ8hi85qIw7QA1PJEszKEelOzmCgN/8T5drsGmtmJ7MO8772srtB9zSaL3RDlfszhn/bvxMLLY/IOyjnVmoaJ9DiUp71U+qlW9hhAHty9B2tdd/NpAWB8Fdzxdk51q/5W8623/zsA4aPHlls1gdaCGoM9MTs7FS50d7cLTGW47dxdW307XvzjqFtJNbfsTv5KwaLbTp8BbmnrTOLj0bfoFdB/i7K79GksQXnPRVefoj5/cFz/b133+4qXD3zPriuzF+7IkpUU1A9pzMPRNH2TSaz+aaNyohTDtQhmlefgxmWkc1gAnqlFPkLaRhRGnIZiiiJpFUQRpAzq8r+milRZ+7WAmrWln/RuHvxY3hDRN/SMvPfdrIn5EXzM/FpK8nj84i5a/TwFALWbuwea2kLDKodPbF2K7KqRs9A0/HP50mkJ0iRbzkI5maYjTa+HlNwuTXzySypjxicaOWojTLNRJhQGYyTR2tQLduXuX9Ib6HHT+BSalV+utQNBQhppFScJQvZpLmLnryFdfzQvzZFhRsTghpKYcz5s1u/zNbdFhj2Hho2nMHCZKM+gK7Hlx8wNWTSKRdgDpCtFlKRkBwyOikhCVM1JrhTXvgCPmJm+LErUQp1mo45iZoiqNJo0EimOeGlSDoaUxtPMeiyZ0JJoR51QeSSTpGOEVVmEO8f69NR75+niO+MYuDv1lsvkkcTy75S1WM4MZrGYBV+4TLlnKJLiNbbTmEOUUlAt7v78HsM65G4U/U9odV0HizOYox7h3/EqtXqjPh/f+LZqdok+0l4j7GEZUBva682D9RZM4+UPBvZSjCHL8ZikBIdeF36QoR7VOw+ho79CWqNA5i71EOa618/4yqLVJioXsx2oIJUbnrxAJ11zCzF1SqTDjsL9h9XsriZN39aWvhz4fpUmEVfGM4uKlTilp3eTDQkS1PYADxqoMOou0l1EDQ0NVdQX74piQ/HinEBQCGqcQ3aqJQ4/vemj4tQVdVxC6a4gTnhqWM6IqsHqeMONV/xw9CQ1ep6z7yFISOszMFCdqSbdTX75xuTZcNI6vI0rbKWvHNEtO6Bb2+fPDI6n85i5whMjChY72cPUPrqc2tsorpyWbj87BLDC0Y1pEJdVhOQEPzmPs51ZwydJ4fokwv0JYEllobX61/zF5J3zj+86O3evTuPku2Pi56EzeIIEXFV3ltgqNY3/XmoPqx19Sb0Bz3/HD8yu816UTqjpzUdw+zGECf+PZ0DIwmkmvvyL8hhSIbuGNG8KaxJHtCo+wxTxuAxytwNhV7yyWAmsyKjlZ8yp0OQ///K6vcuE5nZzWVUNiJl4tYh6XcFtglVSv2ShJJVVXm/AKkDZ2RpbRSGN20pky4rw3KX7T1PlPO6Wuw8JYRTnHRl1XkvyIsHwC7f3QmIuS3POgKKN5T8KSrwhHTfkE7ReEl1kvmiS5Af5jk0Y2qWuUsSJ3QfNe+JaFttqpJRi9P0Hx+y8fxJT7d3LE/8T/DgiDBCmW3sU+ia8hbUJcmuqcUYlxcauMpsVdJFdNJNC2H1aq2z+3qLH84wYJuqRCNWt11lWXVth69iRmf3ADUmlO40SaiCQvXr9AXqW9bWKaRYs+4kg4/sxvs/ISxe5D4483U2M28pqTklRBTZsQl6Y6Z1Q0UNbInChcP0BQK023DIeutad/bu5YcfwVukiq858219LTj9/X8LXXwwtvr3Hcn93TtMIA9P4CQYZEAsVp7lNUW8wkNO8nY4lFWN2kthPP44jNb+QPfx/fwRxnsQ8KYb2Mr3EVnx6Wv6DzS8RJiEtandMVIpN3ElrMLk8WzXZMR8N22uJEO20bPdzpGza3OCGvugXbPw9RTvnxJL6RoLkF+Rq63wg/fXYWY44+K3yyJUdnHlKoIYv7so8sC+2kVlasQIhJ2sqnjSYqwW7GvLvZM60amKwWRNyS194qqAu4klu4PDDqyERfhiRcvNRxDN/mKfcweSeM7XecsXlXAA10bHvob4EJe5K1wXQ1JVT8Hb+ul4Ku0qprouprrfdfDplb0Ni7gQU7X9BcdfOQpFppGTWAKJpGIDRyQTbV0KYRRCXYVcaM5/hjFvLsX0Pf4fHGTFLyGsKjjtL0VDBRldPVLm690ynstmlceOSMKeIkwsVJ0PPiXou6Tl/XyH/PkvSv9u74EScHoq1/eDmKqGscKEnN/yzk3SO5yFLXQWRyKovIIcAdwCxgJfAupdSwbBMRGQTcr85qpdRb44zvOpUb3RozS+XTMuONYDqkbQP/771XcNWibw6rc5SVLP2bveTR5jHMsbrggfB6PCbP5z+3yWinoHs2tj84IS/o3Emdz7rjs1QILRN5OYNNRR410qn8CeABpdSxwAP1v4PYpZQ6pf6IJQy8NLo1Zlk6sZnEr/Vs7pvCZ26+ic+95T37ylpkrVvkksVP4CWqJEIadLvZVRPjxdwnJari66gBx9lrqjeB7p5B/OTAJNoE6P0apnbRjSaLKShMAyhDe82sAuFC4Jb677cAb8s4XiCNXpBNd2IrA0HCdM/AWG748Wd4/sMVow1zTPgJFjGPVV9bAdcOwn+tgMf3zyNJPSK/yUkX1VOtxRc+ScxYFy+F+QEZxC6tA46z15Qg0t2bzW3xo7SSdqu7eKnzMU2tJ/w1i0M1b6Ka3ZShvWZWgTBVKbUOoP5TF8A4RkQWi8iDIhIqNESks37s4g0bnIS7Ri/IZah8ahqdMH35lSPZ8Hrh42PCs42TkLV/syuc2DoLqDg/f/CVfUJBt6gPGycg+kUX1aPLePYvsHGzd7303K9/bedos1pQ2GIeN0oraamRNRcJr5/dypq/fgF1bfM4VPMmSgNI017TNJECQUR+KiJPBDwuTHCeGXWb1nuAz4vIMboDlVK9Sqm5Sqm5U6Y4RfsavSCXofKpafRCVph9wp2s3WO2YU5SR7SXIKc0/ePgAUc4bRsdbwcdZD4Jiuq57BF9rpdf+KQ1YyWtd5S2KmuWulEuSXI+XrxQeGGeMPs1P6bl4CNijd9oR2pRRGkAeTus4xDZQkMp9We610TkZRGZppRaJyLTgPWaMdbWfz4vIr8ATgWeiztJd+E10RozLXEb2jQLnZ3BjvrOThj3R29lbGsffXuH16k22TAnLlohtNV53q0DFOXw1S2qm9pgoyfTdtYV+hIX20cPbcST1L4+pA2oYlhDIJ2zN23SnLcpTxbneFATHj9r3iGseadwyskPMPa4N8QaN6w5zEjTKqKa3bjXm4fDOi5ZTUZ3A5fVf78MuMt/gIgcLCKj67+3A2cDy5OeyFRrTItDlNZz2QfaqMjQL2+e+QFhaIXQxP3Pr5oYrSXoFlVh6HvDduN7W4bu/pNm777f19YS2FdQr/cHToOcrDt6P0kT+NKwep6w5h0VTjntV1phEKQJlMGRWhRxNIBG5y5kFQifBd4kIs8Ab6r/jYjMFZGb6secACwWkceAnwOfVUolFgiNpFmT0qIIE7I9PfDh+VXGtu5CpMb0scns/iYJckrTuhPO8QinGB3TdBE+SuIt8i5egZHEJKPrzobA+P79u/CkJTlcTOVoJGXlpRXW/UWVU874LWNedXbgMTqHahkcqUUR1uymLNjidhHociBguOkqa2XSMuG9ljGtu/n4hz7Ie+V2jl5YoxKzOqpJ3IY7q5iBHLQa9WdXwh8NF05RMfxyDYEOAm+Btqhqov5zRPUPdgmrthqnQFzYefLI0Yii1gLPfqzCllOrnPy6hxh95CnaY+M0hPEyUnIWGoEtbheDtLv8sFwHb8ZyM2cz+/Ffy+7+MVzf8w1u33E5j/9nhb0RDk5T+QteXKe0osqtM4+C2cGaSpTzVefM9WoFSWsexTXJhGkeUVpJVDRTHjkaYeyZDI9+ocLeY9s57fznQ4UBJN/xj5ScBT9ld6AfEAIhzmKtExhxch16exufPOdiwrwVNGdFhetv+woHtc1lSa+w7bjg9ybNX0gjPC5eGm9hDyKuiSeo5lES843u3EHdyloHon0EugX/vfX6S6sSOrezsOUkWHIjTB79ek784IuxoonCQirLbkYxRVQeQhk4IExGUaUnwkpjuGaTLBR1i02V+Ajr5awUbPjBP/KHwc9xdC9M88XUJ+27nKY5DmQzkcQ18eTBkCgjYHKf40iOOn9Ucx9Rwa+bLIOhgBcvElZdCq9uu57Jb/6n2O81VZahmdGZzUybx7KYjA4IgRC1wIUJDF14pv84aHy9I1M1l6LuhyMkFVMnr+Zvz76Sv3ngm7TWfb5J6halbo7j8SdUJ6xm8E1XMnPG7YUu7EUTpwaSXyiY9CHsPQie+bsqfUdVOemMH8UOK/WSVw2gZkGu0y9E6hpz67D1IUQQlekcVhojqD+xn87O9MlzJiOYTJX40M35+OO9pjfh5U0zufq+m1jw/vew6TXOMUnqFqVpjuM1SUGFwe2zaLvzKyz4/LwRKwwgXt8DhTnzlpcNr4XFN8Pow2Zz2tvXpRIG0PiQykZThkzkSJRSpX3MmTNHmaCrSylHFxj66OpyXq9Wg1+vVoPHco+vVvePEfVamnklJcl1RBF0LbrxJ4zeon5zR0Utv6qivj5mnmpjx5DX29ihbmPesDfOZEXgeDNZEXyikPdMZr32PSPlcdts1MwrUFyD4trhj5lXmD3f7kNQT1xfVQ/eXlVbftWT7kt5ANN1T5eqXldVXIuqXldVHV/sCPzcuu5J+Q+vAVisVLo1N9WbinqYEghKRS/kJhfmuJhcwJXK/zrC1o/+LevUM18+Wf36TtQXZ89TM1ihhEE1kxWBwkCBuo34wsN9CIOal2qB77uNeWpmjLk00+O22ai2K4cuKm1XOs+bGL8mqDVvRf36+6jnvnKGGtixycwX6ACi656uwMW/44sdQ4SEaWGglLICwQRJd/cmCPu/TIt/Jy9i7priCLBtv7tV/f7rY9TiGytq86kxFreEC7ZOQwClqvQPGSeNwEm6MM+8AiXXOD/TLshpxjF17iGCANTG01EPfbWilny1TW1/5M5sX5gDGHfR9z+q16Xc7SUgi0A4IJzKZSXPxjt5NBWKO2Z31yC337KVrbsmMXXSC3xqzCf5wEvpMpwXMY+PcQObaAdgPNvZwQT05ecc2tjJWPrYxJRhr0U5rWPNKyh5TQ2PGoqKaGpEQlkQWzvg+fkV+idXOOqgv6X9/M8glQPCxRhKWkd4UQ7kwHPbKKPmJM9OcHkJm6hs7KBrGj1qJ9e95cP89UOLaFsT/1yLmMfl3Ew/Y3yvKKIEQthxaTq1+Rf1q87RR/2IcnoenL0merFP2o3MNDtnwfOdVXa8SjFLXcrUixZSafXf7wMTXagsECkc4oSY5hV1ZQVCE3PiibDcU9mpowOWLUs3lnexDiPPj1wniCaPe5nvLzqM9t9VOeKOQcY/Hz2WLiwVnEU9KLx1KMECocoAt3BprLpMuh18X2vg0Pvnp5xS2VFtKnX5BXFKWWRh+3Hwwl9VeeW0GjO2XcDh77iVatukAz401Eucchu6PIqovIs88zJs2GmT0t09VBiA87cbepokJNWfja0j76ZCuvNv2jmVM856ljHHvo7H/0149IYqG89mX7vOIMLCTxXsa7hTJVjlmczG4UXxgEFaYnd/02UIVyPqOSnZn3zmx5s9nLQbWRZqFVj/enj4S1We+FSF8ce8iTNet5IjL/3BPmFQ9kzaIolTbkNXlTWqkF1Zq7xaDaGBpEmI05mTdGPFfb8p4piqart3sOH+j7Nmy830j93D9DsV0+6DFt/aHaYheP0AYRnPAJfxDQYDWn/E8SVoM4QVtPXrC+C5xwRpEdVBuOX7jtmoCB9C/wRY9xfCixfCmO1tTJ/yYdrPWzDMNFRUJm2zELcgXxqfQJ4+BqshNClhiWRJayPF0QyK6PIWJ0GvMmY8Uy/6EnMu38UJM3rZ/qfTefB2eOrjVV45bb/WsIAraWX3sLFGsWdIX4awFp0Xczs1zdc8Tvc33U595lZ9ATyXyX3ByWSD1f2F6bKUuw6jVoHNr4Enr6zyu0Ww809mcNJRt3LqB3Zw6Fv/I9BPYKoUddkLuMUlToG9tEllZU1SsxpCAwnbTYct8EoNd+7qji+ydIZLmjLge1Yt5uVfXsN69QB7J+xhyi8rHPqTGvc8OTTKaDIbeRd3cB9vYTUzmMFqFnBlqD8gbYkMiLeD7z4PbjydwLIRAJe9zRECw86fwnEcFrWkBLaeBOvfVGHDa2uM2TKGQ1v/nKlv/BSjpp8UObYJDWGk1Szy+lSCSHtd1odgGUbYbjqs3EZQ9dak58iTNN3tRs+cy4xL72XuZbs55dU/pPXk1/LUP7VwzO3f4qHOqWw7tkJNKtzAx7iFy2NXU4XgBjtxu7/F2cH33A+3aqqiXrzU2a0HkbQSqa4E9k1vhue6Kzz4LeGZf2hl9Ow3ctqJP2fO5bv4zMGH0/bVU2Lt1k309C2rbdxLEg3GW27DZFXWsjbLsRpCg9HtpnUhqR0dwx3RLiJQqTS2QY/JJkGqVmPHw99h/SP/wcZxDzPQNsg737eSdVtnDjs2bkG8uFqFSUyFlurGmVqF3xx2FlNf83HGnXzhvufT7EKzRhk1Mv4+DiNNgwnChp0WSJFd0fznOv54vTBwaeTHaSqvQnePdz//W8YecyYmcgtMk2c3sz2HwLaTYOobNQ5ughfbuCYgk6GmZXdMZ5lfs4TkZhEIQR1ek5z4ncC1OH2TT1dKBa7eInIucANQBW5SSn02y3kbhX/BcxvtQD5Coadn6LgtEZ9W3iGlUYQ5wuPen/B7fJbWX3LolNX8pkeY8FyVCU8OMv5pxYQ/wOjNya4hDf4F3zXlwH6zEUT3YFDAnnbYcRxsPx62n9DCjqMHqbUqDlp3MIe89AqbAs6vc0SGOYndhVEQlMcr7oaaAqkWu845nYE78LJ0QEvrOPdrFlnvU1nJpCGIyAlADfgy8PdBAkFEqsAfgDcBa4DfA/OUUhF73fJpCHmWmohDWF8HKCaKKIyovhMuYVpW6mZG82v8x9/8iu1P/YAd63/N9tpTbD9sG5UB+MW338uXrIW98QAAEFlJREFU7v0U63YcyfTRq7luwpVcuuN2WoYHMKUiqUlooA12T4Ndhzk/d0+vsOvICtuPHgARJrw8kfEtJzDh0Ncy4YQLGX30WUilktjckbSPsZcsO/oy76Qr11WGCEAXQahdo08uKbvm46VhGoJS6sn6BMIOOx14Vin1fP3YbwEXApECoWyY6jeQlrBookYLA9DPz6u5RGlZUffYvcbhAqUCvIGxx72BQ+vvUbUaf/f+Z/jv7xzFQM3Zvq/ZM4tu9RVWXAHnnnU7Y9ZXGfOy0LqlRsvWGi3boXUHtOyAlu1w52S47mRYMx6O3A7X/y+858n6+BWgqncOr54Iz38ABiZV6Z8k7J5aY9ehNWqjYczGFsbuPIgxTGPM+GM5+OCTGN/xVkbPmKOtIeRNaoqz2Op263FIK0jceZZFAPgJEgZhz7uYCsktO0Z8CCLyC/QawjuAc5VSH6z/fQlwhlLqo5qxOoFOgBkzZsxZtWpV5vmZotEaQp61j0wQZ35R99D0PdbtVapVxc4Vj7F71e/YveFx+netZ2DvZucxuIUB2cn3dm1mwfZB9nj+RUYL/ONMePMhgAKpCX+5XPFy//BzTAEemnUOraMOoWXMVMZMPZmxM8+kdVpHYYXj/Lv1uAtYGXe+Jkjr9I7StsqkCeWqIYjIT4HDAl66Sil1V4xzaPI8g1FK9QK94JiMYoxfGLrs4aJCO/W742LOH0Wc+UVpALp7fPzx+4VF3OsOK/UxOCiMO+oUOjtP0Y5zyvUtDPq+gXsUfHpVlesv279Yvr012JTzjrldzGrwAuHfrcc1I5XF5m8anVCsSjXU1BWlbY0Un0IRGsJZwLVKqT+v//1JAKXUZ6LGLZsPAYqNMhqJxNEA4kZXRWlGWct5JNlNltlu7iWsgieUa6ebB7rr72jvYPnG4V8yQZg/d/6+gnRhSWpQDs2q4WGnEQKhBcepfA7wIo5T+T1KqcianmUUCJZspDF7pTUjRTnho8bJ25HYKCHSLMIrL4KuP2qh9zvuy5xv0bBMZRG5SETWAGcB94rIj+rPHy4i9wEopQaAjwI/Ap4Evh1HGFiKIUlFVRP09DiLv+tojlNjKa0zP24Yrm4cE5m7OvKuLBqWjTvSm91HZSIHXX+UGc2faV3WWkRZySQQlFLfU0odoZQarZSa6pqFlFJrlVLne467Tyl1nFLqGKXUgqyTbnaKXoTD5uEvgbFwYTFCIUlpi7AyHmHE9e3oxsmzvECeJR4O5DLWaa89aiH3C4w8NwuNxNYyKphGLcJBJKmo2kghFqeCahB+bURnQgobJ6/ddJ5hjM1QT8iPqQqpaa89aiH3C4yy1iLKihUIBaNbhBcuLH6xjWuKabQQS2Nm8r7X1UZqtfTjmCZPk0NZYubjLvImNZq01+4u8DqCBMZINL3ZWkYFE9fRWcRCFddZ2+j8i5FInkXWypBVm+T6TM7XVAnvZna62/LXJSCuSSWuo1OnSZgkjimmu7vxGdrNSpRjt6O9Y8jxHe0dRhaeMti3k5huTGo0Jq59JO7842IFggGSmFTiOjpNL7ZBAivKFKMLEXVpdDG9MhNlBum+t3tY3PvyjcuNOH7LYN9OssibNJ+V4dqbGWsyMkBSk4o38UqHSXNM2pIXUYldZSmZUUaiTBdlMOvkSZLrOxB6FBSJNRk1mKQmFa+js0vjxzJZDiNpf2YXKwzSE7VDLovjNy+SmG7srr48ZKp2anGIU+VTRxH1ibIkdumuywqDcMJq5sR5vdlJWpm1zBVSDySshmCAtHHyLml6ECfBdGJXI/o0NxtRO+QyOH7z5kB2zjYrViAYIEucfBGYSuzyXldZsq3LSpQZpNnMJKYSxyzlxjqVDxBMVmltVF8GW2m2MVinb3NhncqWSEyapdI6qbNoFUmzpQ90Dcbkjr4ZS2FY0mEFgiUxaZzUWctfJK27FHaukS4sTBe3G+kRUZb9WJORJTFpSllkLX8RVvLD/xUOO5euI1uZfD5ZMZ3jMNJzJkYa1mRkKZQ0Tuqs5S+SREqFnSutuauZML2jPxAioiwOViBYEpMmqipt6KtLEiEUdq4DoS6T6UqqzRYRZUmPFQgjiCJt40md1CZyNeIKobBzZRVMzUAeO3qbU3CAoJQq7WPOnDnKEo+uLqWc5Xnoo6ur0TPbT1eXUtWqM69qNd+56c7VDPfJBF33dKnqdVXFtajqdVXVdU+2CzQ9niU/gMUq5ZqbyaksIu8ErgVOAE5XSgV6gEVkJbAdGAQGVEyHh3Uqx8f2LIiPzWdIhs1DaC6yOJWzCoQTgBrwZeDvIwTCXKXUxiTjW4EQnyRROBZLEmyUUXORRSBkKm6nlHqyPoEsw1gMkKXAnsUShs1DOHAoyqmsgB+LyBIRsbFqOWAL0ZUfk07/ImsL5dn/2VIuIgWCiPxURJ4IeFyY4DxnK6VOA84DPiIirws5X6eILBaRxRs2bEhwigObshbYK2NWcCPmlDVTe8hYhjORo7B5CAcORjKVReQXhPgQfMdeC+xQSv171LHWh9DcNKoIXhiNmpNJp3/RNv3ue7u5cfGNKPavFdahXF5KnaksIuNEZIL7O/Bm4Im8z2tpPGXMCm7UnFLVf9KYhYq06bvaiFcYAPxy5S9tOewRSCansohcBPw3MAW4V0QeVUr9uYgcDtyklDofmAp8r+54bgG+qZT6YcZ5W5qAMmYFN2pOSZ3+/lBP1ywExXZb01U0Xb5xeeDcrNbQ3GTSEJRS31NKHaGUGq2UmqqU+vP682vrwgCl1PNKqZPrjxOVUgtMTNwSn0bZ8cuYFdyoOSV1+oeVnC7Spp9E67DlsJsfW7pihGPSmZmUMkY+NWpOSZ3+YWahImsLJdE6bBhq82PLX48AwjJvG53BXMas4DLOyU9ZksF0WcpB2ES1clBqp7IlX6I0gEbb8U12amvWOaUx2ZUl1DNIG+lo7wg81oahNj9WQ2hyojSARmsIfpphd26SLGGu3fd207ukl0E1SFWqdM7pLI3TtsxzO9DJoiE0vKJp2MNWO40mqHKn+1CqXNU9o+ZSZDXUonCvx/+oVhszH1u1dORDhmqnmcJOLY0nKpzR3YWWYVcelQPg3Um7pi9obg2i0SY7L2GhrHZ3bwHrQ2h64kTNZLWZmwpbLXNry7xCc8sUehsWymqxgBUITU/eNYxMhq2WtbVlnqG5ZQq9tVVLLVFYgTACyDNqJs7OPe7uuqytLfPUTspUdNBWLbVEYQWCJZSonXuS3XXY4tjInXTe2kmUwC4qk7wsoayW8mLDTi2hFBnWqgtJ9T7vHd+Uc7yRoblFV1/1hosKsq9onQ0dHTnYxDRLbkTt3E3uroN20n4NxDv+SLDzF+1M77mgh4GrB+ia2zWkgmnePRUszYHVECyRNLI0hm580+dpVMJco3phl6U0hsU8VkOw5EqYDTzv3XWUplGUnT8vGuVMtxFHliCsQLBkIu8omqiFUdtPIEdHrcmxG2WushFHliCsQLBkJs/dddTCGPR6nnkFpsduVFiqjTiyBGF9CJbSkzTKKE+/RtmKBWbBFqgbmWTxIViBYBlx5OmobZQT2GKJi3UqWywe8nTUlqk2kcVimkwCQUQ+JyJPicjjIvI9EZmkOe5cEXlaRJ4VkU9kOafFEkWejtoy1SayWEyTVUP4CXCSUuqPgD8An/QfICJV4EvAeUAHME9EglsuWSwGyNNRW6baRBaLaYz5EETkIuAdSqmLfc+fBVyrlPrz+t+fBFBKfSZqTOtDsFgslmRk8SGYbJDzfuCOgOenAy94/l4DnKEbREQ6AVcB3yMiTxibYT60AxsbPYkY2Hmaxc7TLHae5jg+7RsjBYKI/BQ4LOClq5RSd9WPuQoYABYFDRHwnFYtUUr1Ar31cRenlXRF0QxzBDtP09h5msXO0xwiktqsEikQlFJ/FnHyy4C3AOeoYPvTGuBIz99HAGuTTNJisVgs+ZM1yuhc4OPAW5VSfZrDfg8cKyJHicgo4N3A3VnOa7FYLBbzZI0y+iIwAfiJiDwqIjcCiMjhInIfgFJqAPgo8CPgSeDbSqllMcdvhmavzTBHsPM0jZ2nWew8zZF6jqXOVLZYLBZLcdhMZYvFYrEAViBYLBaLpU6pBEIzlMIQkXeKyDIRqYmINvxMRFaKyNK6b6Xw7LoE82xoWREROUREfiIiz9R/Hqw5brB+Lx8VkcKCEqLuj4iMFpE76q//TkRmFTU33zyi5vk+EdnguYcfbMAcbxaR9brcInH4Qv0aHheR04qeY30eUfN8g4hs9dzLqxswxyNF5Oci8mT9//xjAcckv59KqdI8gDcDLfXf/xX414BjqsBzwNHAKOAxoKPAOZ6Ak/jxC2BuyHErgfYG3svIeTb6Xtbn8G/AJ+q/fyLoM6+/tqMB9zDy/gDdwI31398N3FHSeb4P+GLRc/PN4XXAacATmtfPB+7HyV06E/hdSef5BuCeBt/LacBp9d8n4JQO8n/mie9nqTQEpdSPlROVBPAgTs6Cn9OBZ5VSzyul9gLfAi4scI5PKqWeLup8aYk5z4beyzoXArfUf78FeFvB5w8jzv3xzv+7wDkiYUWyc6EMn2MkSqlfAZtDDrkQ+IZyeBCYJCLTipndfmLMs+EopdYppR6u/74dJ4Jzuu+wxPezVALBx/txpJufoFIY/htRBhTwYxFZUi/HUUbKcC+nKqXWgfMlBw7VHDdGRBaLyIMiUpTQiHN/9h1T38xsBSYXMruAOdTRfY5/WTcdfFdEjgx4vdGU4fsYl7NE5DERuV9ETmzkROpmylOB3/leSnw/TdYyikXRpTDSEGeOMThbKbVWRA7FydN4qr7zMIaBeeZ+LyF8ngmGmVG/n0cDPxORpUqp58zMUEuc+1PIPYwgzhx+ANyulNojIvNxtJo/zX1mySjDvYzDw8BMpdQOETkf+D5wbCMmIiLjgf8BrlBKbfO/HPCW0PtZuEBQTVAKI2qOMcdYW/+5XkS+h6PWGxUIBuZZSFmRsHmKyMsiMk0pta6uzq7XjOHez+dF5Bc4O6K8BUKc++Mes0ZEWoCJFG9uiJynUmqT58+v4PjoykZTlLnxLrxKqftEpEdE2pVShRa9E5FWHGGwSCl1Z8Ahie9nqUxGMkJKYYjIOBGZ4P6O4ywvY9XWMtzLu4HL6r9fBgzTbETkYBEZXf+9HTgbWF7A3OLcH+/83wH8TLORyZPIefpsx2/FsTmXjbuBS+vRMWcCW11zYpkQkcNcP5GInI6zjm4Kf5fxOQjwVeBJpdR/ag5Lfj8b6SkP8Jw/i2PzerT+cKM3Dgfu83nP/4CzQ7yq4DlehCN59wAvAz/yzxEn2uOx+mNZ0XOMO89G38v6+ScDDwDP1H8eUn9+LnBT/fc/BpbW7+dS4AMFzm/Y/QGux9m0AIwBvlP/7j4EHF30PYw5z8/Uv4uPAT8HXt2AOd4OrAP669/NDwDzgfn11wWnmdZz9c9ZG8XX4Hl+1HMvHwT+uAFzfC2O+edxz3p5ftb7aUtXWCwWiwUomcnIYrFYLI3DCgSLxWKxAFYgWCwWi6WOFQgWi8ViAaxAsFgsFksdKxAsFovFAliBYLFYLJY6/x+YRD2sKgSZzAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_model(model3,testx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DATA PARALLELISM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "# Parameters and DataLoaders\n",
    "input_size = 5\n",
    "output_size = 2\n",
    "\n",
    "batch_size = 30\n",
    "data_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomDataset(Dataset):\n",
    "\n",
    "    def __init__(self, size, length):\n",
    "        self.len = length\n",
    "        self.data = torch.randn(length, size)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.data[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "rand_loader = DataLoader(dataset=RandomDataset(input_size, data_size),\n",
    "                         batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    # Our model\n",
    "\n",
    "    def __init__(self, input_size, output_size):\n",
    "        super(Model, self).__init__()\n",
    "        self.fc = nn.Linear(input_size, output_size)\n",
    "\n",
    "    def forward(self, input):\n",
    "        output = self.fc(input)\n",
    "        print(\"\\tIn Model: input size\", input.size(),\n",
    "              \"output size\", output.size())\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are using 1 GPUs!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DataParallel(\n",
       "  (module): Model(\n",
       "    (fc): Linear(in_features=5, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Model(input_size, output_size)\n",
    "if torch.cuda.device_count() > 0:\n",
    "  print(\"We are using\", torch.cuda.device_count(), \"GPUs!\")\n",
    "  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs\n",
    "  model = nn.DataParallel(model)\n",
    "\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tIn Model: input size torch.Size([30, 5]) output size torch.Size([30, 2])\n",
      "Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])\n",
      "\tIn Model: input size torch.Size([30, 5]) output size torch.Size([30, 2])\n",
      "Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])\n",
      "\tIn Model: input size torch.Size([30, 5]) output size torch.Size([30, 2])\n",
      "Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])\n",
      "\tIn Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])\n",
      "Outside: input size torch.Size([10, 5]) output_size torch.Size([10, 2])\n"
     ]
    }
   ],
   "source": [
    "for data in rand_loader:\n",
    "    input = data.to(device)\n",
    "    output = model(input)\n",
    "    print(\"Outside: input size\", input.size(),\n",
    "          \"output_size\", output.size())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gputest",
   "language": "python",
   "name": "gputest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}