EEG/Muse-EEG-eyes-open.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81d75cbe",
   "metadata": {},
   "source": [
    "# Eyes Open/Closed Detection\n",
    "\n",
    "In this notebook we will train a model to determine whether a person's eyes are open or closed based on EEG signals from the Muse 2 headset.\n",
    "\n",
    "## Running a Survey\n",
    "\n",
    "First we can import our library and run a survey, so we can train a model on the resulting data. We'll ask the participant to first get into a comfortable position, then open eyes for 30s, close for 30s, etc. 3x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a901c79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Next step -- Just breathe normally, gently relax any tension, get in a comfortable position.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-484722528e9f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0mmuse2_recorder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMuse2EEGRecorder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0meyes_survey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSurvey\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmuse2_recorder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Eyes open-closed\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Eyes open for 30, closed for 30 - repeat 3x.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meyes_schedule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0meyes_survey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Jared\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/workspace/hobby/learning-data-science/neurothink/EEG/../src/survey.py\u001b[0m in \u001b[0;36mrecord\u001b[0;34m(self, subject)\u001b[0m\n\u001b[1;32m     46\u001b[0m                 \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_raw_recording\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mduration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msurvey_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtag\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep_tag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m                 \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mduration\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_seconds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;31m# Play audio to signal end of survey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%matplotlib notebook\n",
    "# Reload external source files when they change\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "from datetime import timedelta\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "sys.path.append(\"../src\")\n",
    "from recorder import Muse2EEGRecorder\n",
    "from survey import Survey\n",
    "\n",
    "eyes_open_step = (timedelta(seconds=30), \"eyes_open\", \"Please open your eyes.\", True)\n",
    "eyes_closed_step = (timedelta(seconds=30), \"eyes_closed\", \"Please close your eyes.\", True)\n",
    "eyes_schedule = [\n",
    "    (timedelta(seconds=30), \"intro\", \"Just breathe normally, gently relax any tension, get in a comfortable position.\", False),\n",
    "    eyes_open_step,\n",
    "    eyes_closed_step,\n",
    "    eyes_open_step,\n",
    "    eyes_closed_step,\n",
    "    eyes_open_step,\n",
    "    eyes_closed_step\n",
    "]\n",
    "\n",
    "test_schedule = [\n",
    "    (timedelta(seconds=5), \"intro\", \"Just breathe normally, gently relax any tension, get in a comfortable position.\", True)\n",
    "]\n",
    "\n",
    "muse2_recorder = Muse2EEGRecorder()\n",
    "eyes_survey = Survey(muse2_recorder, \"Eyes open-closed\", \"Eyes open for 30, closed for 30 - repeat 3x.\", eyes_schedule)\n",
    "eyes_survey.record(\"Jared\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cde1e5e4",
   "metadata": {},
   "source": [
    "## Preparing Data for Learning\n",
    "\n",
    "We need to transform our raw survey data into a format suitable for supervised learning. We will create an input tensor with the shape required by PyTorch - `(batch_size, kernel_size, seq_len)`, aka `(Samples, Variables, Length / time or sequence steps)`, or `[batch_size, channels, num_features (aka: H * W)]`.\n",
    "\n",
    "1. Batch size can be tuned. We will start with `64`.\n",
    "2. The second index is the number of features per batch. In this case, we have four EEG sensors, so that will be `4`.\n",
    "3. The number of samples included for each feature in each batch. This must be the same for every batch, so we will need to use a size <= the size of the smallest dataset we will use.\n",
    "\n",
    "The PyTorch `DataLoader` is an alternative to batching out data manually. After creating a `Dataset`, the DataLoader will batch data with a given batch size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2a63694a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from eeg_preprocessing import preprocess_eeg_channel\n",
    "from eegdata import EEGSurveyDataset, ChunkedDataset, MultiDataset\n",
    "\n",
    "def transform_normalize(data):\n",
    "    for ch in range(data.shape[1]):\n",
    "        data[:, ch] = preprocess_eeg_channel(data[:, ch])\n",
    "        stddev = data[:, ch].std()\n",
    "        if stddev != 0:\n",
    "            data[:, ch] = (data[:, ch] - data[:, ch].mean()) / stddev\n",
    "    return data\n",
    "\n",
    "batch_size = 35\n",
    "\n",
    "# Create PyTorch Datasets\n",
    "# For 30s datasets, the # of samples is between 7666 and 7936, with 7936 being the most common.\n",
    "ds1 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 15:35:13.033803\", 7665, transform=transform_normalize)\n",
    "ds2 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 15:42:18.386982\", 7665, transform=transform_normalize)\n",
    "ds3 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 16:30:50.691073\", 7665, transform=transform_normalize)\n",
    "ds4 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 16:35:02.095245\", 7665, transform=transform_normalize)\n",
    "ds5 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 16:50:59.722742\", 7665, transform=transform_normalize)\n",
    "ds6 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 16:55:04.568128\", 7665, transform=transform_normalize)\n",
    "ds7 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 17:56:48.686557\", 7665, transform=transform_normalize)\n",
    "ds8 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 18:00:38.062883\", 7665, transform=transform_normalize)\n",
    "ds9 = EEGSurveyDataset(\"../data/muse2-recordings/surveys/Eyes open-closed Jared 2021-06-27 18:04:25.933181\", 7665, transform=transform_normalize)\n",
    "ds1, ds2, ds3, ds4, ds5, ds6, ds7, ds8, ds9 = ChunkedDataset(ds1, batch_size), ChunkedDataset(ds2, batch_size), ChunkedDataset(ds3, batch_size), ChunkedDataset(ds4, batch_size), ChunkedDataset(ds5, batch_size), ChunkedDataset(ds6, batch_size), ChunkedDataset(ds7, batch_size), ChunkedDataset(ds8, batch_size), ChunkedDataset(ds9, batch_size)\n",
    "train_dataset = MultiDataset([ds1, ds2, ds3, ds4, ds5, ds9, ds8])\n",
    "test_dataset = MultiDataset([ds6, ds7])\n",
    "# Create DataLoaders\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0284483e",
   "metadata": {},
   "source": [
    "## Evaluate Data Quality\n",
    "\n",
    "Some"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "432f2c95",
   "metadata": {},
   "source": [
    "## PyTorch Model\n",
    "\n",
    "Now we can create the neural network we will be training on the collected data. We will use the [simple keras model by Sentdex](https://github.com/Sentdex/BCI) as a starting point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2ab1e1a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model input shape: torch.Size([35, 4, 219])\n",
      "Model output shape: torch.Size([35, 2])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jared/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/functional.py:652: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "/home/jared/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/modules/container.py:139: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "n_channels = 4\n",
    "n_outputs = 2\n",
    "\n",
    "# Create model\n",
    "net = nn.Sequential(\n",
    "    # Pass input to a 1D convolutional layer with a kernel size of 3, apply to activation function.\n",
    "    nn.Conv1d(n_channels, 32, 3),\n",
    "    nn.ReLU(),\n",
    "\n",
    "    # Pass previous layer output to a 1D convolutional layer with a kernel size of 2, apply to activation function,\n",
    "    # and get the max value from each kernel.\n",
    "    nn.Conv1d(32, 32, 2),\n",
    "    nn.ReLU(),\n",
    "    nn.MaxPool1d(kernel_size=2),\n",
    "\n",
    "    # Pass previous layer output to a 1D convolutional layer with a kernel size of 2, apply to activation function,\n",
    "    # and get the max value from each kernel. (same as previous layer)\n",
    "    nn.Conv1d(32, 32, 2),\n",
    "    nn.ReLU(),\n",
    "    nn.MaxPool1d(kernel_size=2),\n",
    "\n",
    "    # Flatten the convolutions. Input shape: (a, b, c), Output shape: (a, b*c)\n",
    "    nn.Flatten(),\n",
    "    \n",
    "    #nn.Dropout(0.5),\n",
    "    # ?\n",
    "    # XXX: The first number needs to be updated each time the input shapes change. We could instead\n",
    "    #      Create a class-based Module, and do a single pass through the conv portion of the network\n",
    "    #      in order to determine the actual size.\n",
    "    #      (This technique is shown in https://www.youtube.com/watch?v=1gQR24B3ISE&list=PLQVvvaa0QuDdeMyHEYc0gxFpYwHY2Qfdh&index=7).\n",
    "    #      For now, we can update this value as needed by commenting out all layers after Flatten(), then running the code\n",
    "    #      below and inspecting the output shape. The x[1] value should be the first arg in the following line.\n",
    "    nn.Linear(1696, 512),  # ~= nn.LazyLinear(512)\n",
    "\n",
    "    # Flatten the linear layer into the required number of outputs\n",
    "    nn.Linear(512, n_outputs),\n",
    "    nn.Softmax()\n",
    ")\n",
    "\n",
    "for i, data in enumerate(train_dataloader, 0):\n",
    "    features, labels = data\n",
    "    print(\"Model input shape:\", features.shape)\n",
    "    out = net(features.float())\n",
    "    print(\"Model output shape:\", out.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78a72cd8",
   "metadata": {},
   "source": [
    "### Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "04b5b0ed",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 80/80 [01:48<00:00,  1.36s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passes per epoch: 42.0\n",
      "Final Loss: 0.3419230282306671\n",
      "Final Accuracy: 81%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABK8UlEQVR4nO2dd3hUVfrHP28SIJSE3gMEUHpTA6iogBUUC8vuiosd27quu6Lr4lpxf+5a17a4dkRUsCMKitKkl9Br6JDQCSQkhPTz++PMJJPJTObOZHrO53nyZO6959773pl7v/c973nPOaKUwmAwGAyRT0yoDTAYDAaDfzCCbjAYDFGCEXSDwWCIEoygGwwGQ5RgBN1gMBiiBCPoBoPBECUYQTeENSLyo4jc5u+y4Uo0XIMhdIjJQzf4GxHJdVisBxQAJbble5VSnwbfKoMh+jGCbggoIrIXuEspNcfFtjilVHHwrTIYohMTcjEEDREZIiIZIvJ3ETkMTBKRxiLyg4gcE5GTts9JDvssEJG7bJ9vF5HFIvKyreweERnuY9mOIrJQRHJEZI6ITBSRT9zY3cxmV5aInBCRRSISY9vWRkS+ttm/R0QedNhvgIikisgpETkiIv+xrY8XkU9EJNN2zFUi0tLFNcSIyBMisk9EjorIxyLS0LYtWUSUiNwmIvtF5LiIPO63H8sQkRhBNwSbVkAToANwD/oenGRbbg+cAf5bxf4DgTSgGfAi8IGIiA9lPwNWAk2BZ4Bbqjjnw0AG0BxoCfwDUDZR/x5YD7QFLgP+KiJX2fZ7HXhdKZUIdAa+sK2/DWgItLOd/z7bdTtzu+1vKNAJaEDl7+YioKvt3E+JSPcqrsMQ5RhBNwSbUuBppVSBUuqMUipTKfW1UipPKZUDPAcMrmL/fUqp95RSJcBkoDVaZC2XFZH2QH/gKaVUoVJqMTCjinMW2fbtoJQqUkotUjpW2R9orpR61nac3cB7wGiH/c4SkWZKqVyl1HKH9U2Bs5RSJUqp1UqpUy7OOwb4j1Jqt1IqF3gMGC0icQ5lJti+x/XoF0vfKq7DEOUYQTcEm2NKqXz7gojUE5F3bGGFU8BCoJGIxLrZ/7D9g1Iqz/axgZdl2wAnHNYBpFdh80vATuBnEdktIuNt6zsAbWxhkywRyUJ77/YXzFigC7DNFlYZYVs/BZgNTBORgyLyoojUcnHeNsA+h+V9QBwVX2CHHT7n4f67MNQAjKAbgo1zK/zD6JDBQFto4hLbendhFH9wCGgiIvUc1rVzV1gplaOUelgp1Qm4DhgnIpehXwJ7lFKNHP4SlFJX2/bboZS6CWgBvAB8JSL1bV7+BKVUD+BCYARwq4tTH0S/NOy0B4qBIz5fuSGqMYJuCDUJ6Phxlog0AZ4O9AmVUvuAVOAZEaktIhcA17orLyIjROQsW/w9G52CWYqOwefYGnnrikisiPQSkf62/W4WkeZKqVIgy3a4UhEZKiK9bbWQU+gQTKmLU08FHrI14DYA/gV8bjKDDO4wgm4INa8BdYHjwHLgpyCddwxwAZAJ/B/wOTpf3hVnA3OAXGAZ8JZSar4tNj8C6AfsQV/D++gGT4BhwGZbXv7rwGil1Bl0w/BXaDHfCvyKDsM486Ft/ULb8fOBP/t8xYaox+ShGwyAiHwObFNKBbyGYDAECuOhG2okItJfRDrbcr2HAdcD00NslsFQLeI8FzEYopJWwDfo9MEM4I9KqbWhNclgqB4m5GIwGAxRggm5GAwGQ5QQspBLs2bNVHJycqhObzAYDBHJ6tWrjyulmrvaFjJBT05OJjU1NVSnNxgMhohERPa522ZCLgaDwRAlGEE3GAyGKMEIusFgMEQJRtANBoMhSjCCbjAYDFGCEXSDwWCIEoygGwwGQ5QQcYJeVFLKF6nplJaaIQsMBoPBkYgbnOvdhbt5aXYaAvwuxe0kMwaDwVDjiDgP/eTpQgCy8opCbInBECJ2L4DDG0NthSEMiThBF9tMk6VmlEhDTSRzF3z6e/jFzMNhqEwECrpWdCPnhhqHUjBzHJQUQObOUFvjHaWlsOp9WPhSqC2pTGkJzPobHNkcakuqTcTF0O0eunHQDTWOjV/pcEvjZMjaD8WFEFc71FZ5JjsDpt8Pe37Vy/1uhsTWobXJkYxVsPJdyD8Fv3kn1NZUi4jz0GNsim5CLhHKz0/AtDFajELJjl/gv/1h1Qeut2+eDi+dBf9KKv97pZu2fembkL4yuNdw5iTMfgzanAuD/w6qFE7uDd753VFa6n6bUrDhC3jrQshIhYvG6fXbf3Rf3t+UFMOmr+H9y+Gnf7gus+Nn/T9tFhS7myc8Mog8Dz3UBhiqx8avIecg/PAQXP/f8ipXsCjI1S+V1ZOgVj2Y9QgktIZuV5eX2b8cvrkHWnSDDheVr887DukrYNsPerleUxhwLwy4G+o1CazdcyZAXiaM+QpKbAkBJ3ZB8y6Vy+adgLqNA/vdlpboF9uCf0P9FtB+ILQbCM27wuFNkL5cv/RyDun1I9+Gxh1h8zewbRak3Ol0vFJ4/1Jo0BJGfQB1GlQ+Z9Z+/fvZqdMAGrV3bV/RGVj7ibYxax/E1tENyUP/UfnYO36GOg2hIBt2zYOuw6v33YSQyBP0spCL8dAjjpzDWsybd4N1n0DjDjD40eCdP32lFuqTe+HCP2uP8ZNR8PVYuH0mtD1XNzpOHQ2N2sEt010Ldc4R2L8M1k+FBf+CJa/BOTfDxY9AQkv/2lxaCrvm6hfQ+fdDm35asEHbWsm2w/BabxjxqrYpEJzcC9/ep7+Ds6+CWvGwZxFs/LK8TMP20GEQdB4KfW+CmFi9vtsIHd4oyIE6CeXl9y6Eg7YpXT++HsZ8Wf7dF+TCz4/D6o+cDBG4e57+3Zz5/q+wYRokDYBh/9bnmnwt7JgNvUaVlzt1UAv9pU/A0v/C5m+NoAeT8pBLiA0JNMd36Opfq16htsR/HFij/494TT+c85+DRh2g742BPW9xIfz6PCx+FRKTtHgnD9Lb/vA5vH8ZfHaj/vzVnSAxFQXFmYSW0PMG/Xd0m/YCUydpMb1xiut99iyCJh2hYZJne4/vhC3f6ppC+irtOSYmae8StF3xjbSH7szBdVBSCCveCYygr58GMx/W39HId6DPjdrLUkp7wsd3QIse0LCt6/27Xg3L/gs750DPkeXrV0/W13TNKzrePmk43PItZKXDt/fql8j5f4J2A3R5VQozHoTl/4NR71U8R/YB/XIZeB8Mf0GvKy3R3v/mbysK+s45Nruu0efYMgOK8vVLyl/sX6FrTK5qU34m4gTdXomM2hh6aQkseR3m/wtq14O/boL4xFBb5R8OrgGJhdZ94bo34dQB+O5P+uFPvsjz/r5wZAt8e4/2ws65Ga76d8Xvs0ELHcb44Ap471KIqwO3fQ9NOlk7fotucMNEOHPCfeZJzmGYPAJi4qD37+DCB6Flj8rl0ldpb3/bTEBB8+7Qa6QOWZx1RUWPtmln1x76EVt++uEN2uNtc46167BC7lGY/kdtz2/e07UYOyK6sbZxctXHaDcQ6jbRYRe7oJ8+rsNYKXdC799q4Z16E7x9sf5eGzq9hO2kr9SZM1c8W7GRddX7gNI1GjsxsdD9Olg7RXv89rDLjp/1y7JFd+gxUodpds2rGIKrDsd36JeTKtEvs0F/gfbn++fYLoi4RlF7zMUvel5cqIXzTJYfDuYHTuyBSVfD3An65s3Ptt2cIUQpWPQf3bhVUs3OXAfX6gendj2dnXHjFO21fnWnfqj9zZop8O5gOHUIRn8G1090/XJs3hVu/FQLx2/eK/cCvSGxja6+uyIrXf9Pvgi2fAf/u0CHFabfX/73wZXwweWwdzFc8gg8sgP+tByufR36/QEaOE0h2aQznNhd+VxHNmtBjKurvV5v2TZTZ9O4Iu1H7Rlf/XJFMfeG2Dgd0tgxu/x+Wj9V1yrOvU0vd7wYbv9B3yf9xsB9SyqLOcDAe6C0GFIdGrYL83R4quvVOqTnSM+RUJyvzw36+d+1AM6+QutKp8G6lrBlum/X5oq5E6BWXbj4YV3j+vAq/VvvXeK/czgQcYIeY4+h++NgexfCry/ohppQcuYkLHwZ3r4Ijm6Fke/q+O1Zl8OyifomDRXL/6dvym/uhjfO0cuODVNWUUqHXBw9xrqN4Xcf6Rfq9Psrv6Wz9sO3f4Rj270/37HtuuG1/flw/3Lodk3V5TteDA9tgh7XeX8u0A2r+Vm6Mc6ZHJvQX/l/8NBmGPq4TuXbs7D8ryAHhj2vt1/6hK45VEXTzvoYRfkV1x/ZDEn9dTho41fe/VaHN8KXt+vvzdV1bJupGyFb9rR+TFd0vVo7K/uW6N989WRts2OtpU0/+OtG3XDurobapBN0GabDXfbvYeMX+nk6/4+Vy7c/vzzsAroNoDAHzr5SL8fWgu4jdO3B+Xt1ZP9ynTHjyatMXwlbv9c1ssue0vfX8Jd0Q3F+VtX7+kjECbpg99D9IOmHNuj/GSGarDo7A2Y/Dq/2gnn/1I1I9y/VMWUR3ciWdxzW+OBp+YND62HO0/oBvGkaNGwHP42HV3vC3H/qKniF6zmgr+fT31d+ILL26eqzcwNWy55a6HbM1nFfO0e3wQdXwfrP4MvbXAuMO+wdcGrX0xkTzt5tIEhso/+78tLt6xLb6vj34Efhz6v1A27/u3+ZFiFX2R2uaNIZUHByT/m6ojM67NOyp/Z2C3PKxcsThXm6phQTBwWntDfuSEGuzoHvNqL62TOdh0JcvBbO/csgc0e5d+4t59+nn5FNX+nfffnb0LK3fpaciYmFHtfrlNWCXB1uia0NHS8pL9NzpP7eds1zf845E2D5RNc1JDtK6d689VvABX/S62rX17WKP6+FLoFpeLUk6CIyTETSRGSniIx3sb29iMwXkbUiskFE/BSAqozdQ/dLDN0+HoZVQVdK57W6o6ptzuycC6/31R5v16vhvsUw5ouKjWYdLtA35pI3ApMfW1qiG4E+HAYfjajoCReehq/G6tS86/6rq8l3/ghjf9Ghg0Wv6BfRDw/pB/3bP8LrfXSD147ZlR8Ie4NoGxcZCQPu1jf4L0/ql2xGKkwapuOOV/0bjm6Bn5+0fl3rp8HeRXD5M549XX+RYIvh5hyqvO3UAZ02V7ex/87X1Bbjd4yjH92qQyIte2pvtFlX687A7Md0vHf0p5DQBjZ8XnH7rnm6h2pXPzzatetD50u1x7/6I6idAL1+49uxOg7WjbDL39Ydl45t1S9Gdy+dHjfosMv2n7SwdxhU8SXacbD+ndy9CI9th/1L9ec9C93btf0nXW7I+Mov6dg4iAmML+3xqCISC0wEhgM9gJtExLlF5wngC6XUOcBo4C1/G1puj/7vlxi6XdCPp1mLo//8BPyrNbx/hf689QfdAeWnx+DdofBcS5h0DZzcV/VxlIK5z2rx/ss63Urfqrfrspc8oqvs6z7z4sI8UFygq6n/7Q9f3KJF6MhmeOdi/WCUlsKPf9fe3m/ehfpNy/dtN0A/9A+sgr6jdSPSx9fruGP/u+DPa/QD4RyHPLhGi5qr6rqIjm/Xawqfj4HJ10F8Q7hzNlxwP1zwAKx6T3t0nsg7oVPckgbAubdX40vykjIP3ZWgH9Tb/ZkX3qSz/u+Y6WLvut6ylz7XubfqXpBHtlR9rM3TtbAO+osW2t6/1dkfju0a22bq37X9Bf6xv+vVcCpDZ6P0+Z0WeV8Q0dksRzbqVMV6zSpmsTjT/nxo0Eo7HsfTysMtdmJr6VpI2o+uwy5rJutaTN0m2mlwRUkxzHkGmp6lf4MgYuU1MQDYqZTarZQqBKYB1zuVUYA90NUQcNM6VH3EXw9F4WktWO0v1MsH13jeJ22WDjuI6PDA52N0OCD1Q93wkXKnDlP8b5AWOndvnZ1z4dA6nQftrmOEnU5DtVe7+FXvagDuUAq+vgt++KvOmvjdR1qE71+mq54//R3evURnA1w8rmJ11JFmZ8N1b+g456gPdOx3+As6ttvNRRzywFr90oqt5fp49ZvqNLisdB0bvfNn3WAKOv7Yqo/OiHHX8GhnztP65Tzi1YB5QS6xe+inDlTeduqgDrf4k7qN9Asw00nQa9XTHXhA53/H1q7aSz+xB75/UN9jlz5h22+0bmzcZGtbKinWHmeXYdq79Addh+vUR1Xqe7jFTp/fa4E9uUc/g1WlHMbE6nYSe857l6sql7GHXZxrKcUFugG3yzDdvrVnketnfP1ncGybvm/d3e8Bwsod3xZId1jOsK1z5BngZhHJAGYBf3Z1IBG5R0RSRST12LFjPphbTrUd9COb9VHOuw0Qz2GXU4d0zCzlThj7M4xP16Jz11z9+Y5ZcPVLOgbepp8Wn2ljyjuBlBmu9ABFiW31A+cJEbjkbzoGvclN9gFA2k+6saaqxhzQGQBbZ+ib7Z4F+uaNiYWEVvCHL3SOeOZu3Ug15DHP9iW00h6dY852zxsqxiFLS/QLzFMKXafB8MclcOdPFTvoxNWB336oq8pf3qG767v6W/QKrPlYe/XBzt+PT4TaDdyEXA6We/D+xDnT5cgmHX6wv8jqN9Uv1/XTbPnpDg5B5i7t0U4cqGtko94vF5+WPbWXv2GaXt6/VDfi+SPcYqd+M0i+GNqm6OelOtSqq8N2cXWh/1jP5e3pkk06aQfEmU5Dod352ss+nVm+fttM3Vv3vNt1I/rpo3AsreK+9qywtufpNMkg46889JuAj5RSr4jIBcAUEemllKow0INS6l3gXYCUlJRqaXK1Qy6HbQ2iHQbptDVPgm6Pm3WwefS14nV3Z2catYdbZ8Dyt3R2yNTRcOt3+qYD3bKfvhyGv2h9YKUuw3Q8dPVk7T05c2QzTLV1zomtrYWzw4U6VFG/WXm5o1t1eKjzpTDoocohABFIuUM3HMXF++5dOMYhu12tY7OFua579DnjLoOi2dlwzX/0izJ9ufv9m3Wx9iIKBAmtK9cgSku1yAdiMKqmnWG3bcArpbSgO4vIwPv0C/zdwfqF0/Y87cVv/0nfK/1u0lkYzsLW50bdpnF8p65txcXDWZf51/7Rn/pv/JZLHtXOVkIrz2Xbna/DIT1ucL09JgaufU1nnf3yJNxgiyCvmaxr6J0v1RlYoMMuLbqV75u+UtcUBj8a/GEtsCboBwDHpNMk2zpHxgLDAJRSy0QkHmgGOKVBVB+vv6M1U7RgO+cWH96oc04bJkFSir5plXJ/gn3L9APRqo/nc8bEwIUP6GN/ebvuJv3bSXr9wpehfnPvYmsxMbqKuuy/lbtMgw7hAFz/lm4USl+pey+u/cTWoDlMe+5fjdX73vB21eGI6o5LYo9Dbp6uz2sPZ7lqEPWGfjfp76GqBuK6jUM3AmFi68oeel6mzrH2d8gFtIe+fqrOUMnP0ul6zm0x7QfCXzbobJL9y/VYNMe26XDagHvdD1XQ+3fwy1M67JA2EzoN8T3O7Q7n+7g6xMZZE3PQ9/79K8qHI3BFi+56eIjFr+p+AIltdeP/kMf0fo2T9fAGe37VtQM7G6bpmkL3a6tzNT5jRdBXAWeLSEe0kI8G/uBUZj9wGfCRiHQH4oHqxVQ8oKwGXX56DFr30SERRw5t0OtFdLVv7Sf6zequh+C+pfql4E0MsecNkP1P3YA6twN0vx52z4fLJ5R77FbpfKnuRbh3iRZoR3bP1+OjnDOmfN2RzfDNvdpzP/dWQODoZt0r0t/jjbii5w06Dr9rrs5wqd1Ae9nVpW6j6h8jUCS21XFVR+wx9UCEXOyZLid2l79IXNVwGrbVYbHev7V+7MTWOgS24m2dxnjxI9W3N5yw8hxf8qhuR/hhHHS5Usf87cMpiOiwS9osXQuLidEdlTZ9o/s8+PNl5QUeY+hKqWLgAWA2sBWdzbJZRJ4VEXv97mHgbhFZD0wFblcBGj3LnoduSc8LcnQsd/+yirGwkmKdCmf3tpP66//uwi55J7QY2sMt3nDBAzr7Y8nrugE1vpG1OJ8z7c/Xb37ndMCifP2y6TS04vqWPeHuuTDor7qWsmayHgvj7Cu8P7cvOIZdDq6B1v2q9oiigYTWkHu44pCydqENVAwddKaLPWOrhYshBXylz2gt5khED1jlM7Xr6bFljqfpGu9ZV1RMK06+WNeKjtqyi3b8rGtKrsKiQcJSGoBSapZSqotSqrNS6jnbuqeUUjNsn7copQYppfoqpfoppX4OlMHihZ6Tc9hWuLS8uy/o7Jbi/PLqaYvuUKu+TvFyxX5bzNZVZwUrBg97QY9Kl51u6zziw9s7ro7u/rx7vpNty/S1dL7U9T5XTIA7ftQZNZcHcdoyx/Svw5ugrR/HFAlXEtvo7JDTDpXTMg89ACEXe9w7c5eukTVs798aTPdrdby93YDg5fOHG2dfUR5rP88pG6fjxfq/PR99wzQdTnV2roJIBPYU9QLHeOa2meWf7Q2idg89JlY32Lnz0PcvtTU2+hgDjo3TWRojXtNxOV/pNBSOb9c9TO3smgcxtVyPdWGnwwVazOPq+H5uX+g5UjeGlhRUP34eCZR1LnJoGD11UA9IVj8AvVXrJOiu7HZBr26X/ErHb6DTWoe/6N/jRhrX/Ed32e/iFOpsmKRDtHsWaU99+2zo9Vv/pXb6QMQJuh1LER17J49OQ3TDoX1MlMMbdCcXx5huUoqutrpK+9u3VMfZqzOkZp0GOoOkOg1Ldi98l4OXvnu+Dsf4u8HKH3S8pLx3pJUMl0jHnsni2Lno1EEt9IEKNzXprMOHx7f7X9BB52lXN60w0qnfVHfZd/UbJl+sM9c2fa0bvwM9FLQHIk7QveopavfQU8ZC8RndSg1auFt0r5iW1zYFSovKvXc7Bbk6h9eX+Lm/adFd93Kzh11yj+lr6TQkpGa5JbaWzpZo2E6Pex7tJNji5BU89AOBiZ/badpJt1GoksAIuqFqOl6i2xkWvKBTZlv3C6k5kSfo3gRdcg7pcSK6DIM6iTr9Simd4eKc3pWUov87x9EzVuqHpYOfujxXBxE9sNHuBbrhzf6CchU/Dxeu/D+4d2FIcnKDToMWOrxSwUM/FFhBtzeMgvvhIwyBI9kWRz99tHyyjxAScYJux1qjqK1DR1xtPWZD2k+6YfLMCT3JgiMJrbQn6RxH37dUpyu1c9GJKBR0Gqpzmw9v0J563caVryWciKsT+Pk2w4WYWB3TttcMlQpcL1E79obRuHjrk3IY/EdCS93pD3RtNMREnKB7FXI5dai8s0G3q/Uwm/ZZ3l15M0kpLgR9mRbMEOWVVsIeXtk1T/91GhL96YCRRKJDb9H8bCg6HRwPvUV3cx+EioH3wIB7Kk+oEQIiTtC9IudweVzzrCt0NsjK9wBxHW9smwLZ+3XOaUGO7pGYsap8AK9wIKGlHmcjdZL2BEOYImVwgWP3/7Jx0AMp6Dav3MTPQ0f/u/Q4TmFAxAq6x56iSmnBs3vo8Ym6AaPotH4IXHncfUdDh4t0z85Xe+pxQ0oKwqNB1JFOQ/SLB3RM3RA+JLYtD7nYG0cDkYNup3Y9nVI30MUMPYYaR8QJulidUzQvU2etOHpH9olf3TUe1W8Gd8zUIyh2HKyn8JIY/40B7S/sjaBNz/I8/K4huCS21lkPBbnlHro9Pz1QDLwn+KNLGsKS0GXA+4jlNmS7l+Q4YE/Xq/XEDfaMFnckpegJjDN36bCN4wQP4UCHC/XYKM6D8xtCT1nq4qHgCbrBYCPiBN0y9tSxBAcPPbEN/HGZda+2aWfX4yWHmlp1dSpggyAMsmXwjrLORQd1Dnr9FqEb/dFQ44g4QS/PcvEQc3HloQM07+J/o0JBOL5oDJU99EA2iBoMTkReDN1qQbugGy/WEEwqeOgB7lRkMDgRcYJux2Maes4hPSCSqe4agknt+lCnoc1DD3C3f4PBiYgTdMtZLo6digyGYJLYWjeo52cZQTcElQgUdIsFcw5VbBA1GIJFQms4sFp/DmQOusHgRMQJuh2PHYtyjIduCBGJbbV3DsZDNwSViBN0u4NeZcilpEjPGmMeJkMoSHTIOze1REMQiThBtxRzsU89Zzx0Qyhw7EiUaDoVGYJH5Am6DQUcyj5D8viZbMzIrrixTNDNw2QIAfaaYXyj8JxJyhC1RJygO4Zc5m/Tk/F+umIfBcUlpO49oTeWdSoygm4IAfb7zjSIGoJM5Al6WcRFlTWMisCE77fw27eXsetYrhF0Q2ixe+gm3GIIMpEn6DYfvaRU8fi3m8rWbz10CoCsvCIt6DG1oF6YDaplqBnUa6bvP9MobwgyETuWS35RqePaioXsnYpiIu59ZYgGYmLg2tdCPmGwoeYReYJu+z9jffnM6hUTX+wTW5jqriGEnHNzqC0w1EAizoV1lbVYaZXpVGQwGGogESforqgk8jmHjYduMBhqHBEn6OJiAF3HdTFFp/UUYCbDwGAw1DAiTtA9DYheK++I/mA8dIPBUMOwJOgiMkxE0kRkp4iMd7H9VRFZZ/vbLiJZfrfUDdfFLGVE+svEqmIACk4cAKCwnpnYwmAw1Cw8CrqIxAITgeFAD+AmEenhWEYp9ZBSqp9Sqh/wJvBNAGzV9jh8bssxnq/1HgMzv+XenImA4uOflwMwaUN+oEwwGAyGsMRK2uIAYKdSajeAiEwDrge2uCl/E/C0f8yrjJS1gCqeqTUZgOWNr+WKk99zf2wjSmzvqIOljQNlgsFgMIQlVkIubYF0h+UM27pKiEgHoCMwz832e0QkVURSjx075q2tFbgyJpUrYtfwavEoRh8aza91hvBorc8ZFbuQXBXP5NWZHMspqNY5DAaDIZLwd6PoaOArpVSJq41KqXeVUilKqZTmzZv7dII9x3OpzxmeqTWZraXtmVQyDBD+m/gQK0q70SXmAEeU9s7nbTvCydOFfLEqveqDGgwGQxRgRdAPAO0clpNs61wxGphaXaOqYs6WozwU9xWtOMnjRXdSbIsaHT5dyj2F49hR2padqrwC8eC0tTz69QZ2Hs0NpFkGg8EQcqzE0FcBZ4tIR7SQjwb+4FxIRLoBjYFlfrXQibNKdnFH7E9MLbmUNapL2fr0E2eABowofA7l0HRqD7sUFpc6H8pgMBiiCo8eulKqGHgAmA1sBb5QSm0WkWdF5DqHoqOBaUpVOTlcteldupVjNOKF4htdbi+gNoXUCqQJBoPBEJZYGpxLKTULmOW07imn5Wf8Z5Z7Zta9jjdPDOA0dT2WddWr1GAwGKKViOspKoIlMTcYDIaaRgQKuva6H7zsbI9l//XjVrYdzgm0SQaDwRAWRJ6g2/4P6eo57TErryiwxhgMBkMYEXmCblN0paBhXe8aPwuKSygqMdkuBoMhOok8Qbf9V0rRtVWCV/ve8/Fqrnptof+NMhgMhjAg8qags7novuRG/rq9esMNGAwGQzgTwR461ImLOPMNBoMhYEScIpbH0BUv/64v1/drE1qDDAaDIUyIPEGnPOTSMjGe10efE1qDDAaDIUyIOEHHIcvFYDAYDOVEnKDH2AXdy2bRn7ccDoA1BoPBED5EnKCXhVwc9Dwh3nOyzmtzdgTKJIPBYAgLIk7Qxw/vRufm9enbrlHZuhgxg3AZDAZDxOWh923XiLkPD6mwLsboucFgMESeh+4K46EbDAZDlAi6GEE3GAyGaBF078ofzy0IjCEGg8EQQqJC0GvHencZKf83J0CWGAwGQ+iICkEvKTW9jAwGgyEqBP1MUUmoTTAYDIaQExWCPvEP54baBIPBYAg5USHoF53dLNQmGAwGQ8iJCkE3GAwGQwT2FA0kt3ywguO5hfz4l4tDbYrBENEUFRWRkZFBfn5+qE2JWOLj40lKSqJWLetzJ9doQc8rLOboqQKSm9UHYNGO4yG2yGCIDjIyMkhISCA5Odl0/PMBpRSZmZlkZGTQsWNHy/vV6JDL7R+uYsjLC0JthsEQdeTn59O0aVMj5j4iIjRt2tTrGk6NFvSVe0+E2gSDIWoxYl49fPn+aqygJ4+fGWoTDAZDAGnQoEGoTQg6lgRdRIaJSJqI7BSR8W7K/F5EtojIZhH5zL9mhh+7juXy06ZDoTbDYDAYyvAo6CISC0wEhgM9gJtEpIdTmbOBx4BBSqmewF/9b2p4cdkrv3LfJ2tCbYbBYPCCdevWcf7559OnTx9GjhzJyZMnAXjjjTfo0aMHffr0YfTo0QD8+uuv9OvXj379+nHOOeeQk5MTStMtYSXLZQCwUym1G0BEpgHXA1scytwNTFRKnQRQSh31t6GeGHtRRz5YvMenfZVSFaa0U0qZ+J/B4CcmfL+ZLQdP+fWYPdok8vS1Pb3e79Zbb+XNN99k8ODBPPXUU0yYMIHXXnuN559/nj179lCnTh2ysrIAePnll5k4cSKDBg0iNzeX+Ph4v15DILAScmkLpDssZ9jWOdIF6CIiS0RkuYgMc3UgEblHRFJFJPXYsWO+WeyG36e083lfpaDEQdHfWrDLHyYZDIYwIjs7m6ysLAYPHgzAbbfdxsKFCwHo06cPY8aM4ZNPPiEuTvu5gwYNYty4cbzxxhtkZWWVrQ9n/GVhHHA2MARIAhaKSG+lVJZjIaXUu8C7ACkpKWEzRKKi4oiN36zJ4E9DzwqdQQZDFOGLJx1sZs6cycKFC/n+++957rnn2LhxI+PHj+eaa65h1qxZDBo0iNmzZ9OtW7dQm1olVjz0A4Cj+5tkW+dIBjBDKVWklNoDbEcLfNCoToREKcWUZfv8Z4zBYAg7GjZsSOPGjVm0aBEAU6ZMYfDgwZSWlpKens7QoUN54YUXyM7OJjc3l127dtG7d2/+/ve/079/f7Zt2xbiK/CMFQ99FXC2iHREC/lo4A9OZaYDNwGTRKQZOgSz2492eqQ6Ee9SBc/N2uo3WwwGQ+jJy8sjKSmpbHncuHFMnjyZ++67j7y8PDp16sSkSZMoKSnh5ptvJjs7G6UUDz74II0aNeLJJ59k/vz5xMTE0LNnT4YPHx7Cq7GGR0FXShWLyAPAbCAW+FAptVlEngVSlVIzbNuuFJEtQAnwN6VUZiANd6ZaHjphE/0xGAx+orS01OX65cuXV1q3ePHiSuvefPNNv9sUaCzF0JVSs4BZTuuecvisgHG2v4jj0+X7Q22CwWAwVJso6inqu4v+7A9bPBcyGAyGMCdqBN2kjRsMhppO+CdWhoCqIuolpYpnv98cNFsMBoPBKtHjoQfpPOszsphsUhwNBkMYEj2C7seYS1VHUspkxBgMhvAkegQ91AYYDCFk8Y7jLNrh3+E0ooHp06cjIhHRKcgfRI2gGww1mZs/WMEtH6wMtRlhx9SpU7nooouYOnVqwM5RUlISsGN7S9QIeqgCIUt2mnlIDYZwJDc3l8WLF/PBBx8wbdo0QIvvI488Qq9evejTp09Z56FVq1Zx4YUX0rdvXwYMGEBOTg4fffQRDzzwQNnxRowYwYIFCwA9ecbDDz9M3759WbZsGc8++yz9+/enV69e3HPPPWWh2Z07d3L55ZfTt29fzj33XHbt2sWtt97K9OnTy447ZswYvvvuO79cc9RkuTgOrhVInEPoY95fwd7nrwnKuQ2GiOTH8XB4o3+P2ao3DH++yiLfffcdw4YNo0uXLjRt2pTVq1ezcuVK9u7dy7p164iLi+PEiRMUFhZy44038vnnn9O/f39OnTpF3bp1qzz26dOnGThwIK+88goAPXr04KmndF/LW265hR9++IFrr72WMWPGMH78eEaOHEl+fj6lpaWMHTuWV199lRtuuIHs7GyWLl3K5MmT/fK1RI2HXurHxkp3RyotVXy6wvQqNRgigalTp5ZNVjF69GimTp3KnDlzuPfee8uGwm3SpAlpaWm0bt2a/v37A5CYmOhxqNzY2FhGjRpVtjx//nwGDhxI7969mTdvHps3byYnJ4cDBw4wcuRIAOLj46lXrx6DBw9mx44dHDt2jKlTpzJq1Ci/Dc1rPHQXxLjJmJm58RDfrnUeaNJgMFSJB086EJw4cYJ58+axceNGRISSkhJEpEy0rRAXF1dhPJj8/Pyyz/Hx8cTGxpatv//++0lNTaVdu3Y888wzFcq64tZbb+WTTz5h2rRpTJo0ycurc0/UeOj+FHR3GTO5BcV+O4fBYAgcX331Fbfccgv79u1j7969pKen07FjR/r27cs777xDcbF+lk+cOEHXrl05dOgQq1atAiAnJ4fi4mKSk5NZt25d2fC6K1e6bnS2i3ezZs3Izc3lq6++AiAhIYGkpKSyeHlBQQF5eXkA3H777bz22muADtf4i6gRdH+GXERg7tYjJI+fycGsM347rp0dR3LYn5nn9+MaDAbN1KlTy0IddkaNGsWhQ4do3749ffr0oW/fvnz22WfUrl2bzz//nD//+c/07duXK664gvz8fAYNGkTHjh3p0aMHDz74IOeee67LczVq1Ii7776bXr16cdVVV1WoBUyZMoU33niDPn36cOGFF3L48GEAWrZsSffu3bnjjjv8et0Sqo4yKSkpKjU11W/HW7P/JL95a6lfjtW1ZQJpR/SEsOOu6EKDOnHcdmEyQ19ewP4TlYXY20bR5PEzfdrPYHBHuN1TW7dupXv37qE2I2zJy8ujd+/erFmzhoYNG7ot5+p7FJHVSqkUV+WjJoZev3ZgLuV/C3ZxpqiEurVjXYq5wWAweMOcOXMYO3YsDz30UJVi7gtRI+hdWyX47ViObaJninSngdMmfm4wGPzA5Zdfzr59gRkPKmpi6ADJTesF7NjbbSEYg8FgCFeiStDHXtTRL8dxlbb4RWqGX45tMNQUzEB21cOX7y+qBN1ft0+wJsv4z89pZY1ZBkM0ER8fT2ZmphF1H1FKkZmZSXx8vFf7RU0M3Z9sPngqKOd5Y97OoJzHYAg2SUlJZGRkcOyYGQHSV+Lj40lKSvJqn6gS9A5N64faBIPBANSqVYuOHf0TAjVYJ6pCLoO7NA+1CQaDwRAyokrQQ0VRSannQgaDwRBgjKD7gelmwC6DIerYcvAUL/y0LaIado2g+4FT+cVMnL+TuVuPkF8UPrOXGAwG3/nd20v534Jd5BVGzjNtBN0P/OfnNF6ancbYyalM+H5L2fonp2+i99Oz/XKOF3/aRt8JP7tcP+j5eX45h8FgKMc+gGuw0pj9gRF0P5Dn4JXvPX667POU5fvIKSjm0xX7WLYrs1rneGvBLrLPFLlcf8DFiJCXvbKAm95dXq1zGgw1mTMRWNuOqrTFUBErQrEtzqZcdG96/NtNQHBHwtt17DS7jp32XNBgMEQNljx0ERkmImkislNExrvYfruIHBORdba/u/xvavgSE1NeJ6uq/cTRe3fk+olLKgz+lV9Uwt++XM/x3AK/2RhIcguKKSiOPG/GYADd+LkxI7vCukhtC/Mo6CISC0wEhgM9gJtExNUUG58rpfrZ/t73s51hjYOes2LPCUrdzJ701WrX48GsT89i5d4TZcvfrTvAl6szeOHHbX61M1D0eno2Iyf6Zyx6gyHYXP3GIq797+IK656Zsbnss7idwyz8sBJyGQDsVErtBhCRacD1wJYq96pBOP/gX6/JoJ6347M7vANUBDbGbDkUnOESDIZgEKmjq1pRnbZAusNyBjDQRblRInIJsB14SCmV7lxARO4B7gFo376999aGKTFOwvu3rzb45biR5BkYDIbQ468sl++BZKVUH+AXYLKrQkqpd5VSKUqplObNo6ebvqvhdl3hqsHUX0RS5weDwRAYrAj6AaCdw3KSbV0ZSqlMpZS9Be994Dz/mBcZ5FiczSjj5BlLk0P7Is1Gzw0G/6EqfI6ch8uKoK8CzhaRjiJSGxgNzHAsICKtHRavA7b6z8To4bt1B7nkpfmWy3sTQy81im6IULLPFDHstYXsPBqZcetwwqOgK6WKgQeA2Wih/kIptVlEnhWR62zFHhSRzSKyHngQuD1QBtcEfNHm7zcc5Nx//sLJ04X+N8hgCCAL0o6y7XAOr88Nz/kBXD2P2WeKLNW2g42lVAyl1CxgltO6pxw+PwY85l/TahbVrdY99Pl6AC55cT4bJ1zlD5MMhhqLo4i7ejKvfn0RB7LOBLWzoBVM1/8ow2o832AIN0LdsP/+ot2Wy7oabiMcqNGC/vS1rvpHhR67t+4phr7pQHbVBQwGg2X+Nct101+oXzTeYMZyCTPST+SVjf0Cwtr9J1m7P8tl2RFvLna53mAweI9y8zmSMIIeZizbXXFUxpFvmS71huhGwrxLdOq+kwzt2iLUZlgi6kIul3dvyZU9WloqG041Kau2FJvp7gyGoJJ2OHLSKaNO0N+/LYV3b00JtRkBY32GiZsbIp89x0/z/I/hNb1bGJniM1En6N4Qjr/fcoeJMFzVRMO5dnq6oJjk8TP5ZPk+t2Xyi0qYn3Y0iFYZwpE7P1rF27/uIuNkebZIWD2PEaruNVvQw+hHm7P1CADfWJxwOjuv8uxFnrYdyDrDvszATXpx5FQ+UHX614Tvt3DHpFUmQyfMuGPSSpLHzwza+YpsoUOlCPsh6MJIJjwStYI++c4BoTbBK6auTGft/pMV1rnKbrHf/CVV3GV9n6089yjAoOfnMfilBT5aaJ2qGrl2H8sF4FS++xeSIfjMTzsWahOCglKKZbsyvXLmom0sl4iklvOYthGAc0ZLQYBmTXE3AUd1iZzb3hBtrN53sspQn53PV6Vz03vLmbH+YJXlKqQwRtCNHbWCHq2cLijh5dlplHgpypsPloc4xk5e5W+zAIeJOQJy9MhnyvJ9JI+fSaafpxa84j+/ln0OZtjE71RDOEf9bylPTN/ksdxe2/grGSfPoJTivYWuw4ORJOKO1BhBv7p3K49lZjwwKAiWWMfVPfXSz2n8d/5Ojx6GI0dP5XPNG+WdkAJevXaj6EopVuw5YStS82T/y1Q954tjQ6A/2HE016/HqynsPJrLc256h7ojr7A4rNrenIlaQXf+yl+78RxeH92vYhmnQn2SGgXSJL9w6oyOPRd5kY8evPFdqr7Rtx6KnHxeQ2AJh2ytohLvhDnjZB49nprNFAuhnVARtYLevkm9Csu142JoUr92hXWR1NhhZ89xnaUSjk6Cp5CLGbM9vJi9+TBZeaEfbjkU4u7pnK60Ye9xHa75adPhQJjkF6JW0Ns5CborIllfwlEc7RZZ6codDh5aIPlsxf6w9uTWpWdx75TV3DtldahNCQmZuYX8sMF62DJSiFpBd+SpEXpUxTDUwCqxe+Ou8CZTxZV2frB4D3mF/g3FmEbRcv7x7UaetNBIFypumLgE8E88/+TpQv7x7UbyA5SVFQg+WLyHtxbs8mqfSHBConpwrs0TrkIE6tXWl+ksgRGm7xXYdLB6HXP++cMW9mWe5tnre1ne51hOAcdyCujRJtHldqvD/hqiixdnpzF15X56t23ITQPah9ocv1BhggvbQiQ4hFHtodevE1cm5tHG7M1HLJV7buYW8otcN6Bmn/Guc88Vr/7K1W8s8ljOagbLjiM5bD9SMxtKB/5rDpe9siDUZvgFZ8FTSnG6oJhPV+zjPg8hHcdYdbi2aTkLuSuHJftMUVjUUKJT7SwSCW/c6vLeoj2cdDMUgKuoTWmp4kxRCfXrVL41sqoYbgC8/z6veHUhQNhN4xUoHL+fI6cKOILrfPRPlu+jQZ04bjinbZAsKyevsJjM3EJLbVB2nAXurQW7eGl2ml/sKS1VdPrHLJ64pjsiwj9/2MLO54YTF+ubL+rLS8PKHn0n/EyXlg34+aHB3hvlR6LaQ3fGOX+0a6sGIbIkuLgbctdVPu2E7zfT8+nZFBb7PkyvlZBLKKMyy3dnkjx+JruOhWf+9hPTN/HXz9eF5Ny3fbiSi1+c79O+drH83kIfiZz84gr/3VFou3df/CmNl20viYJq3JtW8cXZ234k9PdTzRJ0p+UOTeuHxI5g4y7r5IcNh1ixO5PPV+0HYP62o0xepjMzCh1eAqcLijmcne/xPNWt8RSVlDJj/cEqO24Ul5QyeeleDmad4bMV+306z3frtOAs25XpoWRwKCwuddt7NL+ohNmbvU+Tyy0otjQAmvN3vWrvSTclq6L8/pqyfJ+l+Tbttb0Pl+xxG6LLziti8tK9gL4fTduMZ2p0yKWmUNVzcOO7ywEY1rM1d3xUPiTA375cz4+bDnPHoGSW7swkzUKsu7qplG/O3cEb83ZSOzaGYb1c9+ydsnwfE77fwtMzNgNwSZdmJDW2Hh6A0Dfa3vdJxbjyXz9fy6yNh12Gnp7/cRsfLd1L77YNOV1QzLxHhlg6x92TU1m2O5Mdzw2nlo/hCWcKi0tZu/8kAzs1dbl97/HTvLdoj3cHreKWGf/NBn50yPnOK/RDjLoat6i9BrJkZ3g4Aq6oUR66wT3OYmx/kCYt2WtJzMEhbdFHxTxsG363qs4up85UrKIXe9nbL9g8/+M2DmVX9FgPOdV2Zm1074HvP6E7s2w8kM3uKtJYnVm9T3vansb88ea3+tesrdz47nK2HjrlcrvXYk7VTsBJN/dBMH5xMzhXJODwwyTEx9G2Ud3Q2RJMguSRuns4X5+zg25P/lhhnSshsVe9q3p+/JkJEYzn9O1fd/GQxXi4q74Fvowbcv6/5lYImTkye/Nhujz+o8ttjrgKR9mnYjtxuqLQVqfGE6nCaWeFrT0mXKhZgm5jSNfmbHzmKuJrxZatu75fmxBaFFiCNRCWXdDtZ/v9O8t4+Iv1vDpnO/lFpR57JVoRBn+M/BvsiIvVkTGthKw2W+h/YK/pQGWRfHl2mluxd+Sm95ZXWmf/ffwpvFUdy9/37ZnCEkvxfW/4xMd2nEBRIwXdFa+PPqfSut+nJIXAksjF3rPV/uCv3HOCr9dklG139zCl7j1RYdkbwYgEpy7GogvrSvedVzmOmmmV7LyisolFnE3xpgZQJuh+/NarGjXU3dfm62iHd360ih82HLJU1vEc4Zof74oaJeje/DBzxg3mxv7R0estWI2A475YX/Y5x4sZiW7+YAVg0U6nh7laQ5l62Dd5/Ezu/jjV9+PbiPUw2Yr9ur1pVC4pVZaGf1Aohr++kEtf0WOmV8frLQuJOZ32UDW93mDdn8t2+9aYqZTOz/dLo2yAqVGC3qR+HQA6N3eff96pmU5lDHUmhD8J9qWkn8ij9zOup8FzhbNAVPXiddawDxZ73xDnzW/7y5byHrk5+UUs2Xnc6/Ot3HOiyu12c1zpuTuN7/yPWVzzpmdvvVTBQYdGWOdr96ZRtNxDr4i/xte3+j6rqlhxSanfxytXQI+nZrsMGYabTFgSdBEZJiJpIrJTRMZXUW6UiCgRSfGfif6jX7tGfHrXQP4+rJvbMo5x4GgS9WByykNnEaAs7RAcH1D9hR/LKc/JLi1VrHGYa9VZ7FN9ypv2jb9OW8eY91dw9JTnnHxHiksVf/psjcttp/KLykTVlYfuSprs+eXusk0q7O9HcbPbGeoJHqo6/VmP/8jrc3f49Xy/bo+c+VY9CrqIxAITgeFAD+AmEenholwC8Bdghb+N9CeDzmpG7Tj3l22/V2IkeubUCccXUwUxcnpAX5tT/kC+vXAXv3lrKcnjZ/LE9I0BzYRIP5FXZRhj+1Gd5XHGhzE7ZrqJ3fZ55uey+8xZ0HMLilnoQkxOeTEGj/PVuPLIrXQaA4eahOWzVw+3960HA6atTPerHevTs/x6vEBixUMfAOxUSu1WShUC04DrXZT7J/AC4J37EmbYH6oYEZ/zqcMNK68me76zFXYezeXkaf9NjGDPunD1dS/eUR7i+GT5fq+HPK0KR13YdSyXi1+cz1sLdgIwaUnlUI67GLJVNrrpuRlT5qHrBkw76/ZnuSzvKSbviHJKaHHeUylleVApcVL0i1+cx6Dn51m2xe1xvSzvqS2sOo/t6n0nKSwu9TgkgZ2Mk66fmz3HT1dK7wwGVnqKtgUcX3kZwEDHAiJyLtBOKTVTRP7mR/uCTqntAYgSLQfg81TPHsv1tvGxrXC5bVLiuy7qyBMjKlXWfGJjhmuxW+qhe779d3pmxmY+WrqXZg1qk/rEFQBMnL+TFgl1+F1KOwBe+TmNj5ft47q+lVNUD9jGBV+++wQPXAoTvt/i9lwK+M8v23lj7g5mPDCIPcdPs+1wDg8MPYtLXpxPl5YJVi7Z4cC24ypFv2d/KVttbyx2Ji7WC0H3IH4Hs/MZ8vICl9s2ZmTTO6mhs5kUFJdy98eppJ/wbwrgT5sPsy/zND9sOMQ9l3RyW04p+PePW9mfmUfa4Rz+ekWXCtvtL8gzhSXUra1Tk//x7UZLNoz631JaJNThaI61ibzXOL10X/xpG/XrxPHS7DQa1avF7RcmM6BjE/7w3gpeHNWH3/dvZ+m4vlLtrv8iEgP8B7jdQtl7gHsA2rcP/wySKNL0gPD+4j2870OjpCvOFJVU6/v+yDbmx/Hccq/IPuKfXdDfnLfT7f522ct1Mf/q3uOnSW5W36HxUvGGLU770uw0FtlqER2b1SfzdKHX2RTlIRdr5a2mQbo65hYLcXc7475Yxy/jykcPtNdYNx3IrtBY7E8Gv7QAgMT4OLdZJdlninjn191lyw9OXVthuwh8syaDcV+sZ864SzirRYJX4/5YFXNXONYgs/KKKoQP31m4iy6tEmiVGE+rhvE+n6MqrIRcDgCOr5Uk2zo7CUAvYIGI7AXOB2a4ahhVSr2rlEpRSqU0b97cd6sDiL3BRyS6vPRwRylV6fu2MhF2dcJiriYxWOciXjrsdT3Mr11IHTXScegBXy3xNm3Rm4iPXxtFbf+DMf1hflEpa92EnDydXQTmbj0KwLbD4TPevlJ6pqjBL/k2mqUVrAj6KuBsEekoIrWB0cAM+0alVLZSqplSKlkplQwsB65TSlU/gTcEVGwUNYoeTJy/78EWh3H11lt09Q6oSiTKJghxCI24Pq5v90tMFVkurnh9jvUsDn/0rLVT/uLx3zEDgSBhaavdlEAO/+sx5KKUKhaRB4DZQCzwoVJqs4g8C6QqpWZUfYTw5o5ByTSuV7tsuWmD2hzKzicutvym6Nkmkc0HrVdVDd7j/Nzl5BdVyJ92hwCfrvDDZMwWHnxX+eKOGu7r698eWrDq+HqTRledXo6V97TXUHw/5qYD2ezNtD7ImC/o2nV4pFg6EoyajaUYulJqFjDLad1TbsoOqb5ZwePpa3tWWP7wtv7MTztKi4R4jp7SsTSloG6tWJ/S1QzWKHUKuaT83xzL+7oaK2XWRs9dvL3t3u0q5OJO3H0hEA+8Pw9ZnbFcRr61hD3HT7uc9crfoU3HlOMw0vOgCHqN6ilqhRaJ8WVd/js01eNs/3FIZxNPDzRO97rVamlJqXIp6Pd/Wt6R54vUdOanHS1bdpVPXeAw76qrODqUC4/jeCCOcf7qC3r19ndFdTTE+XLKYug+GLp2f5bbKQxd2fjqnO1uj+XJ6xaBmLKQS/goemngJ1oyE1xURUJ8rbJJB/7+9QYAzm7RgB1HQz/VVLTh62OXdiQHPITQH/1qQ4Vle3U8J7+Y7k/+xPu3pVTouXqDmxROu9/3pUMaaOq+k5W2+4ovQukJfw4s5a7rvycKir2v2VY1borHRlEq5vaHU9gl0BgP3SL2R/Xr+y8MqR3RTLArQZsPZnOmqITX5+7wKlXNeYIKO9X10H2Zy/N/HjpaVWeyZmcZtIuk1eGA7fSdUPW4Pt5+b549dPHYgB0KTMgljHDMYHjnlvNCaEl0kn2myHLvvOpiz1nPOOldx5hwDLu98NO2Krd/s6Y8w9jbgcyc00bFR5HML/JvrMHT6XcezS0ro6jc+SdUhE2jqKEcAZrWr+2xnME7HGPe1eWp7zZZKudN5lJ2XpHLTkeOJMSH9+P0zx8q936tin2ZeWw6kM3Ulft54poeFNraNfwdGbrvE+9+eyvtK9+u1S+yNftOVgq5hYojpyoOOhfjxRAOVjEeupcooE9SIzo1q89zI3u5LDPp9v7BNcpQgY+XeZfG6Gl4W4C+z/7s0aN3nAErWhjx5mI+XbGf7k/9xBxbZ50py/2QJlpNm6wybZV/B+ryF185TPziT4ygW8TxXVo7LoZ5jwxhzMAOLssO7hKevWANASZ8wrWGMCfbTcZPdTGCHgACUZUyhD9Gzw2hxgi6l4RRo7khzAinnGdDzcQIulXcON3LHruU5Y9dVq1DPzqsa4Vle4cmQ2Rh9NxglUILA8/5ghH0atK6YV1aNYzny/su8Nsxa8eanyUScTeBhcHgjKeMKV8xyuEtbryw/slNgmuHIez41U+TJRuin0A1sxlBt8i7t6RwWbcWHnON2zWpW2H57BYNPB7bVNWjA1+6uBtqJt5MUuIN4d0TIoy4oHNTLujctMoy65++slK4ZPzwboyd7N3Q8EbfI5NAjnNtMFjBeOh+pGHdWmVzGNpx5X1PuqPqjkedm9f3p1mGIHGsGlOXGWoWgaqVG0EPEH+7qit3DEqmZWLluQM7NavPZd1aADqWdrNTB6X6deL4zbltg2KnwX9khmCWd0Nksjb9pOdCPmBCLgHiT0PPqrROpPKb+Z1bUmhYr1blsi7yJDs1r8/uY9Wf7eWmAe2ZutL6pLkGg8G/bDoQmBnQjIceBJIa64bSR67sSoxQwWt31zTi3Ar+/q0pXN69pV/sqRVrerIaDNGIEfQgYG/QHtGnNbv/fY2lQZwcG8Gv6dOay3u05NGrujJn3ODq21PtIxgMhuoQqKGYjaAHAV9msnHcp5bNXY+LjeEsC2mQHo8djgN7Gww1iEA9gUbQg4g3LdsxDr9MnB97jl58djO/HctgMPhGoJwqI+hBwLe5GMt/8Gv6tPabLYnxtQI+8455aRgMVWM89AjG6x9Plb8E/nlDL4Z2bWF5173PX1M2sbV7ewKr6FPGDgzo8Q0Gg2uMoAcRK3Mx/va8JB67unt5louLfeJr+f6z1a8T65OH3q1Vgs/nNBgMFTGNohGMPV5mJeTy8u/60jyhTpkX7Wr+xm3/HO5y39sucD2DkiNPjujhk3/+3MjePuxlMBhcY2LoEUu7Jnp88zpx5V/3P67pzqCzmnLhWa7Hh/FlhvUJ17ue4/Q35+hep9f0bk2CjzH08zo05pre7mP5zoOSGQwG9xgPPYJ5c/Q5/G/MuSQ1Lp+4onPzBnx61/nUq607675983m8Prpf2faHLu/Cb85ty+/7t3N5zOdG9uLpa3sA0CoxngZ1XHf6XfToUMZd2QXQ4RyA+4dU7sVqhS4tddjl/VtTKrycAP49so+lYzx8RZdK65o1qOOTPYbI5qHLK98LNYX6tQMzobh44wH6k5SUFJWa6t0ohIbKnDxdSOP6tSutzz5TRPqJPHq1behyv7TDOTzw2RqaNajDY1d3Y/ex0xQUl1BQXMrxnAL6tW8EwNKdmTxyVVfia8VSXFLK4p3HGdK1BQXFJdz6wUoa16vNY1d3o0PT+kxasodr+7ahWYM6zN16hEU7jjO0Wws+XLyHE6cL+eiO/jRtUId3ft1F+yb1SKxbi0b1apFQpxZPz9jEfNt44n3bNWJ/5ml+GTeYq15dSE5+Md/cfyGHs/O56+NUkpvWY29mHi0S6nDUNiDWHYOSmb/tKKVKD5J1pqjiULZzxl3C/G3HeH/xbgTh8Kn8CttH9GnNeR0aM+H7LV7/BikdGpO6r3xsjmv6tGbmhkN0a5XAM9f1ZENGFlsP5TCka3OW7z7BpgPZbDyQTcdm9enRJpGZGw4x44FBPP7tJh4b3o37P1vDqTNFlCrok9SQsRd1pLC4lA5N67PpQDbP/rCFzs3rU1yqOK9DY75Zc4BP7xrImn0n+XbtAXYfP02Xlrq/Qo/WicTECFsP5dAqsQ492iSyNzOPWjFC5ulCFu04zsCOTXh0WDd+/84yXvldX3IKinly+qay67ljUDKTluyt8ju4vHtL5mw9Uml920Z1OZB1hm6tEth2OIdmDWpzPLeQ2y9M5pnrevLh4j3E14pl0pI97Diay7ntG7FmfxatEuPp0LQeK/ac8Oq3uO2CDkxets/t9nq1Y8kr1PdGzzaJgB4KY8/x02w7fIo/DT2LlXtO8GVqBgeyzjDq3CS+XpPBLed3YH7aUe14zd3B9iM5XN+3DWvTs1i04zhX9mhJbkEx+UUlXHRWMxLia/HcrK1l9wJUHLpj/VNXuhzywwoislopleJymxF0g8FgiByqEnRLIRcRGSYiaSKyU0TGu9h+n4hsFJF1IrJYRHpU12iDwWAweIdHQReRWGAiMBzoAdzkQrA/U0r1Vkr1A14E/uNvQw0Gg8FQNVY89AHATqXUbqVUITANuN6xgFLKcSzI+phJdwwGgyHoWBkPvS2Q7rCcAVTqCigifwLGAbWBS10dSETuAe4BaN++vbe2GgwGg6EK/Ja2qJSaqJTqDPwdeMJNmXeVUilKqZTmzZv769QGg8FgwJqgHwAck6GTbOvcMQ24oRo2GQwGg8EHrAj6KuBsEekoIrWB0cAMxwIicrbD4jXADv+ZaDAYDAYreIyhK6WKReQBYDYQC3yolNosIs8CqUqpGcADInI5UAScBG4LpNEGg8FgqEzIOhaJyDHAfZeuqmkGHPejOZGAueaagbnmmkF1rrmDUsplI2TIBL06iEiqu55S0Yq55pqBueaaQaCu2QzOZTAYDFGCEXSDwWCIEiJV0N8NtQEhwFxzzcBcc80gINcckTF0g8FgMFQmUj10g8FgMDhhBN1gMBiihLAWdAvjsNcRkc9t21eISHIIzPQrFq55nIhsEZENIjJXRDzPDB3meLpmh3KjRESJSMSnuFm5ZhH5ve233iwinwXbRn9j4d5uLyLzRWSt7f6+OhR2+gsR+VBEjorIJjfbRUTesH0fG0Tk3GqfVCkVln/oXqm7gE7oERzXAz2cytwPvG37PBr4PNR2B+GahwL1bJ//WBOu2VYuAVgILAdSQm13EH7ns4G1QGPbcotQ2x2Ea34X+KPtcw9gb6jtruY1XwKcC2xys/1q4EdAgPOBFdU9Zzh76B7HYbctT7Z9/gq4TCRQ82kHBStjz89XSuXZFpejB0uLZKz8zgD/BF4A8l1sizSsXPPdwESl1EkApdTRINvob6xcswISbZ8bAgeDaJ/fUUotBKqaFPV64GOlWQ40EpHW1TlnOAu6q3HY27oro5QqBrKBpkGxLjBYuWZHxqLf8JGMx2u2VUXbKaVmBtOwAGLld+4CdBGRJSKyXESGBc26wGDlmp8BbhaRDGAW8OfgmBYyvH3ePWJlggtDGCIiNwMpwOBQ2xJIRCQGPaXh7SE2JdjEocMuQ9C1sIUi0lsplRVKowLMTcBHSqlXROQCYIqI9FJKlYbasEghnD10K+Owl5URkTh0NS0zKNYFBktjz9tGtnwcuE4pVRAk2wKFp2tOAHoBC0RkLzrWOCPCG0at/M4ZwAylVJFSag+wHS3wkYqVax4LfAGglFoGxKMHsYpWvJ1rwiPhLOgex2G3LduH6v0tME/ZWhsiFCtjz58DvIMW80iPq4KHa1ZKZSulmimlkpVSyeh2g+uUUqmhMdcvWLm3p6O9c0SkGToEszuINvobK9e8H7gMQES6owX9WFCtDC4zgFtt2S7nA9lKqUPVOmKoW4I9tBJfjfZMdgGP29Y9i36gQf/gXwI7gZVAp1DbHIRrngMcAdbZ/maE2uZAX7NT2QVEeJaLxd9Z0KGmLcBGYHSobQ7CNfcAlqAzYNYBV4ba5mpe71TgEHqeiAx0DeQ+4D6H33ii7fvY6I/72nT9NxgMhighnEMuBoPBYPACI+gGg8EQJRhBNxgMhijBCLrBYDBECUbQDQaDIUowgm4wGAxRghF0g8FgiBL+H+bz6hFC4JEFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Define criterion and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=0.001)\n",
    "\n",
    "# Train for n epochs\n",
    "n = 80\n",
    "test_while_training = True\n",
    "\n",
    "loss_history = []\n",
    "eval_history = []\n",
    "count = 0\n",
    "net.train()\n",
    "for epoch in tqdm(range(n)):\n",
    "    for i, data in enumerate(train_dataloader, 0):\n",
    "        features, labels = data\n",
    "\n",
    "        # Zero gradients\n",
    "        optimizer.zero_grad()\n",
    "        #net.zero_grad()\n",
    "\n",
    "        # Forward\n",
    "        predictions = net(features.float())\n",
    "        \n",
    "        # Compute loss\n",
    "        loss = criterion(predictions, labels.long())\n",
    "        loss_history.append(loss.item())\n",
    "        \n",
    "        # Backward\n",
    "        loss.backward()\n",
    "        \n",
    "        # Optimize\n",
    "        optimizer.step()\n",
    "        \n",
    "        count += 1\n",
    "        \n",
    "    # Evaluate the model against the test dataset\n",
    "    if test_while_training:\n",
    "        total = 0\n",
    "        correct = 0\n",
    "        with torch.no_grad():\n",
    "            for i, data in enumerate(test_dataloader, 0):\n",
    "                features, labels = data\n",
    "                out = net(features.float())\n",
    "                preds = F.log_softmax(out, dim=1).argmax(dim=1)\n",
    "                total += labels.size(0)\n",
    "                correct += (preds == labels).sum().item()\n",
    "\n",
    "        eval_history.append(correct / total)\n",
    "\n",
    "%matplotlib inline\n",
    "plt.title(\"Training session\")\n",
    "print(\"Passes per epoch:\", count / n)\n",
    "print(\"Final Loss:\", loss_history[-1])\n",
    "plt.plot(np.linspace(0, 1, len(loss_history)), loss_history, label=\"Loss\")\n",
    "if test_while_training:\n",
    "    print(f\"Final Accuracy: {int(100*eval_history[-1])}%\")\n",
    "    plt.plot(np.linspace(0, 1, len(eval_history)), eval_history, label=\"Accuracy\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "20128bbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.1660e-31, 1.0000e+00],\n",
      "        [3.5037e-09, 1.0000e+00],\n",
      "        [1.0000e+00, 2.2550e-26],\n",
      "        [8.7600e-22, 1.0000e+00],\n",
      "        [9.3897e-20, 1.0000e+00],\n",
      "        [2.0502e-06, 1.0000e+00],\n",
      "        [9.9999e-01, 1.3380e-05],\n",
      "        [2.9598e-05, 9.9997e-01],\n",
      "        [1.7979e-29, 1.0000e+00],\n",
      "        [3.8447e-26, 1.0000e+00],\n",
      "        [1.0000e+00, 4.1799e-07],\n",
      "        [5.8915e-18, 1.0000e+00],\n",
      "        [9.9433e-01, 5.6722e-03],\n",
      "        [1.0000e+00, 7.9259e-07],\n",
      "        [2.6468e-35, 1.0000e+00],\n",
      "        [2.8654e-11, 1.0000e+00],\n",
      "        [1.0000e+00, 5.2129e-33],\n",
      "        [1.0000e+00, 3.5406e-06],\n",
      "        [4.0764e-19, 1.0000e+00],\n",
      "        [1.0000e+00, 4.2954e-18],\n",
      "        [9.0891e-04, 9.9909e-01],\n",
      "        [3.0461e-14, 1.0000e+00],\n",
      "        [7.2323e-13, 1.0000e+00],\n",
      "        [3.5736e-22, 1.0000e+00],\n",
      "        [1.0000e+00, 6.4795e-11],\n",
      "        [1.8755e-03, 9.9812e-01],\n",
      "        [1.0000e+00, 1.1711e-08],\n",
      "        [1.0000e+00, 4.5160e-41],\n",
      "        [5.0137e-30, 1.0000e+00],\n",
      "        [3.4935e-12, 1.0000e+00],\n",
      "        [1.2079e-05, 9.9999e-01],\n",
      "        [3.8973e-07, 1.0000e+00],\n",
      "        [1.0000e+00, 4.0188e-07],\n",
      "        [2.9430e-05, 9.9997e-01],\n",
      "        [1.0023e-22, 1.0000e+00]])\n",
      "tensor([[9.9555e-01, 4.4456e-03],\n",
      "        [3.2050e-13, 1.0000e+00],\n",
      "        [5.4395e-28, 1.0000e+00],\n",
      "        [1.0440e-18, 1.0000e+00],\n",
      "        [3.7894e-01, 6.2106e-01],\n",
      "        [1.3323e-06, 1.0000e+00],\n",
      "        [9.9990e-01, 1.0312e-04],\n",
      "        [5.5898e-30, 1.0000e+00],\n",
      "        [1.0000e+00, 7.8954e-18],\n",
      "        [1.7087e-08, 1.0000e+00],\n",
      "        [6.0435e-06, 9.9999e-01],\n",
      "        [1.0000e+00, 7.0906e-15],\n",
      "        [0.0000e+00, 1.0000e+00],\n",
      "        [1.0000e+00, 4.0650e-13],\n",
      "        [3.1048e-13, 1.0000e+00],\n",
      "        [1.0000e+00, 2.4748e-26],\n",
      "        [1.0000e+00, 1.9744e-06],\n",
      "        [1.0000e+00, 4.1684e-08],\n",
      "        [1.0000e+00, 3.1118e-12],\n",
      "        [3.1721e-13, 1.0000e+00],\n",
      "        [1.0000e+00, 5.6710e-29],\n",
      "        [1.0000e+00, 5.7277e-10],\n",
      "        [7.2064e-21, 1.0000e+00],\n",
      "        [1.0000e+00, 3.6344e-08],\n",
      "        [1.0000e+00, 7.1584e-21],\n",
      "        [7.3325e-06, 9.9999e-01],\n",
      "        [8.4707e-23, 1.0000e+00],\n",
      "        [2.3052e-13, 1.0000e+00],\n",
      "        [1.0000e+00, 5.2915e-27],\n",
      "        [1.0000e+00, 2.0903e-11],\n",
      "        [9.2401e-06, 9.9999e-01],\n",
      "        [1.1803e-02, 9.8820e-01],\n",
      "        [1.0000e+00, 1.8027e-22],\n",
      "        [1.0000e+00, 1.6292e-08],\n",
      "        [9.6835e-14, 1.0000e+00]])\n",
      "tensor([[1.2256e-09, 1.0000e+00],\n",
      "        [1.0000e+00, 1.5923e-09],\n",
      "        [1.0000e+00, 1.8423e-07],\n",
      "        [1.0000e+00, 5.8340e-10],\n",
      "        [1.0000e+00, 4.0884e-14],\n",
      "        [7.7950e-12, 1.0000e+00],\n",
      "        [1.0000e+00, 1.0644e-11],\n",
      "        [4.3365e-11, 1.0000e+00],\n",
      "        [9.9999e-01, 1.0482e-05],\n",
      "        [3.8303e-26, 1.0000e+00],\n",
      "        [1.0000e+00, 1.3439e-08],\n",
      "        [1.0000e+00, 1.3174e-10],\n",
      "        [2.2556e-10, 1.0000e+00],\n",
      "        [8.0737e-15, 1.0000e+00],\n",
      "        [1.0000e+00, 1.5970e-10],\n",
      "        [1.0150e-25, 1.0000e+00],\n",
      "        [3.2726e-33, 1.0000e+00],\n",
      "        [1.2853e-01, 8.7147e-01],\n",
      "        [5.4289e-04, 9.9946e-01],\n",
      "        [1.5300e-05, 9.9998e-01],\n",
      "        [1.0000e+00, 1.6659e-38],\n",
      "        [9.9382e-01, 6.1825e-03],\n",
      "        [1.1881e-20, 1.0000e+00],\n",
      "        [1.0000e+00, 6.5959e-17],\n",
      "        [9.9995e-01, 4.7600e-05],\n",
      "        [0.0000e+00, 1.0000e+00],\n",
      "        [8.0561e-01, 1.9439e-01],\n",
      "        [1.0000e+00, 2.5298e-23],\n",
      "        [1.0000e+00, 7.5577e-11],\n",
      "        [1.0000e+00, 4.4243e-15],\n",
      "        [1.0000e+00, 2.3660e-15],\n",
      "        [1.0000e+00, 2.3886e-17],\n",
      "        [3.6992e-11, 1.0000e+00],\n",
      "        [1.0000e+00, 1.2433e-26],\n",
      "        [1.0000e+00, 1.5678e-24]])\n",
      "tensor([[7.1083e-18, 1.0000e+00],\n",
      "        [9.5542e-10, 1.0000e+00],\n",
      "        [9.7940e-01, 2.0596e-02],\n",
      "        [1.8183e-03, 9.9818e-01],\n",
      "        [1.0000e+00, 6.0691e-17],\n",
      "        [2.9750e-21, 1.0000e+00],\n",
      "        [1.0000e+00, 2.5925e-06],\n",
      "        [9.1381e-01, 8.6191e-02],\n",
      "        [1.0000e+00, 1.3956e-08],\n",
      "        [7.8019e-19, 1.0000e+00],\n",
      "        [1.0000e+00, 1.5455e-10],\n",
      "        [1.3295e-02, 9.8670e-01],\n",
      "        [5.9816e-14, 1.0000e+00],\n",
      "        [4.1805e-10, 1.0000e+00],\n",
      "        [1.0000e+00, 1.2013e-21],\n",
      "        [9.9921e-01, 7.9238e-04],\n",
      "        [1.2537e-05, 9.9999e-01],\n",
      "        [1.0000e+00, 7.1260e-20],\n",
      "        [2.7397e-04, 9.9973e-01],\n",
      "        [2.2554e-11, 1.0000e+00],\n",
      "        [2.1102e-28, 1.0000e+00],\n",
      "        [3.0829e-44, 1.0000e+00],\n",
      "        [1.0000e+00, 4.3143e-08],\n",
      "        [3.4359e-02, 9.6564e-01],\n",
      "        [1.0000e+00, 2.9504e-36],\n",
      "        [1.0000e+00, 4.4794e-14],\n",
      "        [3.2859e-15, 1.0000e+00],\n",
      "        [1.0000e+00, 3.7278e-10],\n",
      "        [9.9984e-01, 1.5962e-04],\n",
      "        [1.6664e-10, 1.0000e+00],\n",
      "        [3.7378e-02, 9.6262e-01],\n",
      "        [1.0000e+00, 1.0064e-15],\n",
      "        [9.9943e-01, 5.7423e-04],\n",
      "        [3.7659e-06, 1.0000e+00],\n",
      "        [9.9618e-01, 3.8237e-03]])\n",
      "tensor([[1.0000e+00, 1.1935e-11],\n",
      "        [3.6314e-09, 1.0000e+00],\n",
      "        [1.0000e+00, 9.7662e-15],\n",
      "        [4.0144e-04, 9.9960e-01],\n",
      "        [1.0000e+00, 2.8628e-30],\n",
      "        [1.7548e-07, 1.0000e+00],\n",
      "        [2.4636e-20, 1.0000e+00],\n",
      "        [1.0000e+00, 8.4589e-08],\n",
      "        [1.0000e+00, 8.3285e-07],\n",
      "        [4.1178e-04, 9.9959e-01],\n",
      "        [1.0000e+00, 1.2409e-12],\n",
      "        [1.0000e+00, 1.1294e-13],\n",
      "        [6.7802e-09, 1.0000e+00],\n",
      "        [2.2821e-12, 1.0000e+00],\n",
      "        [2.3322e-21, 1.0000e+00],\n",
      "        [3.9108e-02, 9.6089e-01],\n",
      "        [1.0000e+00, 7.0762e-09],\n",
      "        [4.1233e-33, 1.0000e+00],\n",
      "        [1.0000e+00, 1.6577e-24],\n",
      "        [1.0000e+00, 1.0170e-09],\n",
      "        [9.8809e-01, 1.1913e-02],\n",
      "        [8.7780e-12, 1.0000e+00],\n",
      "        [7.1415e-12, 1.0000e+00],\n",
      "        [3.7046e-27, 1.0000e+00],\n",
      "        [1.0948e-09, 1.0000e+00],\n",
      "        [9.5128e-06, 9.9999e-01],\n",
      "        [4.8230e-09, 1.0000e+00],\n",
      "        [7.1434e-24, 1.0000e+00],\n",
      "        [4.1693e-17, 1.0000e+00],\n",
      "        [2.2335e-20, 1.0000e+00],\n",
      "        [1.0000e+00, 1.2607e-29],\n",
      "        [1.1909e-15, 1.0000e+00],\n",
      "        [2.3340e-32, 1.0000e+00],\n",
      "        [1.0000e+00, 2.4777e-21],\n",
      "        [5.3882e-16, 1.0000e+00]])\n",
      "tensor([[3.1088e-13, 1.0000e+00],\n",
      "        [6.8253e-08, 1.0000e+00],\n",
      "        [1.0000e+00, 9.1060e-14],\n",
      "        [1.0000e+00, 2.4301e-09],\n",
      "        [1.0000e+00, 5.2055e-10],\n",
      "        [1.2365e-31, 1.0000e+00],\n",
      "        [9.9158e-01, 8.4238e-03],\n",
      "        [5.8906e-11, 1.0000e+00],\n",
      "        [5.4984e-23, 1.0000e+00],\n",
      "        [1.0000e+00, 5.8581e-09],\n",
      "        [6.0078e-16, 1.0000e+00],\n",
      "        [3.4602e-12, 1.0000e+00],\n",
      "        [1.0000e+00, 8.9110e-17],\n",
      "        [1.0000e+00, 2.3142e-23],\n",
      "        [9.9987e-01, 1.2556e-04],\n",
      "        [8.0189e-04, 9.9920e-01],\n",
      "        [1.0000e+00, 9.7698e-09],\n",
      "        [7.5757e-12, 1.0000e+00],\n",
      "        [9.4296e-15, 1.0000e+00],\n",
      "        [1.0000e+00, 4.4714e-17],\n",
      "        [3.8752e-28, 1.0000e+00],\n",
      "        [1.3467e-11, 1.0000e+00],\n",
      "        [2.7279e-04, 9.9973e-01],\n",
      "        [2.3825e-09, 1.0000e+00],\n",
      "        [1.1166e-02, 9.8883e-01],\n",
      "        [1.0000e+00, 7.6707e-07],\n",
      "        [4.6969e-10, 1.0000e+00],\n",
      "        [4.9621e-09, 1.0000e+00],\n",
      "        [2.2945e-10, 1.0000e+00],\n",
      "        [5.1383e-30, 1.0000e+00],\n",
      "        [6.2035e-29, 1.0000e+00],\n",
      "        [2.7051e-14, 1.0000e+00],\n",
      "        [5.0889e-03, 9.9491e-01],\n",
      "        [1.4013e-45, 1.0000e+00],\n",
      "        [1.0000e+00, 2.6120e-17]])\n",
      "tensor([[8.5139e-01, 1.4861e-01],\n",
      "        [2.3434e-26, 1.0000e+00],\n",
      "        [1.3703e-41, 1.0000e+00],\n",
      "        [1.2471e-31, 1.0000e+00],\n",
      "        [1.0000e+00, 1.6276e-16],\n",
      "        [1.0000e+00, 6.3258e-21],\n",
      "        [1.0000e+00, 1.8674e-09],\n",
      "        [9.9973e-01, 2.6819e-04],\n",
      "        [1.0000e+00, 2.5342e-17],\n",
      "        [9.3861e-18, 1.0000e+00],\n",
      "        [1.0000e+00, 2.6255e-07],\n",
      "        [1.9134e-17, 1.0000e+00],\n",
      "        [9.2863e-24, 1.0000e+00],\n",
      "        [9.8648e-29, 1.0000e+00],\n",
      "        [5.9232e-03, 9.9408e-01],\n",
      "        [2.6781e-11, 1.0000e+00],\n",
      "        [1.0000e+00, 3.4266e-08],\n",
      "        [9.7251e-34, 1.0000e+00],\n",
      "        [9.9999e-01, 6.4209e-06],\n",
      "        [1.0000e+00, 1.4054e-20],\n",
      "        [1.1716e-28, 1.0000e+00],\n",
      "        [7.8623e-33, 1.0000e+00],\n",
      "        [5.5319e-01, 4.4681e-01],\n",
      "        [1.0000e+00, 4.1377e-09],\n",
      "        [1.0000e+00, 3.2543e-26],\n",
      "        [1.0000e+00, 7.6726e-18],\n",
      "        [4.3142e-19, 1.0000e+00],\n",
      "        [1.0000e+00, 3.7068e-12],\n",
      "        [5.2690e-29, 1.0000e+00],\n",
      "        [7.7093e-06, 9.9999e-01],\n",
      "        [1.0000e+00, 3.2524e-10],\n",
      "        [8.7203e-08, 1.0000e+00],\n",
      "        [1.0000e+00, 6.9661e-20],\n",
      "        [1.0000e+00, 1.2958e-25],\n",
      "        [6.8710e-30, 1.0000e+00]])\n",
      "tensor([[5.5976e-06, 9.9999e-01],\n",
      "        [1.6334e-09, 1.0000e+00],\n",
      "        [2.9298e-03, 9.9707e-01],\n",
      "        [1.0000e+00, 4.1683e-20],\n",
      "        [1.0000e+00, 5.2715e-26],\n",
      "        [9.9970e-01, 3.0434e-04],\n",
      "        [2.7277e-13, 1.0000e+00],\n",
      "        [1.0000e+00, 5.8104e-09],\n",
      "        [4.0643e-14, 1.0000e+00],\n",
      "        [1.0000e+00, 2.2168e-21],\n",
      "        [7.0226e-16, 1.0000e+00],\n",
      "        [4.0366e-13, 1.0000e+00],\n",
      "        [1.0000e+00, 8.0793e-18],\n",
      "        [9.9626e-01, 3.7403e-03],\n",
      "        [1.0000e+00, 4.6307e-14],\n",
      "        [1.0000e+00, 4.0756e-15],\n",
      "        [2.5676e-02, 9.7432e-01],\n",
      "        [3.1762e-10, 1.0000e+00],\n",
      "        [1.6179e-21, 1.0000e+00],\n",
      "        [7.0772e-01, 2.9228e-01],\n",
      "        [1.0000e+00, 1.4378e-16],\n",
      "        [1.0000e+00, 4.9367e-12],\n",
      "        [3.4617e-39, 1.0000e+00],\n",
      "        [9.9985e-01, 1.4924e-04],\n",
      "        [4.7234e-01, 5.2766e-01],\n",
      "        [1.0000e+00, 7.8949e-24],\n",
      "        [7.5324e-27, 1.0000e+00],\n",
      "        [6.4077e-11, 1.0000e+00],\n",
      "        [8.4481e-36, 1.0000e+00],\n",
      "        [9.9981e-01, 1.8654e-04],\n",
      "        [1.0000e+00, 3.1767e-15],\n",
      "        [2.3864e-11, 1.0000e+00],\n",
      "        [6.3410e-19, 1.0000e+00],\n",
      "        [9.9623e-01, 3.7725e-03],\n",
      "        [1.0000e+00, 1.3713e-08]])\n",
      "tensor([[9.8433e-01, 1.5673e-02],\n",
      "        [1.0000e+00, 1.4453e-08],\n",
      "        [3.9657e-43, 1.0000e+00],\n",
      "        [1.0000e+00, 3.6294e-22],\n",
      "        [5.0554e-01, 4.9446e-01],\n",
      "        [1.5973e-07, 1.0000e+00],\n",
      "        [1.0000e+00, 3.7895e-11],\n",
      "        [3.5755e-26, 1.0000e+00],\n",
      "        [7.6930e-22, 1.0000e+00],\n",
      "        [1.0000e+00, 5.7343e-10],\n",
      "        [2.0571e-17, 1.0000e+00],\n",
      "        [1.7914e-12, 1.0000e+00],\n",
      "        [1.2042e-21, 1.0000e+00],\n",
      "        [1.0785e-03, 9.9892e-01],\n",
      "        [9.9993e-01, 7.2569e-05],\n",
      "        [2.2618e-01, 7.7382e-01],\n",
      "        [1.0000e+00, 6.3897e-09],\n",
      "        [3.1420e-07, 1.0000e+00],\n",
      "        [1.0000e+00, 3.9635e-11],\n",
      "        [5.6046e-19, 1.0000e+00],\n",
      "        [6.6800e-39, 1.0000e+00],\n",
      "        [6.9516e-01, 3.0484e-01],\n",
      "        [6.7721e-24, 1.0000e+00],\n",
      "        [1.9507e-33, 1.0000e+00],\n",
      "        [1.0000e+00, 1.5521e-07],\n",
      "        [1.0000e+00, 1.3088e-14],\n",
      "        [3.1529e-10, 1.0000e+00],\n",
      "        [9.9999e-01, 6.3957e-06],\n",
      "        [7.8642e-12, 1.0000e+00],\n",
      "        [2.1795e-06, 1.0000e+00],\n",
      "        [3.0448e-04, 9.9970e-01],\n",
      "        [1.0000e+00, 8.3852e-08],\n",
      "        [1.0000e+00, 1.3159e-11],\n",
      "        [1.0000e+00, 4.4857e-12],\n",
      "        [1.2099e-18, 1.0000e+00]])\n",
      "tensor([[1.0000e+00, 1.3310e-08],\n",
      "        [1.0000e+00, 5.4625e-16],\n",
      "        [8.0101e-13, 1.0000e+00],\n",
      "        [1.9196e-09, 1.0000e+00],\n",
      "        [7.0189e-20, 1.0000e+00],\n",
      "        [1.0000e+00, 4.6545e-17],\n",
      "        [1.0000e+00, 3.3091e-09],\n",
      "        [9.9330e-01, 6.6970e-03],\n",
      "        [3.4916e-17, 1.0000e+00],\n",
      "        [1.2987e-03, 9.9870e-01],\n",
      "        [3.6350e-10, 1.0000e+00],\n",
      "        [9.9999e-01, 7.8984e-06],\n",
      "        [2.5575e-26, 1.0000e+00],\n",
      "        [1.0000e+00, 2.1926e-24],\n",
      "        [1.2975e-06, 1.0000e+00],\n",
      "        [1.0000e+00, 2.9543e-07],\n",
      "        [1.1553e-05, 9.9999e-01],\n",
      "        [1.4225e-25, 1.0000e+00],\n",
      "        [8.6666e-03, 9.9133e-01],\n",
      "        [4.8802e-41, 1.0000e+00],\n",
      "        [7.0150e-03, 9.9298e-01],\n",
      "        [1.0000e+00, 4.5134e-24],\n",
      "        [1.8995e-13, 1.0000e+00],\n",
      "        [3.0132e-01, 6.9868e-01],\n",
      "        [1.0000e+00, 3.7341e-11],\n",
      "        [1.4284e-07, 1.0000e+00],\n",
      "        [1.0000e+00, 1.3351e-34],\n",
      "        [8.3391e-14, 1.0000e+00],\n",
      "        [1.0000e+00, 1.4290e-18],\n",
      "        [1.0000e+00, 1.4925e-20],\n",
      "        [1.6413e-13, 1.0000e+00],\n",
      "        [6.0521e-01, 3.9479e-01],\n",
      "        [4.8172e-28, 1.0000e+00],\n",
      "        [3.4730e-13, 1.0000e+00],\n",
      "        [9.8420e-01, 1.5796e-02]])\n",
      "tensor([[1.0000e+00, 8.9615e-19],\n",
      "        [8.3487e-30, 1.0000e+00],\n",
      "        [1.0000e+00, 4.7145e-27],\n",
      "        [7.3719e-08, 1.0000e+00],\n",
      "        [1.0000e+00, 1.5099e-18],\n",
      "        [9.7416e-28, 1.0000e+00],\n",
      "        [5.7065e-14, 1.0000e+00],\n",
      "        [9.9996e-01, 3.7661e-05],\n",
      "        [1.0000e+00, 8.1893e-20],\n",
      "        [3.3308e-14, 1.0000e+00],\n",
      "        [9.9827e-01, 1.7338e-03],\n",
      "        [2.2062e-10, 1.0000e+00],\n",
      "        [1.0000e+00, 2.1109e-11],\n",
      "        [3.5117e-14, 1.0000e+00],\n",
      "        [5.6676e-27, 1.0000e+00],\n",
      "        [1.0000e+00, 3.9996e-12],\n",
      "        [1.0000e+00, 4.2744e-13],\n",
      "        [9.9997e-01, 3.1850e-05],\n",
      "        [1.0000e+00, 2.2244e-22],\n",
      "        [1.0000e+00, 3.4402e-07],\n",
      "        [6.2350e-01, 3.7650e-01],\n",
      "        [1.0000e+00, 7.8464e-19],\n",
      "        [1.0000e+00, 1.4423e-10],\n",
      "        [5.7505e-09, 1.0000e+00],\n",
      "        [1.3163e-08, 1.0000e+00],\n",
      "        [9.7805e-01, 2.1953e-02],\n",
      "        [1.0000e+00, 1.3055e-10],\n",
      "        [5.4092e-22, 1.0000e+00],\n",
      "        [1.0000e+00, 2.6004e-13],\n",
      "        [3.1595e-14, 1.0000e+00],\n",
      "        [2.6679e-31, 1.0000e+00],\n",
      "        [7.6009e-01, 2.3991e-01],\n",
      "        [8.5115e-21, 1.0000e+00],\n",
      "        [3.8829e-06, 1.0000e+00],\n",
      "        [9.9960e-01, 4.0446e-04]])\n",
      "tensor([[7.4141e-27, 1.0000e+00],\n",
      "        [1.0000e+00, 2.5914e-13],\n",
      "        [9.9959e-01, 4.1287e-04],\n",
      "        [9.1225e-01, 8.7754e-02],\n",
      "        [1.0000e+00, 7.2421e-16],\n",
      "        [9.2844e-01, 7.1556e-02],\n",
      "        [1.3744e-01, 8.6256e-01],\n",
      "        [1.0000e+00, 5.8037e-16],\n",
      "        [9.9998e-01, 2.0155e-05],\n",
      "        [9.9999e-01, 4.9604e-06],\n",
      "        [8.8263e-20, 1.0000e+00],\n",
      "        [4.7069e-19, 1.0000e+00],\n",
      "        [9.7678e-01, 2.3225e-02],\n",
      "        [1.0000e+00, 2.6892e-23],\n",
      "        [3.7650e-04, 9.9962e-01],\n",
      "        [9.9970e-01, 2.9611e-04],\n",
      "        [7.1185e-09, 1.0000e+00],\n",
      "        [9.9974e-01, 2.6188e-04],\n",
      "        [0.0000e+00, 1.0000e+00],\n",
      "        [9.9961e-01, 3.9391e-04],\n",
      "        [5.6220e-01, 4.3780e-01],\n",
      "        [3.3374e-09, 1.0000e+00],\n",
      "        [6.4692e-05, 9.9994e-01],\n",
      "        [2.5470e-29, 1.0000e+00],\n",
      "        [6.0208e-14, 1.0000e+00],\n",
      "        [1.0000e+00, 7.3097e-11],\n",
      "        [1.0000e+00, 8.5138e-09],\n",
      "        [8.0375e-12, 1.0000e+00],\n",
      "        [9.9995e-01, 4.6375e-05],\n",
      "        [9.9996e-01, 3.5275e-05],\n",
      "        [1.7817e-22, 1.0000e+00],\n",
      "        [1.0000e+00, 1.9036e-08],\n",
      "        [9.5906e-01, 4.0939e-02],\n",
      "        [1.0000e+00, 1.8384e-14],\n",
      "        [1.4732e-13, 1.0000e+00]])\n",
      "Correct: 342 / 420 - 81%\n"
     ]
    }
   ],
   "source": [
    "net.eval()\n",
    "total = 0\n",
    "correct = 0\n",
    "with torch.no_grad():\n",
    "    for i, data in enumerate(test_dataloader, 0):\n",
    "        features, labels = data\n",
    "        out = net(features.float())\n",
    "        preds = F.log_softmax(out, dim=1).argmax(dim=1)\n",
    "        total += labels.size(0)\n",
    "        correct += (preds == labels).sum().item()\n",
    "\n",
    "model_accuracy = int(correct / total * 100)\n",
    "print(\"Correct:\", correct, \"/\", total, \"-\", f\"{model_accuracy}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6cfa73f",
   "metadata": {},
   "source": [
    "## Save the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "a4cc6c12",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "timestamp = str(datetime.now())\n",
    "torch.save(net.state_dict(), f\"../models/{timestamp}-Muse_EEG_eyes_open-{model_accuracy}percent.pt\")\n",
    "with open(f\"../models/{timestamp}-Muse_EEG_eyes_open-{model_accuracy}percent.model\", \"w\") as f:\n",
    "    f.write(str(net))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "284c2fa6",
   "metadata": {},
   "source": [
    "## Raw EEG + FFT Model\n",
    "\n",
    "While our previous model had an impressive accuracy given the size of our dataset, it can probably be improved by adding more features. We will provide outputs from a Fast Fourier Transform as additional features, banded over the primary brainwave bands - `Alpha`, `Beta`, `Theta`, `Delta`, `Gamma` (possibly split into `HighGamma` and `LowGamma`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e97a98d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "66753322",
   "metadata": {},
   "source": [
    "## Live Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "1298d5ff",
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "mat1 and mat2 shapes cannot be multiplied (1x1984 and 224x512)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-186-31503514a942>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmuse2_recorder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseconds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    137\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/neurothink--t8FFcF9/lib/python3.9/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1845\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhas_torch_function_variadic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1846\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1847\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1848\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (1x1984 and 224x512)"
     ]
    }
   ],
   "source": [
    "# TODO\n",
    "muse2_recorder.connect()\n",
    "with torch.no_grad():\n",
    "    for samples in muse2_recorder.stream(timedelta(seconds=5)):\n",
    "        samples = np.expand_dims(samples[:, :4].T, axis=0)\n",
    "        net(torch.from_numpy(samples).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0463713",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}