CNN_model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9484ab86",
   "metadata": {},
   "source": [
    "## CNN model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "0fbfc786",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40853587925449375\n"
     ]
    }
   ],
   "source": [
    "## setting random seed \n",
    "\n",
    "import random\n",
    "random.seed(44)\n",
    "print(random.random())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3f460ab",
   "metadata": {},
   "source": [
    "## Gridsearch  over class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4aea58b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1, 2]), array([528, 528, 528], dtype=int64))\n",
      "testing weight: {0: 1, 1: 1, 2: 1}\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1781 - accuracy: 0.3630\n",
      "Epoch 1: val_loss improved from inf to 0.98714, saving model to model1.h5\n",
      "99/99 [==============================] - 75s 718ms/step - loss: 1.1781 - accuracy: 0.3630 - val_loss: 0.9871 - val_accuracy: 0.7234\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9888 - accuracy: 0.4937\n",
      "Epoch 2: val_loss improved from 0.98714 to 0.88019, saving model to model1.h5\n",
      "99/99 [==============================] - 70s 705ms/step - loss: 0.9888 - accuracy: 0.4937 - val_loss: 0.8802 - val_accuracy: 0.6170\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8698 - accuracy: 0.5789\n",
      "Epoch 3: val_loss improved from 0.88019 to 0.62811, saving model to model1.h5\n",
      "99/99 [==============================] - 94s 956ms/step - loss: 0.8698 - accuracy: 0.5789 - val_loss: 0.6281 - val_accuracy: 0.7447\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7953 - accuracy: 0.6042\n",
      "Epoch 4: val_loss did not improve from 0.62811\n",
      "99/99 [==============================] - 84s 842ms/step - loss: 0.7953 - accuracy: 0.6042 - val_loss: 0.6499 - val_accuracy: 0.7660\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7421 - accuracy: 0.6193\n",
      "Epoch 5: val_loss improved from 0.62811 to 0.62591, saving model to model1.h5\n",
      "99/99 [==============================] - 96s 969ms/step - loss: 0.7421 - accuracy: 0.6193 - val_loss: 0.6259 - val_accuracy: 0.7234\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6938 - accuracy: 0.6484\n",
      "Epoch 6: val_loss improved from 0.62591 to 0.57540, saving model to model1.h5\n",
      "99/99 [==============================] - 62s 625ms/step - loss: 0.6938 - accuracy: 0.6484 - val_loss: 0.5754 - val_accuracy: 0.7660\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6690 - accuracy: 0.6420\n",
      "Epoch 7: val_loss did not improve from 0.57540\n",
      "99/99 [==============================] - 44s 444ms/step - loss: 0.6690 - accuracy: 0.6420 - val_loss: 0.6418 - val_accuracy: 0.7021\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6369 - accuracy: 0.6578\n",
      "Epoch 8: val_loss improved from 0.57540 to 0.56355, saving model to model1.h5\n",
      "99/99 [==============================] - 58s 586ms/step - loss: 0.6369 - accuracy: 0.6578 - val_loss: 0.5635 - val_accuracy: 0.6809\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6219 - accuracy: 0.6799\n",
      "Epoch 9: val_loss improved from 0.56355 to 0.52123, saving model to model1.h5\n",
      "99/99 [==============================] - 51s 517ms/step - loss: 0.6219 - accuracy: 0.6799 - val_loss: 0.5212 - val_accuracy: 0.7872\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6077 - accuracy: 0.6793\n",
      "Epoch 10: val_loss improved from 0.52123 to 0.45782, saving model to model1.h5\n",
      "99/99 [==============================] - 43s 438ms/step - loss: 0.6077 - accuracy: 0.6793 - val_loss: 0.4578 - val_accuracy: 0.8085\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5831 - accuracy: 0.6944\n",
      "Epoch 11: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 46s 462ms/step - loss: 0.5831 - accuracy: 0.6944 - val_loss: 0.6237 - val_accuracy: 0.7021\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5516 - accuracy: 0.7153\n",
      "Epoch 12: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 47s 475ms/step - loss: 0.5516 - accuracy: 0.7153 - val_loss: 0.5213 - val_accuracy: 0.7234\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5137 - accuracy: 0.7178\n",
      "Epoch 13: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 49s 490ms/step - loss: 0.5137 - accuracy: 0.7178 - val_loss: 0.4724 - val_accuracy: 0.8085\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5083 - accuracy: 0.7165\n",
      "Epoch 14: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 81s 824ms/step - loss: 0.5083 - accuracy: 0.7165 - val_loss: 0.5068 - val_accuracy: 0.8298\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4701 - accuracy: 0.7431\n",
      "Epoch 15: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 46s 455ms/step - loss: 0.4701 - accuracy: 0.7431 - val_loss: 0.4772 - val_accuracy: 0.8085\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4453 - accuracy: 0.7639\n",
      "Epoch 16: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 45s 452ms/step - loss: 0.4453 - accuracy: 0.7639 - val_loss: 0.4954 - val_accuracy: 0.7872\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4417 - accuracy: 0.7481\n",
      "Epoch 17: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 43s 439ms/step - loss: 0.4417 - accuracy: 0.7481 - val_loss: 0.4807 - val_accuracy: 0.8085\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4250 - accuracy: 0.7639\n",
      "Epoch 18: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 48s 491ms/step - loss: 0.4250 - accuracy: 0.7639 - val_loss: 0.4974 - val_accuracy: 0.7872\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4225 - accuracy: 0.7664\n",
      "Epoch 19: val_loss did not improve from 0.45782\n",
      "99/99 [==============================] - 91s 920ms/step - loss: 0.4225 - accuracy: 0.7664 - val_loss: 0.5220 - val_accuracy: 0.7872\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.4256 - accuracy: 0.7708\n",
      "Epoch 20: val_loss improved from 0.45782 to 0.43042, saving model to model1.h5\n",
      "99/99 [==============================] - 83s 833ms/step - loss: 0.4256 - accuracy: 0.7708 - val_loss: 0.4304 - val_accuracy: 0.7872\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3947 - accuracy: 0.7866\n",
      "Epoch 21: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 81s 819ms/step - loss: 0.3947 - accuracy: 0.7866 - val_loss: 0.5050 - val_accuracy: 0.7872\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3839 - accuracy: 0.7860\n",
      "Epoch 22: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 80s 807ms/step - loss: 0.3839 - accuracy: 0.7860 - val_loss: 0.5686 - val_accuracy: 0.7872\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3688 - accuracy: 0.7936\n",
      "Epoch 23: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 88s 890ms/step - loss: 0.3688 - accuracy: 0.7936 - val_loss: 0.7650 - val_accuracy: 0.7447\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3534 - accuracy: 0.8112\n",
      "Epoch 24: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 85s 863ms/step - loss: 0.3534 - accuracy: 0.8112 - val_loss: 0.6936 - val_accuracy: 0.8085\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3444 - accuracy: 0.8213\n",
      "Epoch 25: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 90s 913ms/step - loss: 0.3444 - accuracy: 0.8213 - val_loss: 0.5911 - val_accuracy: 0.7872\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3194 - accuracy: 0.8378\n",
      "Epoch 26: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 95s 957ms/step - loss: 0.3194 - accuracy: 0.8378 - val_loss: 0.8492 - val_accuracy: 0.7872\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3222 - accuracy: 0.8314\n",
      "Epoch 27: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 47s 473ms/step - loss: 0.3222 - accuracy: 0.8314 - val_loss: 0.9262 - val_accuracy: 0.8298\n",
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3180 - accuracy: 0.8409\n",
      "Epoch 28: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 45s 459ms/step - loss: 0.3180 - accuracy: 0.8409 - val_loss: 0.9246 - val_accuracy: 0.7660\n",
      "Epoch 29/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.2854 - accuracy: 0.8617\n",
      "Epoch 29: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 46s 462ms/step - loss: 0.2854 - accuracy: 0.8617 - val_loss: 0.5490 - val_accuracy: 0.7660\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.3042 - accuracy: 0.8554\n",
      "Epoch 30: val_loss did not improve from 0.43042\n",
      "99/99 [==============================] - 44s 446ms/step - loss: 0.3042 - accuracy: 0.8554 - val_loss: 0.5258 - val_accuracy: 0.7447\n",
      "validation loss:  0.43042194843292236\n",
      "validation accuracy:  0.7872340679168701\n",
      "2/2 [==============================] - 0s 69ms/step\n",
      "balanced accuracy:  0.6030039255845708\n",
      "[[30  1  0]\n",
      " [ 1  5  3]\n",
      " [ 1  4  2]]\n",
      "new best weight is: {0: 1, 1: 1, 2: 1}\n",
      "new best acc is:  0.6030039255845708\n",
      "testing weight: {0: 1, 1: 2, 2: 3}\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1571 - accuracy: 0.3239\n",
      "Epoch 1: val_loss improved from inf to 1.42731, saving model to model2.h5\n",
      "99/99 [==============================] - 47s 455ms/step - loss: 2.1571 - accuracy: 0.3239 - val_loss: 1.4273 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0000 - accuracy: 0.3415\n",
      "Epoch 2: val_loss did not improve from 1.42731\n",
      "99/99 [==============================] - 64s 649ms/step - loss: 2.0000 - accuracy: 0.3415 - val_loss: 1.4359 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8808 - accuracy: 0.3807\n",
      "Epoch 3: val_loss improved from 1.42731 to 1.06874, saving model to model2.h5\n",
      "99/99 [==============================] - 81s 813ms/step - loss: 1.8808 - accuracy: 0.3807 - val_loss: 1.0687 - val_accuracy: 0.4468\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7487 - accuracy: 0.4893\n",
      "Epoch 4: val_loss did not improve from 1.06874\n",
      "99/99 [==============================] - 77s 776ms/step - loss: 1.7487 - accuracy: 0.4893 - val_loss: 1.1991 - val_accuracy: 0.3404\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6717 - accuracy: 0.5360\n",
      "Epoch 5: val_loss improved from 1.06874 to 1.02257, saving model to model2.h5\n",
      "99/99 [==============================] - 78s 789ms/step - loss: 1.6717 - accuracy: 0.5360 - val_loss: 1.0226 - val_accuracy: 0.4894\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5867 - accuracy: 0.5537\n",
      "Epoch 6: val_loss improved from 1.02257 to 0.83912, saving model to model2.h5\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 1.5867 - accuracy: 0.5537 - val_loss: 0.8391 - val_accuracy: 0.5745\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5174 - accuracy: 0.5985\n",
      "Epoch 7: val_loss did not improve from 0.83912\n",
      "99/99 [==============================] - 75s 755ms/step - loss: 1.5174 - accuracy: 0.5985 - val_loss: 0.8582 - val_accuracy: 0.5957\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5097 - accuracy: 0.5625\n",
      "Epoch 8: val_loss improved from 0.83912 to 0.77924, saving model to model2.h5\n",
      "99/99 [==============================] - 79s 795ms/step - loss: 1.5097 - accuracy: 0.5625 - val_loss: 0.7792 - val_accuracy: 0.6170\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4287 - accuracy: 0.6124\n",
      "Epoch 9: val_loss improved from 0.77924 to 0.63137, saving model to model2.h5\n",
      "99/99 [==============================] - 42s 429ms/step - loss: 1.4287 - accuracy: 0.6124 - val_loss: 0.6314 - val_accuracy: 0.7872\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3796 - accuracy: 0.6250\n",
      "Epoch 10: val_loss did not improve from 0.63137\n",
      "99/99 [==============================] - 51s 517ms/step - loss: 1.3796 - accuracy: 0.6250 - val_loss: 0.6633 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3408 - accuracy: 0.6528\n",
      "Epoch 11: val_loss improved from 0.63137 to 0.59957, saving model to model2.h5\n",
      "99/99 [==============================] - 75s 755ms/step - loss: 1.3408 - accuracy: 0.6528 - val_loss: 0.5996 - val_accuracy: 0.7021\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2718 - accuracy: 0.6667\n",
      "Epoch 12: val_loss did not improve from 0.59957\n",
      "99/99 [==============================] - 92s 928ms/step - loss: 1.2718 - accuracy: 0.6667 - val_loss: 0.7028 - val_accuracy: 0.7021\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3030 - accuracy: 0.6389\n",
      "Epoch 13: val_loss did not improve from 0.59957\n",
      "99/99 [==============================] - 73s 738ms/step - loss: 1.3030 - accuracy: 0.6389 - val_loss: 0.6550 - val_accuracy: 0.7021\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2196 - accuracy: 0.6761\n",
      "Epoch 14: val_loss improved from 0.59957 to 0.58481, saving model to model2.h5\n",
      "99/99 [==============================] - 77s 775ms/step - loss: 1.2196 - accuracy: 0.6761 - val_loss: 0.5848 - val_accuracy: 0.7447\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1742 - accuracy: 0.6843\n",
      "Epoch 15: val_loss improved from 0.58481 to 0.57514, saving model to model2.h5\n",
      "99/99 [==============================] - 72s 723ms/step - loss: 1.1742 - accuracy: 0.6843 - val_loss: 0.5751 - val_accuracy: 0.7234\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1640 - accuracy: 0.6856\n",
      "Epoch 16: val_loss did not improve from 0.57514\n",
      "99/99 [==============================] - 75s 756ms/step - loss: 1.1640 - accuracy: 0.6856 - val_loss: 0.5813 - val_accuracy: 0.7447\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1357 - accuracy: 0.7027\n",
      "Epoch 17: val_loss did not improve from 0.57514\n",
      "99/99 [==============================] - 84s 847ms/step - loss: 1.1357 - accuracy: 0.7027 - val_loss: 0.8887 - val_accuracy: 0.5319\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1146 - accuracy: 0.7096\n",
      "Epoch 18: val_loss improved from 0.57514 to 0.57426, saving model to model2.h5\n",
      "99/99 [==============================] - 80s 803ms/step - loss: 1.1146 - accuracy: 0.7096 - val_loss: 0.5743 - val_accuracy: 0.7660\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0843 - accuracy: 0.7210\n",
      "Epoch 19: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 79s 802ms/step - loss: 1.0843 - accuracy: 0.7210 - val_loss: 0.5834 - val_accuracy: 0.7660\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0111 - accuracy: 0.7487\n",
      "Epoch 20: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 77s 775ms/step - loss: 1.0111 - accuracy: 0.7487 - val_loss: 0.6209 - val_accuracy: 0.7660\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9744 - accuracy: 0.7790\n",
      "Epoch 21: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 76s 773ms/step - loss: 0.9744 - accuracy: 0.7790 - val_loss: 0.7481 - val_accuracy: 0.7447\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9618 - accuracy: 0.7601\n",
      "Epoch 22: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 84s 849ms/step - loss: 0.9618 - accuracy: 0.7601 - val_loss: 0.5994 - val_accuracy: 0.7660\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9380 - accuracy: 0.7683\n",
      "Epoch 23: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 75s 761ms/step - loss: 0.9380 - accuracy: 0.7683 - val_loss: 0.6669 - val_accuracy: 0.7660\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8672 - accuracy: 0.8157\n",
      "Epoch 24: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 86s 871ms/step - loss: 0.8672 - accuracy: 0.8157 - val_loss: 0.6917 - val_accuracy: 0.7660\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8316 - accuracy: 0.8213\n",
      "Epoch 25: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 77s 771ms/step - loss: 0.8316 - accuracy: 0.8213 - val_loss: 0.7862 - val_accuracy: 0.7660\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7587 - accuracy: 0.8485\n",
      "Epoch 26: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 71s 722ms/step - loss: 0.7587 - accuracy: 0.8485 - val_loss: 0.7993 - val_accuracy: 0.7872\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7224 - accuracy: 0.8668\n",
      "Epoch 27: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 77s 779ms/step - loss: 0.7224 - accuracy: 0.8668 - val_loss: 0.9077 - val_accuracy: 0.7660\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7362 - accuracy: 0.8497\n",
      "Epoch 28: val_loss did not improve from 0.57426\n",
      "99/99 [==============================] - 72s 728ms/step - loss: 0.7362 - accuracy: 0.8497 - val_loss: 0.7093 - val_accuracy: 0.7660\n",
      "validation loss:  0.5742597579956055\n",
      "validation accuracy:  0.7659574747085571\n",
      "2/2 [==============================] - 1s 158ms/step\n",
      "balanced accuracy:  0.6345792797405699\n",
      "[[29  2  0]\n",
      " [ 2  1  6]\n",
      " [ 1  0  6]]\n",
      "new best weight is: {0: 1, 1: 2, 2: 3}\n",
      "new best acc is:  0.6345792797405699\n",
      "testing weight: {0: 2, 1: 3, 2: 4}\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 3.3390 - accuracy: 0.3371\n",
      "Epoch 1: val_loss improved from inf to 1.27044, saving model to model3.h5\n",
      "99/99 [==============================] - 100s 935ms/step - loss: 3.3390 - accuracy: 0.3371 - val_loss: 1.2704 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 3.1235 - accuracy: 0.3529\n",
      "Epoch 2: val_loss improved from 1.27044 to 1.09636, saving model to model3.h5\n",
      "99/99 [==============================] - 78s 785ms/step - loss: 3.1235 - accuracy: 0.3529 - val_loss: 1.0964 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.8028 - accuracy: 0.4628\n",
      "Epoch 3: val_loss improved from 1.09636 to 0.93344, saving model to model3.h5\n",
      "99/99 [==============================] - 85s 861ms/step - loss: 2.8028 - accuracy: 0.4628 - val_loss: 0.9334 - val_accuracy: 0.5319\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.4829 - accuracy: 0.5505\n",
      "Epoch 4: val_loss improved from 0.93344 to 0.60435, saving model to model3.h5\n",
      "99/99 [==============================] - 86s 869ms/step - loss: 2.4829 - accuracy: 0.5505 - val_loss: 0.6044 - val_accuracy: 0.7234\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.3353 - accuracy: 0.5878\n",
      "Epoch 5: val_loss did not improve from 0.60435\n",
      "99/99 [==============================] - 73s 740ms/step - loss: 2.3353 - accuracy: 0.5878 - val_loss: 0.6186 - val_accuracy: 0.7234\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1457 - accuracy: 0.6288\n",
      "Epoch 6: val_loss did not improve from 0.60435\n",
      "99/99 [==============================] - 76s 764ms/step - loss: 2.1457 - accuracy: 0.6288 - val_loss: 0.7859 - val_accuracy: 0.6170\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0888 - accuracy: 0.6307\n",
      "Epoch 7: val_loss improved from 0.60435 to 0.47290, saving model to model3.h5\n",
      "99/99 [==============================] - 80s 807ms/step - loss: 2.0888 - accuracy: 0.6307 - val_loss: 0.4729 - val_accuracy: 0.8085\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9923 - accuracy: 0.6452\n",
      "Epoch 8: val_loss did not improve from 0.47290\n",
      "99/99 [==============================] - 76s 766ms/step - loss: 1.9923 - accuracy: 0.6452 - val_loss: 0.4932 - val_accuracy: 0.7660\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9352 - accuracy: 0.6654\n",
      "Epoch 9: val_loss did not improve from 0.47290\n",
      "99/99 [==============================] - 84s 844ms/step - loss: 1.9352 - accuracy: 0.6654 - val_loss: 0.5535 - val_accuracy: 0.7447\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7826 - accuracy: 0.6824\n",
      "Epoch 10: val_loss did not improve from 0.47290\n",
      "99/99 [==============================] - 72s 728ms/step - loss: 1.7826 - accuracy: 0.6824 - val_loss: 0.5306 - val_accuracy: 0.7234\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7295 - accuracy: 0.6976\n",
      "Epoch 11: val_loss did not improve from 0.47290\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 1.7295 - accuracy: 0.6976 - val_loss: 0.5134 - val_accuracy: 0.7234\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6815 - accuracy: 0.7165\n",
      "Epoch 12: val_loss improved from 0.47290 to 0.43184, saving model to model3.h5\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 1.6815 - accuracy: 0.7165 - val_loss: 0.4318 - val_accuracy: 0.7872\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5762 - accuracy: 0.7210\n",
      "Epoch 13: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 72s 728ms/step - loss: 1.5762 - accuracy: 0.7210 - val_loss: 0.6136 - val_accuracy: 0.7234\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5132 - accuracy: 0.7311\n",
      "Epoch 14: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 72s 727ms/step - loss: 1.5132 - accuracy: 0.7311 - val_loss: 0.4796 - val_accuracy: 0.7872\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5146 - accuracy: 0.7367\n",
      "Epoch 15: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 73s 736ms/step - loss: 1.5146 - accuracy: 0.7367 - val_loss: 0.5340 - val_accuracy: 0.7660\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4038 - accuracy: 0.7532\n",
      "Epoch 16: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 115s 1s/step - loss: 1.4038 - accuracy: 0.7532 - val_loss: 0.5064 - val_accuracy: 0.7447\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3636 - accuracy: 0.7828\n",
      "Epoch 17: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 78s 786ms/step - loss: 1.3636 - accuracy: 0.7828 - val_loss: 0.5011 - val_accuracy: 0.8085\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2662 - accuracy: 0.7822\n",
      "Epoch 18: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 79s 799ms/step - loss: 1.2662 - accuracy: 0.7822 - val_loss: 0.6961 - val_accuracy: 0.7447\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2154 - accuracy: 0.8100\n",
      "Epoch 19: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 76s 764ms/step - loss: 1.2154 - accuracy: 0.8100 - val_loss: 0.6063 - val_accuracy: 0.8511\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1723 - accuracy: 0.8321\n",
      "Epoch 20: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 74s 750ms/step - loss: 1.1723 - accuracy: 0.8321 - val_loss: 0.6722 - val_accuracy: 0.8298\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1888 - accuracy: 0.8201\n",
      "Epoch 21: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 70s 710ms/step - loss: 1.1888 - accuracy: 0.8201 - val_loss: 0.7170 - val_accuracy: 0.7872\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1004 - accuracy: 0.8346\n",
      "Epoch 22: val_loss did not improve from 0.43184\n",
      "99/99 [==============================] - 70s 707ms/step - loss: 1.1004 - accuracy: 0.8346 - val_loss: 0.5927 - val_accuracy: 0.8085\n",
      "validation loss:  0.43183794617652893\n",
      "validation accuracy:  0.7872340679168701\n",
      "2/2 [==============================] - 1s 144ms/step\n",
      "balanced accuracy:  0.5978835978835979\n",
      "[[31  0  0]\n",
      " [ 2  2  5]\n",
      " [ 1  2  4]]\n"
     ]
    }
   ],
   "source": [
    "## import modules\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from keras.utils import np_utils\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from PIL import Image\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Conv2D, MaxPooling2D, Flatten\n",
    "from keras import optimizers\n",
    "from keras import losses\n",
    "from tensorflow.keras.layers import AveragePooling2D\n",
    "import cv2\n",
    "from sklearn import metrics\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import load_model\n",
    "from tensorflow.keras.initializers import he_uniform\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer, balanced_accuracy_score\n",
    "from scikeras.wrappers import KerasClassifier\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from Evaluation import * \n",
    "from tensorflow.keras.initializers import he_uniform\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.models import load_model\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "\n",
    "\n",
    "## loading data\n",
    "x_train = np.load(\"created_data/MIAS/patches/MIAS_X_train_roi_multi.npy\") / 255\n",
    "y_train = np.load(\"created_data/MIAS/patches/MIAS_y_train_roi_multi.npy\")\n",
    "x_valid = np.load(\"created_data/MIAS/patches/MIAS_X_valid_roi_multi.npy\") / 255\n",
    "y_valid = np.load(\"created_data/MIAS/patches/MIAS_y_valid_roi_multi.npy\") \n",
    "x_test = np.load(\"created_data/MIAS/patches/MIAS_X_test_roi_multi.npy\") / 255\n",
    "y_test = np.load(\"created_data/MIAS/patches/MIAS_y_test_roi_multi.npy\") \n",
    "\n",
    "## printing class counts\n",
    "print(np.unique(y_train, return_counts = True))\n",
    "\n",
    "# One-hot encode labels\n",
    "num_classes = 3\n",
    "y_train = np_utils.to_categorical(y_train, num_classes)\n",
    "y_valid = np_utils.to_categorical(y_valid, num_classes)\n",
    "y_test = np_utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "## define class_weights to search over \n",
    "class_weights = [{0: 1, 1: 1, 2: 1}, {0: 1, 1: 2, 2: 3}, {0: 2, 1: 3, 2: 4}]\n",
    "\n",
    "## define model names for each class_weight choice\n",
    "model_names = ['model1.h5', 'model2.h5', 'model3.h5']\n",
    "\n",
    "## store best balanced accuracy and class_weight\n",
    "best_acc = 0\n",
    "best_class_weight = None\n",
    "\n",
    "for w, name in zip(class_weights,model_names):\n",
    "    print(f\"testing weight: {w}\")\n",
    "    # Define model\n",
    "    model = Sequential()\n",
    "\n",
    "    # Add layers to model\n",
    "    \n",
    "    ## first convolutional layer and one maxpooling layer \n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', input_shape=(224, 224, 3), strides = 1, kernel_initializer=he_uniform(), name = 'conv2d'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d'))\n",
    "    \n",
    "    ## second convolutional layer and one maxpooling layer\n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', strides = 1, kernel_initializer=he_uniform(), name = 'conv2d_1'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_1'))\n",
    "    \n",
    "    ## third convolutional layer and 2 maxpooling layers\n",
    "    model.add(Conv2D(14, (3, 3), activation='relu', strides = 1, padding = \"SAME\", kernel_initializer=he_uniform(), name = 'conv2d_2'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_2'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_3'))\n",
    "    \n",
    "    ## flattening model \n",
    "    model.add(Flatten(name = 'flatten'))\n",
    "    \n",
    "    ## 3 dense layers\n",
    "    model.add(Dense(512, activation='relu', kernel_initializer=he_uniform(), name = 'dense'))\n",
    "    model.add(Dense(256, activation='relu', kernel_initializer=he_uniform(), name = 'dense_1'))\n",
    "    model.add(Dense(128, activation='relu', kernel_initializer=he_uniform(), name = 'dense_2'))\n",
    "    \n",
    "    ## dropout\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    ## output layer\n",
    "    model.add(Dense(3, activation='softmax', kernel_initializer=he_uniform(), name = 'dense_3'))\n",
    "\n",
    "    lr = 0.0001 # set the learning rate\n",
    "    adam = Adam(learning_rate=lr) ## adam optimizer \n",
    "\n",
    "    ## compile model \n",
    "    model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    # Define early stopping callback \n",
    "    early_stop = EarlyStopping(monitor='val_loss', patience=10,restore_best_weights=True)\n",
    "    \n",
    "    ## save model with best validation loss \n",
    "    model_checkpoint = ModelCheckpoint(name, save_best_only=True, verbose = 1)\n",
    "\n",
    "    # Train model with early stopping\n",
    "    history = model.fit(x_train, y_train, epochs = 50, batch_size = 16, validation_data = (x_valid, y_valid),\n",
    "    class_weight = w, callbacks = [early_stop,model_checkpoint])\n",
    "\n",
    "    ## load best model\n",
    "    model = load_model(name)\n",
    "    \n",
    "    # Evaluate model on validation set using loss and overall accuracy\n",
    "    valid_loss, valid_acc = model.evaluate(x_valid, y_valid, verbose=0)\n",
    "    print(\"validation loss: \", valid_loss)\n",
    "    print(\"validation accuracy: \", valid_acc)\n",
    "    \n",
    "    ## getting predictions for validation set\n",
    "    y_pred = model.predict(x_valid)\n",
    "    preds_class = np.argmax(y_pred, axis=1)\n",
    "    obs = np.argmax(y_valid, axis=1)\n",
    "\n",
    "    ## compute balanced accuracy\n",
    "    balanced_acc = balanced_accuracy_score(obs, preds_class)\n",
    "    print(\"balanced accuracy: \", balanced_acc)\n",
    "\n",
    "    ## compute confusion matrix\n",
    "    cf = confusion_matrix(obs, preds_class)\n",
    "    print(cf)\n",
    "\n",
    "    ## compare balanced_accuracy and store best weight and best balanced accuracy\n",
    "    if balanced_acc > best_acc:\n",
    "        best_acc = balanced_acc\n",
    "        best_class_weight = w\n",
    "        best_model = name\n",
    "        print(\"new best weight is:\", w)\n",
    "        print(\"new best balanced acc is: \", balanced_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e823a984",
   "metadata": {},
   "source": [
    "## Results of best model on validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "2fe6bf9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the best balanced accuracy is 0.6345792797405699 with the class weight {0: 1, 1: 2, 2: 3}\n",
      "2/2 [==============================] - 0s 110ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.91      0.94      0.92        31\n",
      "     Class 1       0.33      0.11      0.17         9\n",
      "     Class 2       0.50      0.86      0.63         7\n",
      "\n",
      "    accuracy                           0.77        47\n",
      "   macro avg       0.58      0.63      0.57        47\n",
      "weighted avg       0.74      0.77      0.73        47\n",
      "\n",
      "2/2 [==============================] - 1s 174ms/step\n",
      "balanced accuracy:  0.6345792797405699\n",
      "[[29  2  0]\n",
      " [ 2  1  6]\n",
      " [ 1  0  6]]\n"
     ]
    }
   ],
   "source": [
    "## extracting best weights: {0:1, 1:2, 2:3 } with best balanced accuracy 0.634\n",
    "print(f\"the best balanced accuracy is {best_acc} with the class weight {best_class_weight}\")\n",
    "\n",
    "\n",
    "## loading the best model \n",
    "model = load_model(\"model2.h5\")\n",
    "\n",
    "## evaluate model on validation set \n",
    "report = evaluate_model(model,x_valid,y_valid)\n",
    "print(report)\n",
    "\n",
    "## getting predictions on validation set \n",
    "y_pred = model.predict(x_valid)\n",
    "preds_class = np.argmax(y_pred, axis=1)\n",
    "obs = np.argmax(y_valid, axis=1)\n",
    "\n",
    " ## compute balanced accuracy\n",
    "balanced_acc = balanced_accuracy_score(obs, preds_class)\n",
    "print(\"balanced accuracy: \", balanced_acc)\n",
    "\n",
    "## confusion matrix\n",
    "cf = confusion_matrix(obs, preds_class)\n",
    "print(cf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18f21ae6",
   "metadata": {},
   "source": [
    "## Running model with best weights and averaging the confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "b29097e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1, 2]), array([528, 528, 528], dtype=int64))\n",
      "run number 0\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1496 - accuracy: 0.3415\n",
      "Epoch 1: val_loss improved from inf to 1.29120, saving model to best_model.h5\n",
      "99/99 [==============================] - 97s 922ms/step - loss: 2.1496 - accuracy: 0.3415 - val_loss: 1.2912 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8888 - accuracy: 0.4034\n",
      "Epoch 2: val_loss improved from 1.29120 to 0.90513, saving model to best_model.h5\n",
      "99/99 [==============================] - 83s 840ms/step - loss: 1.8888 - accuracy: 0.4034 - val_loss: 0.9051 - val_accuracy: 0.5532\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7157 - accuracy: 0.5032\n",
      "Epoch 3: val_loss improved from 0.90513 to 0.68905, saving model to best_model.h5\n",
      "99/99 [==============================] - 74s 744ms/step - loss: 1.7157 - accuracy: 0.5032 - val_loss: 0.6891 - val_accuracy: 0.7872\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6179 - accuracy: 0.5543\n",
      "Epoch 4: val_loss did not improve from 0.68905\n",
      "99/99 [==============================] - 73s 733ms/step - loss: 1.6179 - accuracy: 0.5543 - val_loss: 1.6230 - val_accuracy: 0.2553\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6423 - accuracy: 0.5391\n",
      "Epoch 5: val_loss did not improve from 0.68905\n",
      "99/99 [==============================] - 71s 719ms/step - loss: 1.6423 - accuracy: 0.5391 - val_loss: 1.1220 - val_accuracy: 0.3404\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5134 - accuracy: 0.5783\n",
      "Epoch 6: val_loss improved from 0.68905 to 0.57889, saving model to best_model.h5\n",
      "99/99 [==============================] - 75s 754ms/step - loss: 1.5134 - accuracy: 0.5783 - val_loss: 0.5789 - val_accuracy: 0.7872\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5185 - accuracy: 0.5890\n",
      "Epoch 7: val_loss did not improve from 0.57889\n",
      "99/99 [==============================] - 83s 842ms/step - loss: 1.5185 - accuracy: 0.5890 - val_loss: 0.6085 - val_accuracy: 0.7447\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4430 - accuracy: 0.6111\n",
      "Epoch 8: val_loss improved from 0.57889 to 0.56120, saving model to best_model.h5\n",
      "99/99 [==============================] - 84s 850ms/step - loss: 1.4430 - accuracy: 0.6111 - val_loss: 0.5612 - val_accuracy: 0.7660\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3922 - accuracy: 0.6288\n",
      "Epoch 9: val_loss did not improve from 0.56120\n",
      "99/99 [==============================] - 103s 1s/step - loss: 1.3922 - accuracy: 0.6288 - val_loss: 0.6085 - val_accuracy: 0.7234\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3571 - accuracy: 0.6206\n",
      "Epoch 10: val_loss improved from 0.56120 to 0.53640, saving model to best_model.h5\n",
      "99/99 [==============================] - 45s 455ms/step - loss: 1.3571 - accuracy: 0.6206 - val_loss: 0.5364 - val_accuracy: 0.7447\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2937 - accuracy: 0.6540\n",
      "Epoch 11: val_loss did not improve from 0.53640\n",
      "99/99 [==============================] - 47s 471ms/step - loss: 1.2937 - accuracy: 0.6540 - val_loss: 0.5610 - val_accuracy: 0.7234\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2848 - accuracy: 0.6503\n",
      "Epoch 12: val_loss did not improve from 0.53640\n",
      "99/99 [==============================] - 65s 659ms/step - loss: 1.2848 - accuracy: 0.6503 - val_loss: 0.5390 - val_accuracy: 0.7660\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2332 - accuracy: 0.6774\n",
      "Epoch 13: val_loss did not improve from 0.53640\n",
      "99/99 [==============================] - 96s 973ms/step - loss: 1.2332 - accuracy: 0.6774 - val_loss: 0.6347 - val_accuracy: 0.6809\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2207 - accuracy: 0.6780\n",
      "Epoch 14: val_loss improved from 0.53640 to 0.50246, saving model to best_model.h5\n",
      "99/99 [==============================] - 89s 896ms/step - loss: 1.2207 - accuracy: 0.6780 - val_loss: 0.5025 - val_accuracy: 0.8085\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1957 - accuracy: 0.6875\n",
      "Epoch 15: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 82s 830ms/step - loss: 1.1957 - accuracy: 0.6875 - val_loss: 0.5635 - val_accuracy: 0.7447\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1631 - accuracy: 0.6888\n",
      "Epoch 16: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 1.1631 - accuracy: 0.6888 - val_loss: 0.5502 - val_accuracy: 0.7234\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1260 - accuracy: 0.7128\n",
      "Epoch 17: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 88s 887ms/step - loss: 1.1260 - accuracy: 0.7128 - val_loss: 0.6193 - val_accuracy: 0.7234\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0799 - accuracy: 0.7247\n",
      "Epoch 18: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 82s 826ms/step - loss: 1.0799 - accuracy: 0.7247 - val_loss: 0.5411 - val_accuracy: 0.7447\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0787 - accuracy: 0.7229\n",
      "Epoch 19: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 89s 898ms/step - loss: 1.0787 - accuracy: 0.7229 - val_loss: 0.5889 - val_accuracy: 0.7660\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0097 - accuracy: 0.7443\n",
      "Epoch 20: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 85s 858ms/step - loss: 1.0097 - accuracy: 0.7443 - val_loss: 0.5936 - val_accuracy: 0.7872\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9461 - accuracy: 0.7759\n",
      "Epoch 21: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 80s 812ms/step - loss: 0.9461 - accuracy: 0.7759 - val_loss: 0.6030 - val_accuracy: 0.8298\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9443 - accuracy: 0.7784\n",
      "Epoch 22: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 80s 807ms/step - loss: 0.9443 - accuracy: 0.7784 - val_loss: 0.5205 - val_accuracy: 0.7872\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9291 - accuracy: 0.7721\n",
      "Epoch 23: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 0.9291 - accuracy: 0.7721 - val_loss: 0.5879 - val_accuracy: 0.7660\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8743 - accuracy: 0.7929\n",
      "Epoch 24: val_loss did not improve from 0.50246\n",
      "99/99 [==============================] - 77s 777ms/step - loss: 0.8743 - accuracy: 0.7929 - val_loss: 0.5583 - val_accuracy: 0.7660\n",
      "Test loss:  0.440203994512558\n",
      "Test accuracy:  0.7916666865348816\n",
      "2/2 [==============================] - 1s 191ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.97      0.94      0.95        31\n",
      "     Class 1       0.55      0.67      0.60         9\n",
      "     Class 2       0.43      0.38      0.40         8\n",
      "\n",
      "    accuracy                           0.79        48\n",
      "   macro avg       0.65      0.66      0.65        48\n",
      "weighted avg       0.80      0.79      0.79        48\n",
      "\n",
      "2/2 [==============================] - 1s 176ms/step\n",
      "[[29  1  1]\n",
      " [ 0  6  3]\n",
      " [ 1  4  3]]\n",
      "run number 1\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0643 - accuracy: 0.3333\n",
      "Epoch 1: val_loss improved from inf to 1.33548, saving model to mod_.h5\n",
      "99/99 [==============================] - 91s 883ms/step - loss: 2.0643 - accuracy: 0.3333 - val_loss: 1.3355 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9433 - accuracy: 0.3630\n",
      "Epoch 2: val_loss did not improve from 1.33548\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 1.9433 - accuracy: 0.3630 - val_loss: 1.3572 - val_accuracy: 0.2128\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7922 - accuracy: 0.4596\n",
      "Epoch 3: val_loss improved from 1.33548 to 0.72196, saving model to mod_.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99/99 [==============================] - 77s 777ms/step - loss: 1.7922 - accuracy: 0.4596 - val_loss: 0.7220 - val_accuracy: 0.6809\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6565 - accuracy: 0.5271\n",
      "Epoch 4: val_loss improved from 0.72196 to 0.67328, saving model to mod_.h5\n",
      "99/99 [==============================] - 73s 735ms/step - loss: 1.6565 - accuracy: 0.5271 - val_loss: 0.6733 - val_accuracy: 0.7447\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5334 - accuracy: 0.5827\n",
      "Epoch 5: val_loss did not improve from 0.67328\n",
      "99/99 [==============================] - 85s 864ms/step - loss: 1.5334 - accuracy: 0.5827 - val_loss: 0.7680 - val_accuracy: 0.5957\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5731 - accuracy: 0.5726\n",
      "Epoch 6: val_loss did not improve from 0.67328\n",
      "99/99 [==============================] - 95s 954ms/step - loss: 1.5731 - accuracy: 0.5726 - val_loss: 0.9016 - val_accuracy: 0.6383\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4596 - accuracy: 0.6035\n",
      "Epoch 7: val_loss did not improve from 0.67328\n",
      "99/99 [==============================] - 100s 1s/step - loss: 1.4596 - accuracy: 0.6035 - val_loss: 0.6825 - val_accuracy: 0.6596\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4270 - accuracy: 0.6155\n",
      "Epoch 8: val_loss did not improve from 0.67328\n",
      "99/99 [==============================] - 85s 862ms/step - loss: 1.4270 - accuracy: 0.6155 - val_loss: 0.6988 - val_accuracy: 0.6596\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3611 - accuracy: 0.6282\n",
      "Epoch 9: val_loss improved from 0.67328 to 0.56468, saving model to mod_.h5\n",
      "99/99 [==============================] - 83s 844ms/step - loss: 1.3611 - accuracy: 0.6282 - val_loss: 0.5647 - val_accuracy: 0.7660\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3152 - accuracy: 0.6389\n",
      "Epoch 10: val_loss improved from 0.56468 to 0.55139, saving model to mod_.h5\n",
      "99/99 [==============================] - 81s 819ms/step - loss: 1.3152 - accuracy: 0.6389 - val_loss: 0.5514 - val_accuracy: 0.7660\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2935 - accuracy: 0.6496\n",
      "Epoch 11: val_loss did not improve from 0.55139\n",
      "99/99 [==============================] - 91s 919ms/step - loss: 1.2935 - accuracy: 0.6496 - val_loss: 0.5933 - val_accuracy: 0.7234\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1891 - accuracy: 0.6970\n",
      "Epoch 12: val_loss improved from 0.55139 to 0.50790, saving model to mod_.h5\n",
      "99/99 [==============================] - 85s 862ms/step - loss: 1.1891 - accuracy: 0.6970 - val_loss: 0.5079 - val_accuracy: 0.8085\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2293 - accuracy: 0.6660\n",
      "Epoch 13: val_loss did not improve from 0.50790\n",
      "99/99 [==============================] - 82s 825ms/step - loss: 1.2293 - accuracy: 0.6660 - val_loss: 0.5675 - val_accuracy: 0.7660\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1153 - accuracy: 0.6938\n",
      "Epoch 14: val_loss improved from 0.50790 to 0.50191, saving model to mod_.h5\n",
      "99/99 [==============================] - 83s 837ms/step - loss: 1.1153 - accuracy: 0.6938 - val_loss: 0.5019 - val_accuracy: 0.7447\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1136 - accuracy: 0.6938\n",
      "Epoch 15: val_loss did not improve from 0.50191\n",
      "99/99 [==============================] - 90s 910ms/step - loss: 1.1136 - accuracy: 0.6938 - val_loss: 0.5438 - val_accuracy: 0.7447\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0763 - accuracy: 0.7247\n",
      "Epoch 16: val_loss improved from 0.50191 to 0.49755, saving model to mod_.h5\n",
      "99/99 [==============================] - 81s 820ms/step - loss: 1.0763 - accuracy: 0.7247 - val_loss: 0.4975 - val_accuracy: 0.7660\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0138 - accuracy: 0.7443\n",
      "Epoch 17: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 94s 941ms/step - loss: 1.0138 - accuracy: 0.7443 - val_loss: 0.5617 - val_accuracy: 0.7660\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9883 - accuracy: 0.7557\n",
      "Epoch 18: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 80s 811ms/step - loss: 0.9883 - accuracy: 0.7557 - val_loss: 0.5318 - val_accuracy: 0.7447\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9456 - accuracy: 0.7588\n",
      "Epoch 19: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 82s 827ms/step - loss: 0.9456 - accuracy: 0.7588 - val_loss: 0.5350 - val_accuracy: 0.7872\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9225 - accuracy: 0.7803\n",
      "Epoch 20: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 86s 874ms/step - loss: 0.9225 - accuracy: 0.7803 - val_loss: 0.5681 - val_accuracy: 0.8085\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8847 - accuracy: 0.7923\n",
      "Epoch 21: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 86s 866ms/step - loss: 0.8847 - accuracy: 0.7923 - val_loss: 0.6078 - val_accuracy: 0.8085\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8537 - accuracy: 0.8074\n",
      "Epoch 22: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 79s 795ms/step - loss: 0.8537 - accuracy: 0.8074 - val_loss: 0.6818 - val_accuracy: 0.7872\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9031 - accuracy: 0.7860\n",
      "Epoch 23: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 92s 931ms/step - loss: 0.9031 - accuracy: 0.7860 - val_loss: 0.5783 - val_accuracy: 0.8085\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8366 - accuracy: 0.8138\n",
      "Epoch 24: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 91s 924ms/step - loss: 0.8366 - accuracy: 0.8138 - val_loss: 0.6725 - val_accuracy: 0.7872\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8007 - accuracy: 0.8163\n",
      "Epoch 25: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 95s 962ms/step - loss: 0.8007 - accuracy: 0.8163 - val_loss: 0.7885 - val_accuracy: 0.7660\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7335 - accuracy: 0.8428\n",
      "Epoch 26: val_loss did not improve from 0.49755\n",
      "99/99 [==============================] - 84s 852ms/step - loss: 0.7335 - accuracy: 0.8428 - val_loss: 0.7773 - val_accuracy: 0.7872\n",
      "Test loss:  0.4280441701412201\n",
      "Test accuracy:  0.8125\n",
      "2/2 [==============================] - 1s 118ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       1.00      0.94      0.97        31\n",
      "     Class 1       0.53      1.00      0.69         9\n",
      "     Class 2       0.50      0.12      0.20         8\n",
      "\n",
      "    accuracy                           0.81        48\n",
      "   macro avg       0.68      0.69      0.62        48\n",
      "weighted avg       0.83      0.81      0.79        48\n",
      "\n",
      "2/2 [==============================] - 0s 120ms/step\n",
      "[[29  1  1]\n",
      " [ 0  9  0]\n",
      " [ 0  7  1]]\n",
      "run number 2\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1240 - accuracy: 0.3359\n",
      "Epoch 1: val_loss improved from inf to 1.40920, saving model to mod_.h5\n",
      "99/99 [==============================] - 100s 948ms/step - loss: 2.1240 - accuracy: 0.3359 - val_loss: 1.4092 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0333 - accuracy: 0.3378\n",
      "Epoch 2: val_loss improved from 1.40920 to 1.40567, saving model to mod_.h5\n",
      "99/99 [==============================] - 86s 868ms/step - loss: 2.0333 - accuracy: 0.3378 - val_loss: 1.4057 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0004 - accuracy: 0.3409\n",
      "Epoch 3: val_loss improved from 1.40567 to 1.39401, saving model to mod_.h5\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 2.0004 - accuracy: 0.3409 - val_loss: 1.3940 - val_accuracy: 0.1489\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9721 - accuracy: 0.3441\n",
      "Epoch 4: val_loss improved from 1.39401 to 1.29769, saving model to mod_.h5\n",
      "99/99 [==============================] - 79s 793ms/step - loss: 1.9721 - accuracy: 0.3441 - val_loss: 1.2977 - val_accuracy: 0.1915\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8819 - accuracy: 0.3895\n",
      "Epoch 5: val_loss improved from 1.29769 to 0.97495, saving model to mod_.h5\n",
      "99/99 [==============================] - 71s 719ms/step - loss: 1.8819 - accuracy: 0.3895 - val_loss: 0.9750 - val_accuracy: 0.4894\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7171 - accuracy: 0.4836\n",
      "Epoch 6: val_loss improved from 0.97495 to 0.88736, saving model to mod_.h5\n",
      "99/99 [==============================] - 75s 756ms/step - loss: 1.7171 - accuracy: 0.4836 - val_loss: 0.8874 - val_accuracy: 0.6383\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6065 - accuracy: 0.5568\n",
      "Epoch 7: val_loss improved from 0.88736 to 0.64965, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 763ms/step - loss: 1.6065 - accuracy: 0.5568 - val_loss: 0.6496 - val_accuracy: 0.7234\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5654 - accuracy: 0.5795\n",
      "Epoch 8: val_loss did not improve from 0.64965\n",
      "99/99 [==============================] - 74s 747ms/step - loss: 1.5654 - accuracy: 0.5795 - val_loss: 1.5644 - val_accuracy: 0.2979\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5409 - accuracy: 0.5783\n",
      "Epoch 9: val_loss did not improve from 0.64965\n",
      "99/99 [==============================] - 78s 786ms/step - loss: 1.5409 - accuracy: 0.5783 - val_loss: 1.2744 - val_accuracy: 0.4043\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5069 - accuracy: 0.5966\n",
      "Epoch 10: val_loss improved from 0.64965 to 0.61615, saving model to mod_.h5\n",
      "99/99 [==============================] - 74s 737ms/step - loss: 1.5069 - accuracy: 0.5966 - val_loss: 0.6162 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4309 - accuracy: 0.6288\n",
      "Epoch 11: val_loss improved from 0.61615 to 0.54767, saving model to mod_.h5\n",
      "99/99 [==============================] - 86s 866ms/step - loss: 1.4309 - accuracy: 0.6288 - val_loss: 0.5477 - val_accuracy: 0.7447\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3716 - accuracy: 0.6294\n",
      "Epoch 12: val_loss did not improve from 0.54767\n",
      "99/99 [==============================] - 85s 865ms/step - loss: 1.3716 - accuracy: 0.6294 - val_loss: 0.5536 - val_accuracy: 0.7234\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2634 - accuracy: 0.6705\n",
      "Epoch 13: val_loss improved from 0.54767 to 0.47186, saving model to mod_.h5\n",
      "99/99 [==============================] - 81s 816ms/step - loss: 1.2634 - accuracy: 0.6705 - val_loss: 0.4719 - val_accuracy: 0.8085\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2743 - accuracy: 0.6641\n",
      "Epoch 14: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 63s 637ms/step - loss: 1.2743 - accuracy: 0.6641 - val_loss: 0.5239 - val_accuracy: 0.7447\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2268 - accuracy: 0.6686\n",
      "Epoch 15: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 46s 468ms/step - loss: 1.2268 - accuracy: 0.6686 - val_loss: 0.6457 - val_accuracy: 0.6383\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1852 - accuracy: 0.6888\n",
      "Epoch 16: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 63s 640ms/step - loss: 1.1852 - accuracy: 0.6888 - val_loss: 0.5208 - val_accuracy: 0.7234\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1526 - accuracy: 0.6982\n",
      "Epoch 17: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 90s 909ms/step - loss: 1.1526 - accuracy: 0.6982 - val_loss: 0.7460 - val_accuracy: 0.6596\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1378 - accuracy: 0.6894\n",
      "Epoch 18: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 79s 798ms/step - loss: 1.1378 - accuracy: 0.6894 - val_loss: 0.6704 - val_accuracy: 0.7021\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1165 - accuracy: 0.7064\n",
      "Epoch 19: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 80s 806ms/step - loss: 1.1165 - accuracy: 0.7064 - val_loss: 0.6025 - val_accuracy: 0.7234\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0458 - accuracy: 0.7216\n",
      "Epoch 20: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 75s 756ms/step - loss: 1.0458 - accuracy: 0.7216 - val_loss: 0.5792 - val_accuracy: 0.7447\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0236 - accuracy: 0.7361\n",
      "Epoch 21: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 79s 795ms/step - loss: 1.0236 - accuracy: 0.7361 - val_loss: 0.8235 - val_accuracy: 0.6809\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0616 - accuracy: 0.7134\n",
      "Epoch 22: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 75s 755ms/step - loss: 1.0616 - accuracy: 0.7134 - val_loss: 0.5732 - val_accuracy: 0.7660\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9752 - accuracy: 0.7475\n",
      "Epoch 23: val_loss did not improve from 0.47186\n",
      "99/99 [==============================] - 78s 791ms/step - loss: 0.9752 - accuracy: 0.7475 - val_loss: 0.5966 - val_accuracy: 0.6809\n",
      "Test loss:  0.5003289580345154\n",
      "Test accuracy:  0.7916666865348816\n",
      "2/2 [==============================] - 1s 180ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.97      0.94      0.95        31\n",
      "     Class 1       0.56      0.56      0.56         9\n",
      "     Class 2       0.44      0.50      0.47         8\n",
      "\n",
      "    accuracy                           0.79        48\n",
      "   macro avg       0.66      0.66      0.66        48\n",
      "weighted avg       0.80      0.79      0.80        48\n",
      "\n",
      "2/2 [==============================] - 1s 285ms/step\n",
      "[[29  1  1]\n",
      " [ 0  5  4]\n",
      " [ 1  3  4]]\n",
      "run number 3\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0928 - accuracy: 0.3239\n",
      "Epoch 1: val_loss improved from inf to 1.19310, saving model to mod_.h5\n",
      "99/99 [==============================] - 108s 1000ms/step - loss: 2.0928 - accuracy: 0.3239 - val_loss: 1.1931 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9314 - accuracy: 0.3712\n",
      "Epoch 2: val_loss did not improve from 1.19310\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 1.9314 - accuracy: 0.3712 - val_loss: 1.3724 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7364 - accuracy: 0.4785\n",
      "Epoch 3: val_loss improved from 1.19310 to 0.96544, saving model to mod_.h5\n",
      "99/99 [==============================] - 91s 920ms/step - loss: 1.7364 - accuracy: 0.4785 - val_loss: 0.9654 - val_accuracy: 0.5106\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6542 - accuracy: 0.5322\n",
      "Epoch 4: val_loss did not improve from 0.96544\n",
      "99/99 [==============================] - 75s 755ms/step - loss: 1.6542 - accuracy: 0.5322 - val_loss: 1.1513 - val_accuracy: 0.4043\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6291 - accuracy: 0.5587\n",
      "Epoch 5: val_loss improved from 0.96544 to 0.66707, saving model to mod_.h5\n",
      "99/99 [==============================] - 8808s 90s/step - loss: 1.6291 - accuracy: 0.5587 - val_loss: 0.6671 - val_accuracy: 0.7021\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5465 - accuracy: 0.5878\n",
      "Epoch 6: val_loss did not improve from 0.66707\n",
      "99/99 [==============================] - 86s 843ms/step - loss: 1.5465 - accuracy: 0.5878 - val_loss: 0.9876 - val_accuracy: 0.5319\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4829 - accuracy: 0.5972\n",
      "Epoch 7: val_loss did not improve from 0.66707\n",
      "99/99 [==============================] - 73s 741ms/step - loss: 1.4829 - accuracy: 0.5972 - val_loss: 0.8476 - val_accuracy: 0.6170\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3836 - accuracy: 0.6484\n",
      "Epoch 8: val_loss did not improve from 0.66707\n",
      "99/99 [==============================] - 76s 767ms/step - loss: 1.3836 - accuracy: 0.6484 - val_loss: 0.8355 - val_accuracy: 0.6809\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3949 - accuracy: 0.6143\n",
      "Epoch 9: val_loss did not improve from 0.66707\n",
      "99/99 [==============================] - 100s 1s/step - loss: 1.3949 - accuracy: 0.6143 - val_loss: 0.9136 - val_accuracy: 0.5957\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2806 - accuracy: 0.6572\n",
      "Epoch 10: val_loss did not improve from 0.66707\n",
      "99/99 [==============================] - 94s 954ms/step - loss: 1.2806 - accuracy: 0.6572 - val_loss: 0.7020 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2854 - accuracy: 0.6477\n",
      "Epoch 11: val_loss improved from 0.66707 to 0.58955, saving model to mod_.h5\n",
      "99/99 [==============================] - 75s 760ms/step - loss: 1.2854 - accuracy: 0.6477 - val_loss: 0.5896 - val_accuracy: 0.7021\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2603 - accuracy: 0.6667\n",
      "Epoch 12: val_loss did not improve from 0.58955\n",
      "99/99 [==============================] - 78s 784ms/step - loss: 1.2603 - accuracy: 0.6667 - val_loss: 0.6502 - val_accuracy: 0.7021\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1720 - accuracy: 0.6856\n",
      "Epoch 13: val_loss improved from 0.58955 to 0.58005, saving model to mod_.h5\n",
      "99/99 [==============================] - 77s 782ms/step - loss: 1.1720 - accuracy: 0.6856 - val_loss: 0.5801 - val_accuracy: 0.7447\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1238 - accuracy: 0.7014\n",
      "Epoch 14: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 75s 762ms/step - loss: 1.1238 - accuracy: 0.7014 - val_loss: 0.7144 - val_accuracy: 0.7021\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1289 - accuracy: 0.6989\n",
      "Epoch 15: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 75s 761ms/step - loss: 1.1289 - accuracy: 0.6989 - val_loss: 0.6490 - val_accuracy: 0.7234\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0271 - accuracy: 0.7532\n",
      "Epoch 16: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 75s 763ms/step - loss: 1.0271 - accuracy: 0.7532 - val_loss: 0.6465 - val_accuracy: 0.8085\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9785 - accuracy: 0.7658\n",
      "Epoch 17: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 75s 760ms/step - loss: 0.9785 - accuracy: 0.7658 - val_loss: 0.6895 - val_accuracy: 0.7872\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9863 - accuracy: 0.7677\n",
      "Epoch 18: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 0.9863 - accuracy: 0.7677 - val_loss: 0.8542 - val_accuracy: 0.6809\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9529 - accuracy: 0.7689\n",
      "Epoch 19: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 80s 810ms/step - loss: 0.9529 - accuracy: 0.7689 - val_loss: 0.6950 - val_accuracy: 0.7872\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8617 - accuracy: 0.8093\n",
      "Epoch 20: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 79s 794ms/step - loss: 0.8617 - accuracy: 0.8093 - val_loss: 0.7250 - val_accuracy: 0.7447\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8429 - accuracy: 0.8062\n",
      "Epoch 21: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 82s 826ms/step - loss: 0.8429 - accuracy: 0.8062 - val_loss: 0.7437 - val_accuracy: 0.7660\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7957 - accuracy: 0.8295\n",
      "Epoch 22: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 96s 969ms/step - loss: 0.7957 - accuracy: 0.8295 - val_loss: 0.9337 - val_accuracy: 0.7447\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7267 - accuracy: 0.8504\n",
      "Epoch 23: val_loss did not improve from 0.58005\n",
      "99/99 [==============================] - 94s 949ms/step - loss: 0.7267 - accuracy: 0.8504 - val_loss: 0.8326 - val_accuracy: 0.8085\n",
      "Test loss:  0.5335456132888794\n",
      "Test accuracy:  0.7916666865348816\n",
      "2/2 [==============================] - 1s 275ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.97      0.90      0.93        31\n",
      "     Class 1       0.58      0.78      0.67         9\n",
      "     Class 2       0.43      0.38      0.40         8\n",
      "\n",
      "    accuracy                           0.79        48\n",
      "   macro avg       0.66      0.69      0.67        48\n",
      "weighted avg       0.80      0.79      0.79        48\n",
      "\n",
      "2/2 [==============================] - 1s 255ms/step\n",
      "[[28  1  2]\n",
      " [ 0  7  2]\n",
      " [ 1  4  3]]\n",
      "run number 4\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.2638 - accuracy: 0.3340\n",
      "Epoch 1: val_loss improved from inf to 1.44009, saving model to mod_.h5\n",
      "99/99 [==============================] - 112s 1s/step - loss: 2.2638 - accuracy: 0.3340 - val_loss: 1.4401 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9769 - accuracy: 0.3598\n",
      "Epoch 2: val_loss did not improve from 1.44009\n",
      "99/99 [==============================] - 81s 817ms/step - loss: 1.9769 - accuracy: 0.3598 - val_loss: 1.5304 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9644 - accuracy: 0.3643\n",
      "Epoch 3: val_loss improved from 1.44009 to 1.32066, saving model to mod_.h5\n",
      "99/99 [==============================] - 81s 823ms/step - loss: 1.9644 - accuracy: 0.3643 - val_loss: 1.3207 - val_accuracy: 0.1489\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8672 - accuracy: 0.4003\n",
      "Epoch 4: val_loss improved from 1.32066 to 1.00698, saving model to mod_.h5\n",
      "99/99 [==============================] - 74s 746ms/step - loss: 1.8672 - accuracy: 0.4003 - val_loss: 1.0070 - val_accuracy: 0.4468\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7440 - accuracy: 0.4653\n",
      "Epoch 5: val_loss improved from 1.00698 to 0.85760, saving model to mod_.h5\n",
      "99/99 [==============================] - 96s 971ms/step - loss: 1.7440 - accuracy: 0.4653 - val_loss: 0.8576 - val_accuracy: 0.5532\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6585 - accuracy: 0.5271\n",
      "Epoch 6: val_loss did not improve from 0.85760\n",
      "99/99 [==============================] - 106s 1s/step - loss: 1.6585 - accuracy: 0.5271 - val_loss: 1.1887 - val_accuracy: 0.5106\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6406 - accuracy: 0.5309\n",
      "Epoch 7: val_loss did not improve from 0.85760\n",
      "99/99 [==============================] - 89s 898ms/step - loss: 1.6406 - accuracy: 0.5309 - val_loss: 1.0742 - val_accuracy: 0.4468\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6743 - accuracy: 0.5221\n",
      "Epoch 8: val_loss did not improve from 0.85760\n",
      "99/99 [==============================] - 90s 905ms/step - loss: 1.6743 - accuracy: 0.5221 - val_loss: 1.0137 - val_accuracy: 0.4681\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5603 - accuracy: 0.5669\n",
      "Epoch 9: val_loss improved from 0.85760 to 0.64071, saving model to mod_.h5\n",
      "99/99 [==============================] - 74s 744ms/step - loss: 1.5603 - accuracy: 0.5669 - val_loss: 0.6407 - val_accuracy: 0.7234\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4994 - accuracy: 0.5846\n",
      "Epoch 10: val_loss did not improve from 0.64071\n",
      "99/99 [==============================] - 82s 826ms/step - loss: 1.4994 - accuracy: 0.5846 - val_loss: 0.7257 - val_accuracy: 0.6596\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4654 - accuracy: 0.5915\n",
      "Epoch 11: val_loss did not improve from 0.64071\n",
      "99/99 [==============================] - 74s 746ms/step - loss: 1.4654 - accuracy: 0.5915 - val_loss: 1.1218 - val_accuracy: 0.3830\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4044 - accuracy: 0.6174\n",
      "Epoch 12: val_loss did not improve from 0.64071\n",
      "99/99 [==============================] - 76s 769ms/step - loss: 1.4044 - accuracy: 0.6174 - val_loss: 0.7298 - val_accuracy: 0.6809\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3266 - accuracy: 0.6351\n",
      "Epoch 13: val_loss improved from 0.64071 to 0.61564, saving model to mod_.h5\n",
      "99/99 [==============================] - 74s 748ms/step - loss: 1.3266 - accuracy: 0.6351 - val_loss: 0.6156 - val_accuracy: 0.7447\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3422 - accuracy: 0.6307  \n",
      "Epoch 14: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 7100s 72s/step - loss: 1.3422 - accuracy: 0.6307 - val_loss: 0.8705 - val_accuracy: 0.5319\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2944 - accuracy: 0.6427\n",
      "Epoch 15: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 43s 430ms/step - loss: 1.2944 - accuracy: 0.6427 - val_loss: 0.6184 - val_accuracy: 0.7447\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2647 - accuracy: 0.6528\n",
      "Epoch 16: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 64s 649ms/step - loss: 1.2647 - accuracy: 0.6528 - val_loss: 0.6176 - val_accuracy: 0.7660\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2174 - accuracy: 0.6755\n",
      "Epoch 17: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 77s 782ms/step - loss: 1.2174 - accuracy: 0.6755 - val_loss: 0.6508 - val_accuracy: 0.7660\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1533 - accuracy: 0.6806\n",
      "Epoch 18: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 79s 796ms/step - loss: 1.1533 - accuracy: 0.6806 - val_loss: 0.6326 - val_accuracy: 0.7660\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1264 - accuracy: 0.7128\n",
      "Epoch 19: val_loss did not improve from 0.61564\n",
      "99/99 [==============================] - 79s 800ms/step - loss: 1.1264 - accuracy: 0.7128 - val_loss: 0.6286 - val_accuracy: 0.7872\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1303 - accuracy: 0.6944\n",
      "Epoch 20: val_loss improved from 0.61564 to 0.61258, saving model to mod_.h5\n",
      "99/99 [==============================] - 80s 802ms/step - loss: 1.1303 - accuracy: 0.6944 - val_loss: 0.6126 - val_accuracy: 0.7660\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0686 - accuracy: 0.7197\n",
      "Epoch 21: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 74s 746ms/step - loss: 1.0686 - accuracy: 0.7197 - val_loss: 0.6446 - val_accuracy: 0.7660\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0442 - accuracy: 0.7393\n",
      "Epoch 22: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 75s 755ms/step - loss: 1.0442 - accuracy: 0.7393 - val_loss: 0.8476 - val_accuracy: 0.7021\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0042 - accuracy: 0.7569\n",
      "Epoch 23: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 75s 754ms/step - loss: 1.0042 - accuracy: 0.7569 - val_loss: 0.6397 - val_accuracy: 0.8085\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9503 - accuracy: 0.7595\n",
      "Epoch 24: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 73s 736ms/step - loss: 0.9503 - accuracy: 0.7595 - val_loss: 0.6239 - val_accuracy: 0.8085\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9251 - accuracy: 0.7715\n",
      "Epoch 25: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 72s 727ms/step - loss: 0.9251 - accuracy: 0.7715 - val_loss: 0.9564 - val_accuracy: 0.6170\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8895 - accuracy: 0.7904\n",
      "Epoch 26: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 70s 708ms/step - loss: 0.8895 - accuracy: 0.7904 - val_loss: 0.6835 - val_accuracy: 0.8085\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8682 - accuracy: 0.8087\n",
      "Epoch 27: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 73s 733ms/step - loss: 0.8682 - accuracy: 0.8087 - val_loss: 0.6866 - val_accuracy: 0.8085\n",
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8846 - accuracy: 0.7967\n",
      "Epoch 28: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 73s 735ms/step - loss: 0.8846 - accuracy: 0.7967 - val_loss: 0.6467 - val_accuracy: 0.7872\n",
      "Epoch 29/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7821 - accuracy: 0.8264\n",
      "Epoch 29: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 73s 733ms/step - loss: 0.7821 - accuracy: 0.8264 - val_loss: 0.8013 - val_accuracy: 0.7660\n",
      "Epoch 30/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7771 - accuracy: 0.8447\n",
      "Epoch 30: val_loss did not improve from 0.61258\n",
      "99/99 [==============================] - 72s 732ms/step - loss: 0.7771 - accuracy: 0.8447 - val_loss: 0.7212 - val_accuracy: 0.8298\n",
      "Test loss:  0.5468400120735168\n",
      "Test accuracy:  0.8333333134651184\n",
      "2/2 [==============================] - 1s 195ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.93      0.90      0.92        31\n",
      "     Class 1       0.64      1.00      0.78         9\n",
      "     Class 2       0.75      0.38      0.50         8\n",
      "\n",
      "    accuracy                           0.83        48\n",
      "   macro avg       0.78      0.76      0.73        48\n",
      "weighted avg       0.85      0.83      0.82        48\n",
      "\n",
      "2/2 [==============================] - 1s 172ms/step\n",
      "[[28  2  1]\n",
      " [ 0  9  0]\n",
      " [ 2  3  3]]\n",
      "run number 5\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1052 - accuracy: 0.3308\n",
      "Epoch 1: val_loss improved from inf to 1.30828, saving model to mod_.h5\n",
      "99/99 [==============================] - 99s 848ms/step - loss: 2.1052 - accuracy: 0.3308 - val_loss: 1.3083 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9593 - accuracy: 0.3592\n",
      "Epoch 2: val_loss improved from 1.30828 to 1.02048, saving model to mod_.h5\n",
      "99/99 [==============================] - 82s 828ms/step - loss: 1.9593 - accuracy: 0.3592 - val_loss: 1.0205 - val_accuracy: 0.3830\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7796 - accuracy: 0.4792\n",
      "Epoch 3: val_loss improved from 1.02048 to 0.92829, saving model to mod_.h5\n",
      "99/99 [==============================] - 82s 825ms/step - loss: 1.7796 - accuracy: 0.4792 - val_loss: 0.9283 - val_accuracy: 0.5319\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6599 - accuracy: 0.5347\n",
      "Epoch 4: val_loss did not improve from 0.92829\n",
      "99/99 [==============================] - 76s 765ms/step - loss: 1.6599 - accuracy: 0.5347 - val_loss: 0.9731 - val_accuracy: 0.5532\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5543 - accuracy: 0.5694\n",
      "Epoch 5: val_loss improved from 0.92829 to 0.75886, saving model to mod_.h5\n",
      "99/99 [==============================] - 83s 835ms/step - loss: 1.5543 - accuracy: 0.5694 - val_loss: 0.7589 - val_accuracy: 0.6596\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4977 - accuracy: 0.5896\n",
      "Epoch 6: val_loss improved from 0.75886 to 0.64653, saving model to mod_.h5\n",
      "99/99 [==============================] - 73s 736ms/step - loss: 1.4977 - accuracy: 0.5896 - val_loss: 0.6465 - val_accuracy: 0.7447\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5102 - accuracy: 0.5903\n",
      "Epoch 7: val_loss improved from 0.64653 to 0.63383, saving model to mod_.h5\n",
      "99/99 [==============================] - 51s 517ms/step - loss: 1.5102 - accuracy: 0.5903 - val_loss: 0.6338 - val_accuracy: 0.7447\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4769 - accuracy: 0.6010\n",
      "Epoch 8: val_loss did not improve from 0.63383\n",
      "99/99 [==============================] - 48s 482ms/step - loss: 1.4769 - accuracy: 0.6010 - val_loss: 0.7117 - val_accuracy: 0.6809\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3297 - accuracy: 0.6616\n",
      "Epoch 9: val_loss improved from 0.63383 to 0.57606, saving model to mod_.h5\n",
      "99/99 [==============================] - 51s 513ms/step - loss: 1.3297 - accuracy: 0.6616 - val_loss: 0.5761 - val_accuracy: 0.7447\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3484 - accuracy: 0.6439\n",
      "Epoch 10: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 47s 480ms/step - loss: 1.3484 - accuracy: 0.6439 - val_loss: 0.6699 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2610 - accuracy: 0.6604\n",
      "Epoch 11: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 51s 511ms/step - loss: 1.2610 - accuracy: 0.6604 - val_loss: 0.8103 - val_accuracy: 0.6170\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2390 - accuracy: 0.6761\n",
      "Epoch 12: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 56s 569ms/step - loss: 1.2390 - accuracy: 0.6761 - val_loss: 0.5912 - val_accuracy: 0.7021\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1919 - accuracy: 0.6686\n",
      "Epoch 13: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 80s 801ms/step - loss: 1.1919 - accuracy: 0.6686 - val_loss: 0.6433 - val_accuracy: 0.7447\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1209 - accuracy: 0.7140\n",
      "Epoch 14: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 78s 782ms/step - loss: 1.1209 - accuracy: 0.7140 - val_loss: 0.7466 - val_accuracy: 0.7447\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1107 - accuracy: 0.7077\n",
      "Epoch 15: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 76s 772ms/step - loss: 1.1107 - accuracy: 0.7077 - val_loss: 0.6678 - val_accuracy: 0.7234\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0744 - accuracy: 0.7235\n",
      "Epoch 16: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 77s 776ms/step - loss: 1.0744 - accuracy: 0.7235 - val_loss: 0.5872 - val_accuracy: 0.7447\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0606 - accuracy: 0.7317\n",
      "Epoch 17: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 78s 789ms/step - loss: 1.0606 - accuracy: 0.7317 - val_loss: 0.7198 - val_accuracy: 0.7660\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9858 - accuracy: 0.7563\n",
      "Epoch 18: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 74s 750ms/step - loss: 0.9858 - accuracy: 0.7563 - val_loss: 0.7034 - val_accuracy: 0.7234\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9784 - accuracy: 0.7582\n",
      "Epoch 19: val_loss did not improve from 0.57606\n",
      "99/99 [==============================] - 72s 728ms/step - loss: 0.9784 - accuracy: 0.7582 - val_loss: 0.7121 - val_accuracy: 0.7872\n",
      "Test loss:  0.5070050358772278\n",
      "Test accuracy:  0.6875\n",
      "2/2 [==============================] - 1s 303ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       1.00      0.84      0.91        31\n",
      "     Class 1       0.00      0.00      0.00         9\n",
      "     Class 2       0.37      0.88      0.52         8\n",
      "\n",
      "    accuracy                           0.69        48\n",
      "   macro avg       0.46      0.57      0.48        48\n",
      "weighted avg       0.71      0.69      0.68        48\n",
      "\n",
      "2/2 [==============================] - 1s 290ms/step\n",
      "[[26  2  3]\n",
      " [ 0  0  9]\n",
      " [ 0  1  7]]\n",
      "run number 6\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.2226 - accuracy: 0.3258\n",
      "Epoch 1: val_loss improved from inf to 1.31983, saving model to mod_.h5\n",
      "99/99 [==============================] - 89s 856ms/step - loss: 2.2226 - accuracy: 0.3258 - val_loss: 1.3198 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9580 - accuracy: 0.3687\n",
      "Epoch 2: val_loss improved from 1.31983 to 1.31297, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 770ms/step - loss: 1.9580 - accuracy: 0.3687 - val_loss: 1.3130 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8590 - accuracy: 0.3889\n",
      "Epoch 3: val_loss improved from 1.31297 to 1.19060, saving model to mod_.h5\n",
      "99/99 [==============================] - 78s 790ms/step - loss: 1.8590 - accuracy: 0.3889 - val_loss: 1.1906 - val_accuracy: 0.1489\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8318 - accuracy: 0.4198\n",
      "Epoch 4: val_loss did not improve from 1.19060\n",
      "99/99 [==============================] - 78s 789ms/step - loss: 1.8318 - accuracy: 0.4198 - val_loss: 1.5625 - val_accuracy: 0.1489\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7037 - accuracy: 0.4949\n",
      "Epoch 5: val_loss improved from 1.19060 to 0.94136, saving model to mod_.h5\n",
      "99/99 [==============================] - 82s 829ms/step - loss: 1.7037 - accuracy: 0.4949 - val_loss: 0.9414 - val_accuracy: 0.5319\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6456 - accuracy: 0.5164\n",
      "Epoch 6: val_loss improved from 0.94136 to 0.76898, saving model to mod_.h5\n",
      "99/99 [==============================] - 77s 781ms/step - loss: 1.6456 - accuracy: 0.5164 - val_loss: 0.7690 - val_accuracy: 0.6170\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6103 - accuracy: 0.5568\n",
      "Epoch 7: val_loss did not improve from 0.76898\n",
      "99/99 [==============================] - 75s 754ms/step - loss: 1.6103 - accuracy: 0.5568 - val_loss: 1.1309 - val_accuracy: 0.3617\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6163 - accuracy: 0.5467\n",
      "Epoch 8: val_loss improved from 0.76898 to 0.63820, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 767ms/step - loss: 1.6163 - accuracy: 0.5467 - val_loss: 0.6382 - val_accuracy: 0.7447\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5881 - accuracy: 0.5549\n",
      "Epoch 9: val_loss did not improve from 0.63820\n",
      "99/99 [==============================] - 72s 725ms/step - loss: 1.5881 - accuracy: 0.5549 - val_loss: 1.1657 - val_accuracy: 0.5106\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5027 - accuracy: 0.5840\n",
      "Epoch 10: val_loss did not improve from 0.63820\n",
      "99/99 [==============================] - 71s 715ms/step - loss: 1.5027 - accuracy: 0.5840 - val_loss: 0.6576 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4549 - accuracy: 0.6029\n",
      "Epoch 11: val_loss did not improve from 0.63820\n",
      "99/99 [==============================] - 75s 760ms/step - loss: 1.4549 - accuracy: 0.6029 - val_loss: 1.5772 - val_accuracy: 0.2766\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4249 - accuracy: 0.6187\n",
      "Epoch 12: val_loss improved from 0.63820 to 0.55910, saving model to mod_.h5\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 1.4249 - accuracy: 0.6187 - val_loss: 0.5591 - val_accuracy: 0.8085\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3517 - accuracy: 0.6357\n",
      "Epoch 13: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 98s 996ms/step - loss: 1.3517 - accuracy: 0.6357 - val_loss: 0.6009 - val_accuracy: 0.7234\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3005 - accuracy: 0.6383\n",
      "Epoch 14: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 136s 1s/step - loss: 1.3005 - accuracy: 0.6383 - val_loss: 0.9321 - val_accuracy: 0.5532\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2498 - accuracy: 0.6622\n",
      "Epoch 15: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 90s 907ms/step - loss: 1.2498 - accuracy: 0.6622 - val_loss: 0.6138 - val_accuracy: 0.7447\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2099 - accuracy: 0.6730\n",
      "Epoch 16: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 105s 1s/step - loss: 1.2099 - accuracy: 0.6730 - val_loss: 0.6203 - val_accuracy: 0.7234\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2087 - accuracy: 0.6730\n",
      "Epoch 17: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 83s 842ms/step - loss: 1.2087 - accuracy: 0.6730 - val_loss: 0.6444 - val_accuracy: 0.6809\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1346 - accuracy: 0.6837\n",
      "Epoch 18: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 84s 844ms/step - loss: 1.1346 - accuracy: 0.6837 - val_loss: 0.5689 - val_accuracy: 0.7872\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1040 - accuracy: 0.7153\n",
      "Epoch 19: val_loss did not improve from 0.55910\n",
      "99/99 [==============================] - 96s 966ms/step - loss: 1.1040 - accuracy: 0.7153 - val_loss: 0.6168 - val_accuracy: 0.7660\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0872 - accuracy: 0.7077\n",
      "Epoch 20: val_loss improved from 0.55910 to 0.53125, saving model to mod_.h5\n",
      "99/99 [==============================] - 88s 887ms/step - loss: 1.0872 - accuracy: 0.7077 - val_loss: 0.5312 - val_accuracy: 0.8298\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0405 - accuracy: 0.7418\n",
      "Epoch 21: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 119s 1s/step - loss: 1.0405 - accuracy: 0.7418 - val_loss: 0.7236 - val_accuracy: 0.6809\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0772 - accuracy: 0.7134\n",
      "Epoch 22: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 120s 1s/step - loss: 1.0772 - accuracy: 0.7134 - val_loss: 0.5695 - val_accuracy: 0.8085\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9887 - accuracy: 0.7481\n",
      "Epoch 23: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 88s 888ms/step - loss: 0.9887 - accuracy: 0.7481 - val_loss: 0.5788 - val_accuracy: 0.7872\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9523 - accuracy: 0.7639\n",
      "Epoch 24: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 98s 988ms/step - loss: 0.9523 - accuracy: 0.7639 - val_loss: 0.6345 - val_accuracy: 0.7447\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9929 - accuracy: 0.7475\n",
      "Epoch 25: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 85s 858ms/step - loss: 0.9929 - accuracy: 0.7475 - val_loss: 0.6362 - val_accuracy: 0.7872\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9909 - accuracy: 0.7569\n",
      "Epoch 26: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 98s 993ms/step - loss: 0.9909 - accuracy: 0.7569 - val_loss: 0.5802 - val_accuracy: 0.7660\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8583 - accuracy: 0.7942\n",
      "Epoch 27: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 86s 871ms/step - loss: 0.8583 - accuracy: 0.7942 - val_loss: 0.5923 - val_accuracy: 0.8298\n",
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8470 - accuracy: 0.8131\n",
      "Epoch 28: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 88s 887ms/step - loss: 0.8470 - accuracy: 0.8131 - val_loss: 0.7147 - val_accuracy: 0.7234\n",
      "Epoch 29/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8274 - accuracy: 0.8074\n",
      "Epoch 29: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 88s 884ms/step - loss: 0.8274 - accuracy: 0.8074 - val_loss: 0.6788 - val_accuracy: 0.8298\n",
      "Epoch 30/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7976 - accuracy: 0.8176\n",
      "Epoch 30: val_loss did not improve from 0.53125\n",
      "99/99 [==============================] - 90s 908ms/step - loss: 0.7976 - accuracy: 0.8176 - val_loss: 0.6538 - val_accuracy: 0.7872\n",
      "Test loss:  0.493171364068985\n",
      "Test accuracy:  0.8125\n",
      "2/2 [==============================] - 1s 129ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.94      0.97      0.95        31\n",
      "     Class 1       0.62      0.56      0.59         9\n",
      "     Class 2       0.50      0.50      0.50         8\n",
      "\n",
      "    accuracy                           0.81        48\n",
      "   macro avg       0.69      0.67      0.68        48\n",
      "weighted avg       0.81      0.81      0.81        48\n",
      "\n",
      "2/2 [==============================] - 0s 128ms/step\n",
      "[[30  1  0]\n",
      " [ 0  5  4]\n",
      " [ 2  2  4]]\n",
      "run number 7\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0842 - accuracy: 0.3194\n",
      "Epoch 1: val_loss improved from inf to 1.33093, saving model to mod_.h5\n",
      "99/99 [==============================] - 95s 905ms/step - loss: 2.0842 - accuracy: 0.3194 - val_loss: 1.3309 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9225 - accuracy: 0.3788\n",
      "Epoch 2: val_loss improved from 1.33093 to 0.81030, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 766ms/step - loss: 1.9225 - accuracy: 0.3788 - val_loss: 0.8103 - val_accuracy: 0.7447\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7810 - accuracy: 0.4855\n",
      "Epoch 3: val_loss did not improve from 0.81030\n",
      "99/99 [==============================] - 86s 871ms/step - loss: 1.7810 - accuracy: 0.4855 - val_loss: 1.5488 - val_accuracy: 0.2340\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6443 - accuracy: 0.5436\n",
      "Epoch 4: val_loss improved from 0.81030 to 0.78686, saving model to mod_.h5\n",
      "99/99 [==============================] - 84s 844ms/step - loss: 1.6443 - accuracy: 0.5436 - val_loss: 0.7869 - val_accuracy: 0.6596\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5581 - accuracy: 0.5682\n",
      "Epoch 5: val_loss improved from 0.78686 to 0.60625, saving model to mod_.h5\n",
      "99/99 [==============================] - 100s 1s/step - loss: 1.5581 - accuracy: 0.5682 - val_loss: 0.6062 - val_accuracy: 0.7447\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5287 - accuracy: 0.5821\n",
      "Epoch 6: val_loss did not improve from 0.60625\n",
      "99/99 [==============================] - 88s 884ms/step - loss: 1.5287 - accuracy: 0.5821 - val_loss: 0.8840 - val_accuracy: 0.5532\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5000 - accuracy: 0.5840\n",
      "Epoch 7: val_loss did not improve from 0.60625\n",
      "99/99 [==============================] - 78s 791ms/step - loss: 1.5000 - accuracy: 0.5840 - val_loss: 0.7792 - val_accuracy: 0.6383\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4406 - accuracy: 0.6048\n",
      "Epoch 8: val_loss improved from 0.60625 to 0.57585, saving model to mod_.h5\n",
      "99/99 [==============================] - 80s 808ms/step - loss: 1.4406 - accuracy: 0.6048 - val_loss: 0.5759 - val_accuracy: 0.7660\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3701 - accuracy: 0.6288\n",
      "Epoch 9: val_loss did not improve from 0.57585\n",
      "99/99 [==============================] - 98s 994ms/step - loss: 1.3701 - accuracy: 0.6288 - val_loss: 0.8047 - val_accuracy: 0.6596\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2922 - accuracy: 0.6458\n",
      "Epoch 10: val_loss did not improve from 0.57585\n",
      "99/99 [==============================] - 84s 844ms/step - loss: 1.2922 - accuracy: 0.6458 - val_loss: 0.6787 - val_accuracy: 0.7021\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2971 - accuracy: 0.6370\n",
      "Epoch 11: val_loss did not improve from 0.57585\n",
      "99/99 [==============================] - 82s 824ms/step - loss: 1.2971 - accuracy: 0.6370 - val_loss: 0.7596 - val_accuracy: 0.6596\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2404 - accuracy: 0.6692\n",
      "Epoch 12: val_loss improved from 0.57585 to 0.51118, saving model to mod_.h5\n",
      "99/99 [==============================] - 95s 959ms/step - loss: 1.2404 - accuracy: 0.6692 - val_loss: 0.5112 - val_accuracy: 0.7872\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1771 - accuracy: 0.6888\n",
      "Epoch 13: val_loss did not improve from 0.51118\n",
      "99/99 [==============================] - 98s 985ms/step - loss: 1.1771 - accuracy: 0.6888 - val_loss: 0.5851 - val_accuracy: 0.7447\n",
      "Epoch 14/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99/99 [==============================] - ETA: 0s - loss: 1.1367 - accuracy: 0.6869\n",
      "Epoch 14: val_loss improved from 0.51118 to 0.49939, saving model to mod_.h5\n",
      "99/99 [==============================] - 94s 953ms/step - loss: 1.1367 - accuracy: 0.6869 - val_loss: 0.4994 - val_accuracy: 0.7660\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1026 - accuracy: 0.7146\n",
      "Epoch 15: val_loss improved from 0.49939 to 0.47579, saving model to mod_.h5\n",
      "99/99 [==============================] - 108s 1s/step - loss: 1.1026 - accuracy: 0.7146 - val_loss: 0.4758 - val_accuracy: 0.7872\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0850 - accuracy: 0.7134\n",
      "Epoch 16: val_loss did not improve from 0.47579\n",
      "99/99 [==============================] - 78s 788ms/step - loss: 1.0850 - accuracy: 0.7134 - val_loss: 0.5713 - val_accuracy: 0.7021\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0233 - accuracy: 0.7500\n",
      "Epoch 17: val_loss did not improve from 0.47579\n",
      "99/99 [==============================] - 79s 799ms/step - loss: 1.0233 - accuracy: 0.7500 - val_loss: 0.4919 - val_accuracy: 0.8085\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0285 - accuracy: 0.7412\n",
      "Epoch 18: val_loss improved from 0.47579 to 0.47202, saving model to mod_.h5\n",
      "99/99 [==============================] - 75s 757ms/step - loss: 1.0285 - accuracy: 0.7412 - val_loss: 0.4720 - val_accuracy: 0.8085\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9708 - accuracy: 0.7771\n",
      "Epoch 19: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 76s 764ms/step - loss: 0.9708 - accuracy: 0.7771 - val_loss: 0.5845 - val_accuracy: 0.6809\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9343 - accuracy: 0.7721\n",
      "Epoch 20: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 74s 741ms/step - loss: 0.9343 - accuracy: 0.7721 - val_loss: 0.5265 - val_accuracy: 0.7447\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8700 - accuracy: 0.8005\n",
      "Epoch 21: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 76s 766ms/step - loss: 0.8700 - accuracy: 0.8005 - val_loss: 0.5418 - val_accuracy: 0.7021\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8453 - accuracy: 0.8011\n",
      "Epoch 22: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 72s 724ms/step - loss: 0.8453 - accuracy: 0.8011 - val_loss: 0.5936 - val_accuracy: 0.8085\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7746 - accuracy: 0.8308\n",
      "Epoch 23: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 110s 1s/step - loss: 0.7746 - accuracy: 0.8308 - val_loss: 0.5530 - val_accuracy: 0.8085\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7479 - accuracy: 0.8497\n",
      "Epoch 24: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 83s 842ms/step - loss: 0.7479 - accuracy: 0.8497 - val_loss: 0.5181 - val_accuracy: 0.8511\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6995 - accuracy: 0.8497\n",
      "Epoch 25: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 85s 850ms/step - loss: 0.6995 - accuracy: 0.8497 - val_loss: 0.5315 - val_accuracy: 0.8511\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6491 - accuracy: 0.8687\n",
      "Epoch 26: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 82s 825ms/step - loss: 0.6491 - accuracy: 0.8687 - val_loss: 0.5889 - val_accuracy: 0.8511\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5965 - accuracy: 0.8933\n",
      "Epoch 27: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 93s 943ms/step - loss: 0.5965 - accuracy: 0.8933 - val_loss: 0.6625 - val_accuracy: 0.7660\n",
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6181 - accuracy: 0.8832\n",
      "Epoch 28: val_loss did not improve from 0.47202\n",
      "99/99 [==============================] - 81s 820ms/step - loss: 0.6181 - accuracy: 0.8832 - val_loss: 0.5976 - val_accuracy: 0.7872\n",
      "Test loss:  0.6184661388397217\n",
      "Test accuracy:  0.7083333134651184\n",
      "2/2 [==============================] - 1s 102ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.93      0.87      0.90        31\n",
      "     Class 1       0.50      0.22      0.31         9\n",
      "     Class 2       0.33      0.62      0.43         8\n",
      "\n",
      "    accuracy                           0.71        48\n",
      "   macro avg       0.59      0.57      0.55        48\n",
      "weighted avg       0.75      0.71      0.71        48\n",
      "\n",
      "2/2 [==============================] - 0s 117ms/step\n",
      "[[27  1  3]\n",
      " [ 0  2  7]\n",
      " [ 2  1  5]]\n",
      "run number 8\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9542 - accuracy: 0.3497\n",
      "Epoch 1: val_loss improved from inf to 1.09642, saving model to mod_.h5\n",
      "99/99 [==============================] - 95s 933ms/step - loss: 1.9542 - accuracy: 0.3497 - val_loss: 1.0964 - val_accuracy: 0.2128\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6949 - accuracy: 0.5114\n",
      "Epoch 2: val_loss improved from 1.09642 to 0.87680, saving model to mod_.h5\n",
      "99/99 [==============================] - 110s 1s/step - loss: 1.6949 - accuracy: 0.5114 - val_loss: 0.8768 - val_accuracy: 0.5957\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6179 - accuracy: 0.5524\n",
      "Epoch 3: val_loss did not improve from 0.87680\n",
      "99/99 [==============================] - 96s 967ms/step - loss: 1.6179 - accuracy: 0.5524 - val_loss: 0.8770 - val_accuracy: 0.5957\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5490 - accuracy: 0.5878\n",
      "Epoch 4: val_loss did not improve from 0.87680\n",
      "99/99 [==============================] - 97s 979ms/step - loss: 1.5490 - accuracy: 0.5878 - val_loss: 1.1195 - val_accuracy: 0.5106\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5422 - accuracy: 0.5878\n",
      "Epoch 5: val_loss improved from 0.87680 to 0.83686, saving model to mod_.h5\n",
      "99/99 [==============================] - 88s 891ms/step - loss: 1.5422 - accuracy: 0.5878 - val_loss: 0.8369 - val_accuracy: 0.6596\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4673 - accuracy: 0.6054\n",
      "Epoch 6: val_loss did not improve from 0.83686\n",
      "99/99 [==============================] - 72s 731ms/step - loss: 1.4673 - accuracy: 0.6054 - val_loss: 0.9075 - val_accuracy: 0.6170\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3983 - accuracy: 0.6389\n",
      "Epoch 7: val_loss improved from 0.83686 to 0.59490, saving model to mod_.h5\n",
      "99/99 [==============================] - 73s 733ms/step - loss: 1.3983 - accuracy: 0.6389 - val_loss: 0.5949 - val_accuracy: 0.7872\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3404 - accuracy: 0.6408\n",
      "Epoch 8: val_loss did not improve from 0.59490\n",
      "99/99 [==============================] - 74s 748ms/step - loss: 1.3404 - accuracy: 0.6408 - val_loss: 0.6036 - val_accuracy: 0.7447\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2964 - accuracy: 0.6686\n",
      "Epoch 9: val_loss did not improve from 0.59490\n",
      "99/99 [==============================] - 76s 761ms/step - loss: 1.2964 - accuracy: 0.6686 - val_loss: 0.8206 - val_accuracy: 0.6383\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2213 - accuracy: 0.6824\n",
      "Epoch 10: val_loss improved from 0.59490 to 0.55122, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 771ms/step - loss: 1.2213 - accuracy: 0.6824 - val_loss: 0.5512 - val_accuracy: 0.7447\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2634 - accuracy: 0.6610\n",
      "Epoch 11: val_loss improved from 0.55122 to 0.54383, saving model to mod_.h5\n",
      "99/99 [==============================] - 91s 920ms/step - loss: 1.2634 - accuracy: 0.6610 - val_loss: 0.5438 - val_accuracy: 0.7234\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1801 - accuracy: 0.6995\n",
      "Epoch 12: val_loss did not improve from 0.54383\n",
      "99/99 [==============================] - 91s 916ms/step - loss: 1.1801 - accuracy: 0.6995 - val_loss: 0.5782 - val_accuracy: 0.7447\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1414 - accuracy: 0.6951\n",
      "Epoch 13: val_loss did not improve from 0.54383\n",
      "99/99 [==============================] - 84s 853ms/step - loss: 1.1414 - accuracy: 0.6951 - val_loss: 0.7253 - val_accuracy: 0.6383\n",
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0713 - accuracy: 0.7184\n",
      "Epoch 14: val_loss did not improve from 0.54383\n",
      "99/99 [==============================] - 73s 735ms/step - loss: 1.0713 - accuracy: 0.7184 - val_loss: 0.7872 - val_accuracy: 0.6383\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0507 - accuracy: 0.7386\n",
      "Epoch 15: val_loss did not improve from 0.54383\n",
      "99/99 [==============================] - 79s 802ms/step - loss: 1.0507 - accuracy: 0.7386 - val_loss: 0.8935 - val_accuracy: 0.6170\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0617 - accuracy: 0.7418\n",
      "Epoch 16: val_loss improved from 0.54383 to 0.53252, saving model to mod_.h5\n",
      "99/99 [==============================] - 77s 774ms/step - loss: 1.0617 - accuracy: 0.7418 - val_loss: 0.5325 - val_accuracy: 0.7872\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9606 - accuracy: 0.7658\n",
      "Epoch 17: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 79s 801ms/step - loss: 0.9606 - accuracy: 0.7658 - val_loss: 0.5680 - val_accuracy: 0.8298\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9948 - accuracy: 0.7487\n",
      "Epoch 18: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 73s 737ms/step - loss: 0.9948 - accuracy: 0.7487 - val_loss: 0.5404 - val_accuracy: 0.7660\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8933 - accuracy: 0.7746\n",
      "Epoch 19: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 82s 833ms/step - loss: 0.8933 - accuracy: 0.7746 - val_loss: 0.6285 - val_accuracy: 0.7660\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8752 - accuracy: 0.8024\n",
      "Epoch 20: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 83s 836ms/step - loss: 0.8752 - accuracy: 0.8024 - val_loss: 0.6317 - val_accuracy: 0.8085\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8787 - accuracy: 0.8081\n",
      "Epoch 21: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 0.8787 - accuracy: 0.8081 - val_loss: 0.6900 - val_accuracy: 0.8298\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7922 - accuracy: 0.8453\n",
      "Epoch 22: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 73s 738ms/step - loss: 0.7922 - accuracy: 0.8453 - val_loss: 0.6680 - val_accuracy: 0.7660\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7068 - accuracy: 0.8529\n",
      "Epoch 23: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 73s 736ms/step - loss: 0.7068 - accuracy: 0.8529 - val_loss: 0.7398 - val_accuracy: 0.7660\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7113 - accuracy: 0.8479\n",
      "Epoch 24: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 75s 758ms/step - loss: 0.7113 - accuracy: 0.8479 - val_loss: 0.7735 - val_accuracy: 0.7660\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6962 - accuracy: 0.8580\n",
      "Epoch 25: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 74s 746ms/step - loss: 0.6962 - accuracy: 0.8580 - val_loss: 0.6134 - val_accuracy: 0.8085\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6418 - accuracy: 0.8668\n",
      "Epoch 26: val_loss did not improve from 0.53252\n",
      "99/99 [==============================] - 70s 702ms/step - loss: 0.6418 - accuracy: 0.8668 - val_loss: 0.6939 - val_accuracy: 0.8298\n",
      "Test loss:  0.5847977995872498\n",
      "Test accuracy:  0.75\n",
      "2/2 [==============================] - 1s 100ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.96      0.84      0.90        31\n",
      "     Class 1       0.71      0.56      0.63         9\n",
      "     Class 2       0.36      0.62      0.45         8\n",
      "\n",
      "    accuracy                           0.75        48\n",
      "   macro avg       0.68      0.67      0.66        48\n",
      "weighted avg       0.82      0.75      0.77        48\n",
      "\n",
      "2/2 [==============================] - 0s 94ms/step\n",
      "[[26  0  5]\n",
      " [ 0  5  4]\n",
      " [ 1  2  5]]\n",
      "run number 9\n",
      "Epoch 1/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.1015 - accuracy: 0.3308\n",
      "Epoch 1: val_loss improved from inf to 1.53201, saving model to mod_.h5\n",
      "99/99 [==============================] - 80s 768ms/step - loss: 2.1015 - accuracy: 0.3308 - val_loss: 1.5320 - val_accuracy: 0.1489\n",
      "Epoch 2/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 2.0315 - accuracy: 0.3220\n",
      "Epoch 2: val_loss improved from 1.53201 to 1.46027, saving model to mod_.h5\n",
      "99/99 [==============================] - 73s 735ms/step - loss: 2.0315 - accuracy: 0.3220 - val_loss: 1.4603 - val_accuracy: 0.1489\n",
      "Epoch 3/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.9670 - accuracy: 0.3327\n",
      "Epoch 3: val_loss improved from 1.46027 to 1.27134, saving model to mod_.h5\n",
      "99/99 [==============================] - 66s 671ms/step - loss: 1.9670 - accuracy: 0.3327 - val_loss: 1.2713 - val_accuracy: 0.1489\n",
      "Epoch 4/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.8328 - accuracy: 0.4179\n",
      "Epoch 4: val_loss improved from 1.27134 to 0.85759, saving model to mod_.h5\n",
      "99/99 [==============================] - 66s 668ms/step - loss: 1.8328 - accuracy: 0.4179 - val_loss: 0.8576 - val_accuracy: 0.5957\n",
      "Epoch 5/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.7373 - accuracy: 0.4773\n",
      "Epoch 5: val_loss did not improve from 0.85759\n",
      "99/99 [==============================] - 72s 724ms/step - loss: 1.7373 - accuracy: 0.4773 - val_loss: 0.9243 - val_accuracy: 0.6383\n",
      "Epoch 6/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.6172 - accuracy: 0.5354\n",
      "Epoch 6: val_loss improved from 0.85759 to 0.70911, saving model to mod_.h5\n",
      "99/99 [==============================] - 76s 768ms/step - loss: 1.6172 - accuracy: 0.5354 - val_loss: 0.7091 - val_accuracy: 0.6809\n",
      "Epoch 7/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5623 - accuracy: 0.5657\n",
      "Epoch 7: val_loss did not improve from 0.70911\n",
      "99/99 [==============================] - 72s 730ms/step - loss: 1.5623 - accuracy: 0.5657 - val_loss: 0.8803 - val_accuracy: 0.6383\n",
      "Epoch 8/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.5315 - accuracy: 0.5896\n",
      "Epoch 8: val_loss did not improve from 0.70911\n",
      "99/99 [==============================] - 71s 720ms/step - loss: 1.5315 - accuracy: 0.5896 - val_loss: 0.7766 - val_accuracy: 0.6383\n",
      "Epoch 9/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4792 - accuracy: 0.5985\n",
      "Epoch 9: val_loss improved from 0.70911 to 0.62357, saving model to mod_.h5\n",
      "99/99 [==============================] - 73s 735ms/step - loss: 1.4792 - accuracy: 0.5985 - val_loss: 0.6236 - val_accuracy: 0.7021\n",
      "Epoch 10/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.4271 - accuracy: 0.6092\n",
      "Epoch 10: val_loss did not improve from 0.62357\n",
      "99/99 [==============================] - 69s 697ms/step - loss: 1.4271 - accuracy: 0.6092 - val_loss: 0.8818 - val_accuracy: 0.6383\n",
      "Epoch 11/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.3903 - accuracy: 0.6206\n",
      "Epoch 11: val_loss did not improve from 0.62357\n",
      "99/99 [==============================] - 75s 761ms/step - loss: 1.3903 - accuracy: 0.6206 - val_loss: 0.7204 - val_accuracy: 0.6170\n",
      "Epoch 12/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2756 - accuracy: 0.6591\n",
      "Epoch 12: val_loss did not improve from 0.62357\n",
      "99/99 [==============================] - 74s 743ms/step - loss: 1.2756 - accuracy: 0.6591 - val_loss: 0.6838 - val_accuracy: 0.6596\n",
      "Epoch 13/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2576 - accuracy: 0.6528\n",
      "Epoch 13: val_loss improved from 0.62357 to 0.57412, saving model to mod_.h5\n",
      "99/99 [==============================] - 72s 724ms/step - loss: 1.2576 - accuracy: 0.6528 - val_loss: 0.5741 - val_accuracy: 0.7660\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.2306 - accuracy: 0.6730\n",
      "Epoch 14: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 81s 820ms/step - loss: 1.2306 - accuracy: 0.6730 - val_loss: 0.5797 - val_accuracy: 0.7021\n",
      "Epoch 15/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1682 - accuracy: 0.6856\n",
      "Epoch 15: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 69s 699ms/step - loss: 1.1682 - accuracy: 0.6856 - val_loss: 0.5768 - val_accuracy: 0.7447\n",
      "Epoch 16/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1441 - accuracy: 0.6881\n",
      "Epoch 16: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 72s 728ms/step - loss: 1.1441 - accuracy: 0.6881 - val_loss: 0.5886 - val_accuracy: 0.7021\n",
      "Epoch 17/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.1253 - accuracy: 0.7153\n",
      "Epoch 17: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 67s 679ms/step - loss: 1.1253 - accuracy: 0.7153 - val_loss: 0.5917 - val_accuracy: 0.7872\n",
      "Epoch 18/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0556 - accuracy: 0.7431\n",
      "Epoch 18: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 67s 673ms/step - loss: 1.0556 - accuracy: 0.7431 - val_loss: 0.6477 - val_accuracy: 0.7872\n",
      "Epoch 19/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 1.0188 - accuracy: 0.7431\n",
      "Epoch 19: val_loss did not improve from 0.57412\n",
      "99/99 [==============================] - 69s 693ms/step - loss: 1.0188 - accuracy: 0.7431 - val_loss: 0.7129 - val_accuracy: 0.7021\n",
      "Epoch 20/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9771 - accuracy: 0.7443\n",
      "Epoch 20: val_loss improved from 0.57412 to 0.56592, saving model to mod_.h5\n",
      "99/99 [==============================] - 67s 673ms/step - loss: 0.9771 - accuracy: 0.7443 - val_loss: 0.5659 - val_accuracy: 0.7447\n",
      "Epoch 21/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9979 - accuracy: 0.7538\n",
      "Epoch 21: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 67s 672ms/step - loss: 0.9979 - accuracy: 0.7538 - val_loss: 0.7929 - val_accuracy: 0.6809\n",
      "Epoch 22/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.9293 - accuracy: 0.7708\n",
      "Epoch 22: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 71s 720ms/step - loss: 0.9293 - accuracy: 0.7708 - val_loss: 0.7180 - val_accuracy: 0.8085\n",
      "Epoch 23/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8840 - accuracy: 0.7986\n",
      "Epoch 23: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 74s 742ms/step - loss: 0.8840 - accuracy: 0.7986 - val_loss: 0.6967 - val_accuracy: 0.7447\n",
      "Epoch 24/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8383 - accuracy: 0.8144\n",
      "Epoch 24: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 73s 737ms/step - loss: 0.8383 - accuracy: 0.8144 - val_loss: 0.7123 - val_accuracy: 0.7872\n",
      "Epoch 25/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7984 - accuracy: 0.8188\n",
      "Epoch 25: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 72s 729ms/step - loss: 0.7984 - accuracy: 0.8188 - val_loss: 0.7778 - val_accuracy: 0.8085\n",
      "Epoch 26/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.7617 - accuracy: 0.8308\n",
      "Epoch 26: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 74s 751ms/step - loss: 0.7617 - accuracy: 0.8308 - val_loss: 0.7742 - val_accuracy: 0.7660\n",
      "Epoch 27/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.8469 - accuracy: 0.8125\n",
      "Epoch 27: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 71s 717ms/step - loss: 0.8469 - accuracy: 0.8125 - val_loss: 0.6494 - val_accuracy: 0.7660\n",
      "Epoch 28/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6792 - accuracy: 0.8598\n",
      "Epoch 28: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 67s 675ms/step - loss: 0.6792 - accuracy: 0.8598 - val_loss: 0.7782 - val_accuracy: 0.7660\n",
      "Epoch 29/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.6362 - accuracy: 0.8763\n",
      "Epoch 29: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 76s 770ms/step - loss: 0.6362 - accuracy: 0.8763 - val_loss: 0.7557 - val_accuracy: 0.8085\n",
      "Epoch 30/50\n",
      "99/99 [==============================] - ETA: 0s - loss: 0.5331 - accuracy: 0.9091\n",
      "Epoch 30: val_loss did not improve from 0.56592\n",
      "99/99 [==============================] - 74s 746ms/step - loss: 0.5331 - accuracy: 0.9091 - val_loss: 0.8927 - val_accuracy: 0.7021\n",
      "Test loss:  0.4913924038410187\n",
      "Test accuracy:  0.7708333134651184\n",
      "2/2 [==============================] - 1s 95ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.93      0.90      0.92        31\n",
      "     Class 1       0.55      0.67      0.60         9\n",
      "     Class 2       0.43      0.38      0.40         8\n",
      "\n",
      "    accuracy                           0.77        48\n",
      "   macro avg       0.64      0.65      0.64        48\n",
      "weighted avg       0.78      0.77      0.77        48\n",
      "\n",
      "2/2 [==============================] - 0s 120ms/step\n",
      "[[28  2  1]\n",
      " [ 0  6  3]\n",
      " [ 2  3  3]]\n"
     ]
    }
   ],
   "source": [
    "## fitting best model and averaging confusion matrix over 10 runs\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from keras.utils import np_utils\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from PIL import Image\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Conv2D, MaxPooling2D, Flatten\n",
    "from keras import optimizers\n",
    "from keras import losses\n",
    "from tensorflow.keras.layers import AveragePooling2D\n",
    "import cv2\n",
    "from sklearn import metrics\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.initializers import he_uniform\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import pickle\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## loading data\n",
    "x_train = np.load(\"created_data/MIAS/patches/MIAS_X_train_roi_multi.npy\") / 255\n",
    "y_train = np.load(\"created_data/MIAS/patches/MIAS_y_train_roi_multi.npy\")\n",
    "x_valid = np.load(\"created_data/MIAS/patches/MIAS_X_valid_roi_multi.npy\") / 255\n",
    "y_valid = np.load(\"created_data/MIAS/patches/MIAS_y_valid_roi_multi.npy\") \n",
    "x_test = np.load(\"created_data/MIAS/patches/MIAS_X_test_roi_multi.npy\") / 255\n",
    "y_test = np.load(\"created_data/MIAS/patches/MIAS_y_test_roi_multi.npy\") \n",
    "\n",
    "## printing class counts for training set \n",
    "print(np.unique(y_train, return_counts = True))\n",
    "\n",
    "# One-hot encode labels\n",
    "num_classes = 3\n",
    "y_train = np_utils.to_categorical(y_train, num_classes)\n",
    "y_valid = np_utils.to_categorical(y_valid, num_classes)\n",
    "y_test = np_utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "## define number of runs\n",
    "n_runs = 10\n",
    "\n",
    "## list for appending confusion matrices and metrices over n_runs\n",
    "confusion_matrices = []\n",
    "metrices = []\n",
    "\n",
    "## run the model n_runs times\n",
    "for n in range(n_runs):\n",
    "    print(f\"run number {n}\")\n",
    "    \n",
    "    # Define model\n",
    "    model = Sequential()\n",
    "\n",
    "    # Add layers to model\n",
    "    \n",
    "    ## first convolutional layer and one maxpooling layer \n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', input_shape=(224, 224, 3), strides = 1, kernel_initializer=he_uniform(), name = 'conv2d'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d'))\n",
    "    \n",
    "    ## second convolutional layer and one maxpooling layer \n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', strides = 1, kernel_initializer=he_uniform(), name = 'conv2d_1'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_1'))\n",
    "    \n",
    "    ## third convolutional layer and one maxpooling layer \n",
    "    model.add(Conv2D(14, (3, 3), activation='relu', strides = 1, padding = \"SAME\", kernel_initializer=he_uniform(), name = 'conv2d_2'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_2'))\n",
    "    model.add(MaxPooling2D((2, 2), strides = 2, name = 'max_pooling2d_3'))\n",
    "    \n",
    "    ## Flattening layer\n",
    "    model.add(Flatten(name = 'flatten'))\n",
    "    \n",
    "    ## Three dense layers\n",
    "    model.add(Dense(512, activation='relu', kernel_initializer=he_uniform(), name = 'dense'))\n",
    "    model.add(Dense(256, activation='relu', kernel_initializer=he_uniform(), name = 'dense_1'))\n",
    "    model.add(Dense(128, activation='relu', kernel_initializer=he_uniform(), name = 'dense_2'))\n",
    "    \n",
    "    ## Dropout\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    ## Output layer\n",
    "    model.add(Dense(3, activation='softmax', kernel_initializer=he_uniform(), name = 'dense_3'))\n",
    "\n",
    "    lr = 0.0001 # set the learning rate\n",
    "    adam = Adam(learning_rate=lr) \n",
    "\n",
    "    # Compile model\n",
    "    model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    # Define early stopping callback\n",
    "    early_stop = EarlyStopping(monitor='val_loss', patience=10,restore_best_weights=True)\n",
    "    \n",
    "    ## store first model separately\n",
    "    if n == 0:\n",
    "        model_checkpoint = ModelCheckpoint('best_model.h5', save_best_only=True, verbose = 1)\n",
    "    else: ## other models get stored per run but overwrite each other \n",
    "        model_checkpoint = ModelCheckpoint('mod_.h5', save_best_only=True, verbose = 1)\n",
    "\n",
    "    # Train model with early stopping\n",
    "    history = model.fit(x_train, y_train, epochs = 50, batch_size = 16, validation_data = (x_valid, y_valid),\n",
    "    class_weight = best_class_weight, callbacks = [early_stop,model_checkpoint])\n",
    "\n",
    "    ## load best model\n",
    "    from keras.models import load_model\n",
    "    if n == 0:\n",
    "        model = load_model('best_model.h5')\n",
    "    \n",
    "    else:\n",
    "        model = load_model('mod_.h5')\n",
    "\n",
    "    # Evaluate model\n",
    "    test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0)\n",
    "    print(\"Test loss: \", test_loss)\n",
    "    print(\"Test accuracy: \", test_acc)\n",
    "\n",
    "    if n == 0: ## store history for first model \n",
    "        with open('history_best_model.pkl', 'wb') as f:\n",
    "            pickle.dump(history.history, f)\n",
    "    \n",
    "    ## evaluate model on test set \n",
    "    report = evaluate_model(model,x_test,y_test)\n",
    "    print(report)\n",
    "    \n",
    "    ## Get predictions for test est\n",
    "    y_pred = model.predict(x_test)\n",
    "    y_pred_classes = np.argmax(y_pred, axis=1)\n",
    "    y_true = np.argmax(y_test, axis=1)\n",
    "\n",
    "    # Calculate confusion matrix for test data\n",
    "    cm = confusion_matrix(y_true, y_pred_classes)\n",
    "    print(cm)\n",
    "    \n",
    "    class_names = ['Class 0', 'Class 1', 'Class 2']\n",
    "    \n",
    "    ## calculate classification report for test data\n",
    "    report = classification_report(y_true, y_pred_classes, target_names=class_names, output_dict = True)\n",
    "    \n",
    "    ## append confusion matrices and classification report\n",
    "    confusion_matrices.append(cm)\n",
    "    metrices.append(report)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e98112b",
   "metadata": {},
   "source": [
    "## Create dataframe of averaged confusion matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "15671b1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[28.   1.2  1.8]\n",
      " [ 0.   5.4  3.6]\n",
      " [ 1.2  3.   3.8]]\n",
      "<class 'numpy.ndarray'>\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT0AAAE9CAYAAAB5m7WdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiBUlEQVR4nO3dd3gVZfrG8e9DYqEFBBFRVCwoC4iIqKCigGXtaxfWioW1rRXbT6WKFVxXsaCiWFbWAiqsvbHuYqOqYF8LSrXQSyAnz++PmcAhJOGQZM4kmftzXefinJkz8z4Zws209x1zd0REkqJW3AWIiGSTQk9EEkWhJyKJotATkURR6IlIoij0RCRRcuMuoDS197xE99JkyZz3/x53CYmwOqVf6WxpUi/XSpunPT0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEyY1ipWbWoaz57j4linZFRDYkktADhpYxz4HuEbUrIlKmSELP3btFsV4RkYqKak9vDTNrC7QGNi+a5u5PRN2uiEhJIg09M+sHdCUIvVeAI4D/Ago9EYlF1FdvTwIOBua6ey9gD6BBxG2KiJQq6tBb4e6FQIGZ5QHzge0iblNEpFRRn9ObZGYNgYeBycBS4IOI2xQRKVWkoefuF4VvHzSz14A8d/80yjZFRMqSjau37YAWRW2Z2S7uPibqdkVEShL11dtHgXbADKAwnOyAQk9EYhH1nl4nd28dcRuxaN60IY8MOpOtGtfHHR4dPYH7Ro2n3a7bcu8NPdhss00oSBVy+S3PMGnGj3GXWy0N6ncDE977N1s0asSo0WPXm//ay+N4cuQI3J06depyzQ192XW3VjFUWv3dMuBG3v9PsK2ffPal9eYvXbKEgTddy7y5c0ilUvQ8oxdHHXt8DJVWXNRXbz8wsxoZegWpQq67awwdThzMQWcO4S+nHkirnbZm8OXHMfihV+nU4zYGPfAvBl9+XNylVltHH3s8d9//UKnzt9m2OQ+MeJynn3+Jc3pfwG2D+mWxuprlyGOOY+i9w0udP+a5UbTYaWce/+cL3PvQSIb97Q5Wr16VxQorT9R7ek8QBN9cIB8wwN29XcTtRm7ur4uZ++tiAJYuz+fL7+eyTZOGuENe3aDzSYN6tZnzy6I4y6zW9tyrI7NnzSp1frv2e65537bdHsyfNy8bZdVI7Tt0ZM7s0re1YSxftgx3Z8Xy5eTlNSAnJ/JLApGIuuoRwBnAZ6w9p1fjbN+sEe13a87E6T9w9ZDnGXffxdx6xfHUqmV0O7ussReksox9YTSdD+gSdxk11omn/plrr7iY4/7YleXLlzHg1qHUqlU9R6aLuupf3H2su3/v7j8WvUr7spn1NrNJZjap4NcZEZdWOerW3pRRQ87j6iGjWbJsJb1P7sI1Q8fQ8oibuGbIaB7od1rcJdZ4kyZ+xLgXx3DJZVfFXUqN9dEH/6Xlbq148fXxPDZqNH+7YzDLli6Nu6xyiTr0pprZ02bW08xOKHqV9mV3f8jdO7p7x9wt20RcWsXl5tZi1JDzeebVSbz0zicAnHb0vrz49jQARr85lY5tdoixwprvm6+/4pYBfbnz7mE0aNgw7nJqrFfGvshB3Q/FzGi+3Q4022Zbfvzhu7jLKpeoQ682wbm8w4BjwtfREbeZNQ/2O42vvp/LPU+9s2banF8W0WWvlgB03WdXvp35S1zl1Xhz58zmuqsupf/Nt7H9Di3iLqdGa7p1MyZ9/CEAv//2KzN//IFttq2ePUrN3aNZsVkOcLu79ynP8rX3vCSawirJfu134u3HruSzr2dRGG7DfsPGsmTpSu68+iRyc2uRn1/AZbc+w9Qvfoq52rLNef/vcZdQohuv68OUSR+zcOFCGjVqTO8LL6GgYDUAJ5zcg8EDbuLdt95k62bNAMjJzeXxp5+Ls+QyrU5V3V/pfv/Xh2mTJgbbunFjzv3LxRQUFABw3Emn8usv8xnc7wZ++/UXHOf0s8/jj0ceE3PVpWtSL9dKmxdZ6AGY2Qfu3rk8y1b10KtJqmro1TRVOfRqmrJCL+qrt9PMbCzwHLCsaKK6oYlIXKIOvc2B31j3mRjqhiYisYl6lJVeUa5fRGRjRXr11syam9kLZjY/fI02s+ZRtikiUpaob1l5DBgLbBO+xoXTRERiEXXoNXH3x9y9IHyNBJpE3KaISKmiDr3fzOx0M8sJX6cTXNgQEYlF1KF3DnAKMBeYQ/B0NF3cEJHYRH319kfg2CjbEBHZGJGEnpn1LWO2u/ugKNoVEdmQqPb0lpUwrS5wLtAYUOiJSCwiCT13XzNyppnVBy4jOJf3T0CjaopIbCI7p2dmjYArgdOAx4EO7r4gqvZERDIR1Tm9O4ETgIeA3d29eg6xKiI1TlS3rFxF0APjRmC2mS0OX0vMbHFEbYqIbFBU5/Sq5xNDRKTGUziJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKIo9EQkURR6IpIoCj0RSRSFnogkikJPRBJFoSciiaLQE5FEUeiJSKKYu8ddQ4lWFlA1C6uBvp6zNO4SEqF+7dy4S0iMHbfc3Eqbpz09EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiiKPREJFEUeiKSKAo9EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiibFTomVktM8uLqhgRkahtMPTM7GkzyzOzusB04HMzuzr60kREKl8me3qt3X0xcBzwKrAjcEaURYmIRCWT0NvEzDYhCL2x7r4aNKqxiFRPmYTecOAHoC7wnpntACyOsigRkaiU6xkZZpbr7gUR1LOGnpGRPXpGRnboGRnZU6FnZJhZUzMbYWavhp9bA2dVYn0iIlmTyeHtSOB1YJvw89fA5RHVIyISqUxCb0t3fxYoBAgPa1ORViUiEpFMQm+ZmTUmvGJrZp2ARZFWJSISkUzOrF4JjAV2NrMJQBPgpEirEhGJyAZDz92nmNlBwG6AAV+F9+qJiFQ7mVy9PRmo7e4zCG5QfsbMOkRdmIhIFDI5p3eTuy8xswOAg4ERwAPRliUiEo1MQq/oSu1RwMPu/jKwaXQliYhEJ5MLGbPMbDhwKHC7mW3GRgxJZWY5QNP0ttx95sYWKiJSGTIJvVOAw4Eh7r7QzJoBGQ0tZWZ/BfoB8wjv8yO49aVdOWoVEamwTEKvGfCyu+ebWVeCwHoiw/VfBuzm7r+VrzwRkcqVyWHqaCBlZrsADwHbAU9nuP6f0I3MIlKFZLKnV+juBWZ2AnCvu99rZlMzXP93wHgzexnIL5ro7neVo1YRkQrLJPRWm1lP4EzgmHDaJhmuf2b42hRd8RWRKiCT0OsFXAAMdvfvzWxH4MlMVu7uAypSnIhIZSvXIKIZr9xsHOsPLb8ImAQMd/eVpS2rQUSzR4OIZocGEc2eig4i2tLMnjezz83su6JXhm1/BywFHg5fi4ElwK7hZxGRrMrkv57HCO61+xvQjeBwN9Obk/dz973TPo8zs4nuvreZzdi4UkVEKi6T0Kvt7m+bmbn7j0B/M5sM9M1g2Xpmtn1RDwwz2x6oF85bVb6Sq6YJ/3mP228bTGGqkONPPJlzz+8dd0k1wkWnHc3mtetQKyeHnJwcbr//qRK/9+2XM7jh0l5cfuMtdD7wkCxXWb2tys+nz8W9WL16NamCArp0O5Qzzrtove+99/brPPXogwDs1HI3rut/W7ZLrRSZhF6+mdUCvjGzS4BZrA2uDbkK+K+Z/Y9gWKodgYvCB4c/Xp6Cq6JUKsUtgwcy/OHHaNq0KX8+9SS6duvOzrvsEndpNUL/ocPJa7BFqfNTqRRPPXIPe3TslMWqao5NNt2U2+95hNp16lBQsJqrLjybjp0O4A9t13acmvXTjzzz5AiGPvA49fPyWLig+vY3yCT0LgPqAJcCg4DuZPhgIHd/xcxaAq3CSV+lXby4e+NKrbqmf/Yp2223A8232w6Aw488ivHvvq3Qy5LXXnyGTl0O5tuvdMakPMyM2nXqAFBQUEBBQQFW7DLAq2PHcPQJPaiflwdAwy0aZ7vMSpPJIKITw7dLCc7nbZCZdXf3d8IbmtPtbGa4+5iNrLNKmz9vHls323rN562aNuWzTz+NsaIaxIybr70YzDj0qBM59Oh1f6V++3U+H014l/5Dhiv0KiCVSvHXc3oye9ZMjjnhVFq1Wbd7/KyffgTgygvOojCV4vRzL6Rjp/3jKLXCSg29Um43WcPdjy1jvQcB77D2ZuZ1FgVqVOhJdAbdPYLGW27FogW/M+jai9h2+xa0brd2DNuR9w/h9PMupVatjAf+kRLk5ORw/+PPsnTJYgZefwU/fPcNLXZquWZ+KlXA7J9/5I5hj/Dr/Hn0ufgcHnzieerVz4ux6vIpa09vSHlX6u79wj8z2jMsYma9gd4Aw+4fXm0uBmzVtClz58xd83n+vHk0bdo0xopqjsZbbgVAgy0asc/+3fj2y+nrhN7/vv6CuwdfD8DiRQuZ+vEEcnJy2Gf/brHUW93Vq5/HHh32ZtKH768Tels2aUqrNruTm7sJW2/TnObb7cCsn2ey2x/axlht+ZQaeu7+b4DwosMKdy8MP+cAm2Wy8nDsvROBFqw7nt7AUtp8iGBQg2p1c3Kbtrszc+YP/PzzTzTdqimvvfIyt945NO6yqr2VK1bgXkjtOnVZuWIFn0z+kJNOP3+d79z/1Lg174fd0Y+9OnVR4G2khQt+Jzc3l3r188jPX8mUiR9yyunr7q/sd2B3xr/5KocddRyLFi7g559+pNk2zWOquGIyuZDxNnAIwTk9gNrAG8B+GSz7EkEPjMmkDThQ0+Tm5nL9DX25sPd5FBamOO74E9lll5YbXlDKtGjBb9zZvw8QnHM6oPvh7LnPfrwx7nkADjtGD+WrDL//9itDb76RVGEhXljIgd0PY9/9D+KJh++jZas2dO7Slb323Y/JH79P79OOp1atWpx38RXkNWgYd+nlssFuaGY2zd3bb2haKctOd/dy7f9Wpz296k7d0LJD3dCyp0Ld0Age9r3mJIqZ7QWsyLDt981s9wy/KyISuUz+67kceM7MZhPcYLw1cGqG6z8AONvMvic4vDXA3V3DxYtILDK6T8/MWhE87Bs27mHfR5S7MhGRCGR0c5O7r3b36eEr08Aj7Ku7HdA9fL880zZFRKIQaQCZWT/gWuD6cNImQMk9xkVEsiDqva7jgWOBZQDuPhuoH3GbIiKlymQQUTOz082sb/h5ezPbJ8P1r/LgnhgPl61b/lJFRCoukz29+4HOQM/w8xLgvgzX/6yZDQcamtn5wFtoxGQRiVEmt6zs6+4dih776O4LzCyjJ5u5+xAzO5RgmPjdgL7u/mb5yxURqZhMHwGZw9pD1CZAYaYNhCH3ppltCVTfkQdFpEbI5PD2HuAFYCszGwz8F7ilrAXMrJOZjTezMWa2p5lNB6YD88zs8ApXLSJSTpncnPyP8JkYBxP0qDjO3b/YwGLDgP8DGhCMq3eEu38Y3uQ8CnitYmWLiJTPBkMvfJjPcmBc+rSih/2Utl53fyP87kB3/xDA3b+04uNQi4hkUSbn9F4mOJ9nwOYED/f5CmhTxjLp5/yKD06g0VNEJDaZHN6uM0pKOOLK+s+HW9ceZraYIChrh+9hbXCKiMRiowf4cvcpZrbvBr6TU/6SRESik8k5vSvTPtYCOgCzI6tIRCRCmezppfeVLSA4xzc6mnJERKJVZuiFNyXXd/c+WapHRCRSpd6cbGa57p4CqucTfUVESlDWnt7HBOfvppnZWOA5wiGiANxdD+wWkWonk3N6mxP0me3O2vv1HFDoiUi1U1bobRVeuZ3O2rArohuMRaRaKiv0coB6rBt2RRR6IlItlRV6c9x9YNYqERHJgrKGltLIACJS45QVegdnrQoRkSwpNfTc/fdsFiIikg168LaIJIpCT0QSRaEnIomi0BORRFHoiUiiKPREJFEUeiKSKAo9EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiiZPKMjFisXJ2Ku4TE2KLuJnGXkAirUoVxlyBoT09EEkahJyKJotATkURR6IlIoij0RCRRFHoikigKPRFJFIWeiCSKQk9EEkWhJyKJotATkURR6IlIoij0RCRRFHoikigKPRFJFIWeiCSKQk9EEkWhJyKJotATkURR6IlIoij0RCRRFHoikiiRhp6ZnZzJNBGRbIl6T+/6DKeJiGRFJA/7NrMjgCOBbc3snrRZeUBBFG2KiGQiktADZgOTgGOByWnTlwBXRNSmiMgGmbtHt3KzTdx9dXmWXbgiFV1hso4lK7TznQ2rUoVxl5AYOzepbaXNi2pPr8g+ZtYf2CFsywB3950ibldEpERRh94IgsPZyUAq4rZERDYo6tBb5O6vRtyGiEjGog69d83sTmAMkF800d2nRNyuiEiJog69fcM/O6ZNc6B7xO2KiJQo0tBz925Rrl9EZGNFvaeHmR0FtAE2L5rm7gOjbldEpCRR9719EDgV+CvB7SonE9y+IiISi6j73u7n7mcCC9x9ANAZ2DXiNkVEShV16K0I/1xuZtsAq4FmEbcpIlKqqM/p/cvMGgJ3AlMIrtw+EnGbIiKlirTv7ToNmW0GbO7uizL5vvreZo/63maH+t5mT5x9bzGz/YAWRW2ZGe7+RNTtioiUJNLQM7MngZ2Baazte+tAtQ69Qf1uYMJ7/2aLRo0YNXrsevNfe3kcT44cgbtTp05drrmhL7vu1iqGSmueVfn5XHFhL1avXkUqleLAbodw1vkXx11WtbcqP59rLjmH1atWk0oVcEC3Qzj93IvW+c78uXO4a/BNLF26hMLCQnpdcCl7d+4SU8XlF/XQUl8Arb0cjVTlw9upkydRu04dBtx4XYmh9+m0qbTYaSfy8hrw/n/f45EH7+PRp56JodLMVKfDW3dn5YoV1K5Th4KC1Vz+l7O46Iprad12j7hL26CqfHhbfLv2ubAXF1x2Da3atlvznXtuH8jOu7biqONPYeb3/6Pv1Zcw8vmq2bW+rMPbqK/eTge2jriNrNtzr47k5TUodX679nuumd+23R7MnzcvW6XVeGZG7Tp1ACgoKKCgoACzUn+/JUPFt2sqVQDFtquZsXzZMgCWLVtK4y2bZL3OyhD1Ob0tgc/N7GPWHXDg2IjbrTLGvjCazgdUv0OAqiyVSnFRrx7M+nkmfzqxB39o027DC8kGpVIpLju3J7Nn/cTRx59Kqza7rzP/tHMu4IYrL2Ts6FHkr1jB4LuHx1RpxUQdev0jXn+VNmniR4x7cQwPPfZU3KXUKDk5OQx/4jmWLllMv+uu4Pv/fcOOO7eMu6xqLycnh2Ejn2XpksXc/H9X8sN339Jip13WzB//1mscesSxnNDzTL6Y/glDbr6RB554nlq1qteTZCOt1t3/XdKrtO+bWW8zm2Rmk0aOeDjK0iL3zddfccuAvtx59zAaNGwYdzk1Ur36ebTvsDcTP5wQdyk1Sr36ebTrsDeTi23XN/71Al26HwbAH9ruwer8fBYvWhhDhRUTdd/bJWa2uNjrJzN7wczWGzLe3R9y947u3vHsc8+PsrRIzZ0zm+uuupT+N9/G9ju0iLucGmXhgt9ZumQxAPkrVzJ54gdsv8OOMVdV/S1K3675K5k68UOaF9uuTZo2Y9rkjwCY+cN3rFq1igYNt8h6rRUV9dXbQcDPwNMEAw70ILiFZQpwobt3LW3Zqnz19sbr+jBl0scsXLiQRo0a0/vCSygoCJ5/dMLJPRg84CbefetNtm4W9LjLyc3l8aefi7PkMlWnq7ffffs1tw+8kcLCFO6FHNT9j5xx7gVxl5WRqnz19vtvv2bo4JsoLCzECwvp0v0w/tzrLzz5yP20bNWaTgd0Zeb3/+Pvdwxk5fIVmME5F11Oh332i7v0EpV19Tbq0PvE3fcoNm2au7cvaV66qhx6NU11Cr3qrCqHXk0T5y0ry83sFDOrFb5OAVaG8xRqIpJ1UYfeacAZwHxgXvj+dDOrDVwScdsiIuvJ2oADG0uHt9mjw9vs0OFt9mR9wAEzu8bd7zCzeynhMNbdL42iXRGRDYnq5uQvwj8nRbR+EZFy0eGt6PA2S3R4mz1xHN6Oo4yrs0nqeysiVUtUh7dDIlqviEiFRBJ6ZfWvFRGJU9QjJ7cEbgVas+7Dvtfrdysikg1R35z8GPAAUAB0IxgmXuMsiUhsog692u7+NsFV4h/dvT9wVMRtioiUKupBRPPNrBbwjZldAswC6kXcpohIqaLe07sMqANcCuxF0Pf2rIjbFBEplW5OFt2cnCW6OTl74rg5ef3nIqbRzckiEpeozul1Bn4CRgEfEYyaLCISu6hCb2vgUKAn8GfgZWCUu8+IqD0RkYxEciHD3VPu/pq7nwV0Ar4FxodXcEVEYhPZLStmthnBPXk9gRbAPcALUbUnIpKJqC5kPAG0BV4BBrj79CjaERHZWJHcsmJmhcCy8GN6Awa4u+dtaB26ZSV7dMtKduiWlezJ+i0r7h71Tc8iIuWicBKRRFHoiUiiKPREJFEUeiKSKAo9EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiiKPREJFEUeiKSKAo9EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiiRPKMjCQzs97u/lDcddR02s7ZU9O2tfb0Kl/vuAtICG3n7KlR21qhJyKJotATkURR6FW+GnPuo4rTds6eGrWtdSFDRBJFe3oikigKvZCZuZkNTfvcx8z6Z7mG8WbWMZttxsXMUmY2zcw+MbMpZrZfBdY10MwOqcz6qpPwd/eptM+5ZvaLmf1rA8t1LfqOmR1rZtdFXWta2+3N7MhstZdOobdWPnCCmW1ZnoXNLLeS66npVrh7e3ffA7geuLW8K3L3vu7+VuWVVu0sA9qaWe3w86HArI1ZgbuPdffbKr2y0rUHFHoxKyA4YXtF8Rlm1sLM3jGzT83sbTPbPpw+0sweNLOPgDvCzw+Y2Ydm9l34P+mjZvaFmY1MW98DZjbJzGaY2YBs/YBVWB6woOiDmV1tZhPD7T0gnNYi3I4Ph9vtjaJ/5OF2Pyl8f6SZfWlmk83snrQ9mf7h38X48O/m0hh+zii9AhwVvu8JjCqaYWb7mNkHZjbVzN43s92KL2xmZ5vZsPD9zuHv8GdmdrOZLQ2ndw233/PhNv6HmVk4r2/4dzbdzB5Kmz7ezG43s4/N7Gsz62JmmwIDgVPDvf1TI90yxSj01nUfcJqZNSg2/V7gcXdvB/wDuCdtXnNgP3e/Mvy8BdCZIDzHAn8D2gC7m1n78Ds3uHtHoB1wkJm1i+KHqeJqh7/wXwKPAIMAzOwwoCWwD8HewF5mdmC4TEvgPndvAywETkxfoZltDgwHjnD3vYAmxdpsBfwxXHc/M9skgp8rLv8EeoTboB3wUdq8L4Eu7r4n0Be4ZQPr+jvwd3ffHfi52Lw9gcuB1sBOwP7h9GHuvre7twVqA0enLZPr7vuEy/Vz91VhHc+Ee/vPbNRPWkEKvTTuvhh4Aii+F9AZeDp8/yRwQNq859w9lfZ5nAeXxD8D5rn7Z+5eCMwAWoTfOcXMpgBTCQKxdaX+INVD0eFtK+Bw4Ilw7+Cw8DUVmEIQVC3DZb5392nh+8ms3Z5FWgHfufv34edRxea/7O757v4rMB9oWok/T6zc/VOC7dGTYK8vXQPgOTObztr/hMvSGXgufP90sXkfu/vP4e/0NNb+HXQzs4/M7DOge7E2xoR/lvR3lnU6D7W+uwn+sT2W4feXFfucH/5ZmPa+6HOume0I9AH2dvcF4WHv5uWutgZw9w/Cc6lNAANudffh6d8xsxasuz1TBHsUG6P48jXt938sMAToCjROmz4IeNfdjw+34/gKtLHeNgz3Lu8HOrr7T+EFwM1LWKZKbHPt6RXj7r8DzwLnpk1+H+gRvj8N+E8FmsgjCMpFZtYUOKIC66oRzKwVkAP8BrwOnGNm9cJ525rZVhmu6itgp/AfNkBWzxVVAY8CA9z9s2LTG7D2wsbZGaznQ9aeOuhR1hdDRQH3a/j3dlIGyywB6mfwvUqn0CvZUCD9Ku5fgV5m9ilwBnBZeVfs7p8QHLp9SXDoMKECdVZnRef0pgHPAGe5e8rd3yDYLh+Eh0rPk+E/DndfAVwEvGZmkwn+YS2KpPoqKDzsvKeEWXcAt5rZVDLb07ocuDL8fd+FDWxDd18IPAxMJ/hPa2IGbbwLtI7jQoZ6ZEiNYmb13H1peH7wPuAbd/9b3HVVJ2ZWh+Ccq5tZD6Cnu/8p7roqS+zH1yKV7HwzOwvYlGCPevgGvi/r2wsYFv7HsRA4J95yKpf29EQkUXROT0QSRaEnIomi0BORRFHoyTps7egn083sufBKXnnXld4n9hEzK7XnSdivc6NHWjGzHyzDQSLS+5dKcin0pLii7mFtgVXABekzrZyjybj7ee7+eRlf6QqUe3gpkUwp9KQs/wF2CffC/mNmY4HPzSzHzO5MGwnlLwAWGGZmX5nZW8CanhSWNlagmR1uwRh6n1gwak0LgnC9ItzL7GJmTcxsdNjGRDPbP1y2sQUjrMwws0cIuq2tp3gbJcw/JuwrOtXM3gp7x2BmBxXdNB3Oq29mzczsvbQ94C7hdw+zYPSSKeFecVEvktvM7PNw2wyptL8NqRzurpdea17A0vDPXOAl4EKCvbBlwI7hvN7AjeH7zYBJwI7ACcCbBF3KtiG4x+uk8HvjgY4E/Wt/SltXo/DP/kCftDqeBg4I328PfBG+vwfoG74/CnBgy2I/Q2ltnE0wGggEo+EU3bJ1HjA0fD8O2D98Xy/cDlcRjIxD+LPVJ+ix8x5QN5x+LcHIIY0JusMVrbth3H+neq370s3JUlztsGsYBHt6IwgOOz/2taOXHAa0KzpfR9C3syVwIDDKg1FnZpvZOyWsvxPwXtG6POjrXJJDCLopFX3OC/ekDiQIV9z9ZTNbUMKymbTRHHjGzJoR3Mhc9LNNAO4ys38AY9z9ZzObCDxqwVBUL7r7NDM7iGB0nAlhjZsCHxB02VoJjLBgLL8yRy+W7FPoSXEr3L19+oTwH3X6aDIG/NXdXy/2vcocCbcW0MndV5ZQS2W4F7jL3ceaWVeCPU3c/TYze5lgVN8JZvZHd3/PgjH9jgJGmtldBIOevunuPYuv2Mz2AQ4m6Hh/CcFQS1JF6JyelMfrwIXhng9mtquZ1SU43Ds1POfXDOhWwrIfAgdaMMQWZtYonF581I03CAZ6IPxe+/Dte8Cfw2lHEBymZtpGuvSRR85Ka2dnD8ZAvJ2g43wrM9uBYGzEhwkGPO0QtrG/me0SLlc33A71gAbu/grBQLJ7lNC2xEh7elIejxAMBjkl7J/5C3Ac8ALBXs3nwEyCw711uPsvZtYbGGNmtQgG8zyU4Fza82b2J4KwuxS4z4KRPnIJwu4CYAAwysxmEAz5NXMj2kjXn2BgzQXAOwTnJAEuN7NuBOMfzgBeJRhe6WozWw0sBc4M2zg7rGWzcNkbCcL7JQvGmDOgaERtqSLU91ZEEkWHtyKSKAo9EUkUhZ6IJIpCT0QSRaEnIomi0BORRFHoiUiiKPREJFH+H0nYGnPseZ/VAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[28.   1.2  1.8]\n",
      " [ 0.   5.4  3.6]\n",
      " [ 1.2  3.   3.8]]\n"
     ]
    }
   ],
   "source": [
    "class_names = ['Class 0', 'Class 1', 'Class 2']\n",
    "total_cm = np.zeros((len(class_names), len(class_names)))\n",
    "\n",
    "# Loop over the classification reports and confusion matrices and add them to the running totals\n",
    "for cm in confusion_matrices:\n",
    "    # Add the confusion matrix to the running total\n",
    "    total_cm += cm\n",
    "avg_cm = total_cm / len(confusion_matrices)\n",
    "\n",
    "print(avg_cm)\n",
    "print(type(avg_cm))\n",
    "\n",
    "# Create confusion matrix\n",
    "plt.figure(figsize=(5,5))\n",
    "class_names = ['Normal', 'Benign', 'Malignant']\n",
    "\n",
    "sns.heatmap(avg_cm, annot=True, fmt='g', cmap='Blues', square=True, cbar=False,xticklabels=class_names, yticklabels=class_names)\n",
    "plt.xlabel('Predicted classes')\n",
    "plt.ylabel('True classes')\n",
    "plt.show()\n",
    "\n",
    "## save avg confusion matrix\n",
    "\n",
    "# Save the matrix to a file\n",
    "np.save('avg_cm.npy', avg_cm)\n",
    "\n",
    "# Load the matrix from the file\n",
    "avg_cm = np.load('avg_cm.npy')\n",
    "print(avg_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8a03765",
   "metadata": {},
   "source": [
    "## Calculating some metrices based on the averaged confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "05b2d956",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[28.   1.2  1.8]\n",
      " [ 0.   5.4  3.6]\n",
      " [ 1.2  3.   3.8]]\n",
      "Overall Accuracy: 0.775\n",
      "Balanced Accuracy: 0.659\n",
      "Class 0:\n",
      "\tPrecision: 0.959\n",
      "\tRecall: 0.903\n",
      "\tF1-score: 0.930\n",
      "Class 1:\n",
      "\tPrecision: 0.562\n",
      "\tRecall: 0.600\n",
      "\tF1-score: 0.581\n",
      "Class 2:\n",
      "\tPrecision: 0.413\n",
      "\tRecall: 0.475\n",
      "\tF1-score: 0.442\n",
      "Macro Precision, 0.645 \n",
      "Macro Recall, 0.659 \n",
      "Macro F1-score, 0.651 \n"
     ]
    }
   ],
   "source": [
    "## loading confusion matrix \n",
    "conf_mat = np.load('avg_cm.npy')\n",
    "print(conf_mat)\n",
    "\n",
    "# Calculate total number of predictions\n",
    "total_predictions = np.sum(conf_mat)\n",
    "\n",
    "# Calculate number of correct predictions\n",
    "correct_predictions = np.sum(np.diag(conf_mat))\n",
    "\n",
    "# Calculate overall accuracy\n",
    "accuracy = correct_predictions / total_predictions\n",
    "print(\"Overall Accuracy: {:.3f}\".format(accuracy))\n",
    "\n",
    "# Calculate balanced accuracy\n",
    "class_counts = np.sum(conf_mat, axis=1) ## true instances per class (row marginal - sum over cols)\n",
    "class_correct_predictions = np.diag(conf_mat) ## true positives (sum over diagoal)\n",
    "class_accuracy = class_correct_predictions / class_counts.flatten() ## calculate accuracy per class = tp/tp + fn\n",
    "balanced_accuracy = np.mean(class_accuracy) ## averaging over all class_accuracies = balanced accuracy\n",
    "print(\"Balanced Accuracy: {:.3f}\".format(balanced_accuracy))\n",
    "\n",
    "# Compute metrics for each class\n",
    "metrics_per_class = {} ## dict to store metrices\n",
    "for i in range(len(conf_mat)):\n",
    "    tp = conf_mat[i, i] # true positive\n",
    "    fp = np.sum(conf_mat[:, i]) - tp # false positive (predicted class i but not class i)\n",
    "    fn = np.sum(conf_mat[i, :]) - tp # false negatives (not predicted class i but class i)\n",
    "    \n",
    "    precision = tp / (tp + fp) ## precision of class i = tp / (tp + fp) -> correctly classified out of all predicted\n",
    "    recall = tp / (tp + fn) ## recall of class i = tp / (tp + fn)  -> correctly classified out of all instances of class i\n",
    "    f1_score = 2 * (precision * recall) / (precision + recall) ## f1-score = harmonic mean of precision and recall \n",
    "    ## store metrices per class\n",
    "    metrics_per_class[f'Class {i}'] = {\n",
    "        'Precision': precision,\n",
    "        'Recall': recall,\n",
    "        'F1-score': f1_score\n",
    "    }\n",
    "\n",
    "## define dict to store macro averages\n",
    "macro_metrics = {}\n",
    "\n",
    "# loop over classes\n",
    "for class_name, metrics in metrics_per_class.items():\n",
    "    print(f\"{class_name}:\")\n",
    "    for metric_name, metric_value in metrics.items(): ## loop over metrices \n",
    "        print(f\"\\t{metric_name}: {metric_value:.3f}\") ## print metrics for each class\n",
    "        macro_metrics[metric_name] = macro_metrics.get(metric_name, 0) + metric_value ## add class-wise metrics to total count\n",
    "\n",
    "## loop over metrices and print macro average\n",
    "for k, v in macro_metrics.items():\n",
    "    macro_metrics[k] = v / len(conf_mat[0])\n",
    "    print( f\"Macro {k}, {round(macro_metrics[k],3)} \")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d68d4a81",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 1. parameter tuning - balanced accuracy - for class_weights (done)\n",
    "## 2. average confusion matrix over min of 10 runs (done)\n",
    "## 3. shap values (to do) - done? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a1ecd7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## what I want for avg: \n",
    "## - macro-average precision\n",
    "## - macro-average recall\n",
    "## - macro-average f1-score\n",
    "## - overall accuracy\n",
    "## - balanced accuracy\n",
    "## precision, recall, f1-score per class\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe6e2285",
   "metadata": {},
   "source": [
    "## Generate confusion matrix of first model and plot history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "2129079e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 1s 338ms/step\n",
      "[[29  1  1]\n",
      " [ 0  6  3]\n",
      " [ 1  4  3]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAABEGUlEQVR4nO3dd3xUVdrA8d+TThqEhN4SihTpRECxgKKC2BvounZZe9l131fdXde2++6urrq6roqKHbFgQRcrogiKUkNHILQ0CAHS+5z3jzOJQ0iZTGYyQ+b5fj75zMy9d+6cDOE+97TniDEGpZRSwSvE3wVQSinlXxoIlFIqyGkgUEqpIKeBQCmlgpwGAqWUCnIaCJRSKshpIFBBQUSSRcSISJgbx14tIktao1xKBQINBCrgiMhOEakQkaQ621c7L+bJfiqaa1liRaRIRD71d1mUaikNBCpQ7QAuq3khIsOAaP8V5wgXAeXA6SLStTU/2J1ajVLNoYFABarXgStdXl8FvOZ6gIi0F5HXRCRXRHaJyB9FJMS5L1REHhOR/SKSDkyr570viUi2iGSKyCMiEtqM8l0FPAesBa6oc+4TReR7ETkkIntE5Grn9nYi8k9nWfNFZIlz20QRyahzjp0iMtn5/AEReU9E3hCRAuBqERkrIj84PyNbRP4tIhEu7z9WRL4UkQMisldE7hORriJSIiKJLseNdn5/4c343VUbo4FABaplQLyIDHZeoGcAb9Q55mmgPdAXOAUbOK5x7rsBOBsYBaQCF9d57ytAFdDfecwZwPXuFExE+gATgTedP1fW2feps2ydgJHAGufux4AxwAlAR+B/AIc7nwmcB7wHdHB+ZjVwF5AEHA+cBtzsLEMc8BXwGdDd+TsuNMbkAN8Al7qc99fAXGNMpZvlUG2RMUZ/9CegfoCdwGTgj8D/AVOAL4EwwADJQChQAQxxed9vgG+cz78GbnTZd4bzvWFAF2yzTjuX/ZcBi5zPrwaWNFK+PwJrnM97YC/Ko5yv7wU+qOc9IUApMKKefROBjPq+A+fzB4DFTXxnd9Z8rvN3Wd3AcdOBpc7noUAOMNbf/+b6498fbWtUgex1YDGQQp1mIeydcDiwy2XbLuyFGeyd8J46+2r0cb43W0RqtoXUOb4xVwIvABhjMkXkW2xT0WqgF7C9nvckAVEN7HPHYWUTkWOAx7G1nWhsgFvp3N1QGQA+Ap4TkRRgIJBvjPnJwzKpNkKbhlTAMsbswnYanwW8X2f3fqASe1Gv0RvIdD7Pxl4QXffV2IOtESQZYzo4f+KNMcc2VSYROQEYANwrIjkikgOMAy53duLuAfrV89b9QFkD+4px6Qh3NoV1qnNM3TTBzwKbgQHGmHjgPqAmqu3BNpcdwRhTBryD7df4NTbYqiCngUAFuuuAU40xxa4bjTHV2AvaX0Qkztk2/1t+6Ud4B7hdRHqKSAJwj8t7s4EvgH+KSLyIhIhIPxE5xY3yXIVtphqCbf8fCQwF2gFTse33k0XkUhEJE5FEERlpjHEAs4HHRaS7szP7eBGJBH4GokRkmrPT9o9AZBPliAMKgCIRGQTc5LLvE6CbiNwpIpHO72ecy/7XsM1f56KBQKGBQAU4Y8x2Y8yKBnbfhr2bTgeWAHOwF1uwTTefA2nAKo6sUVwJRAAbgYPYjthujZVFRKKwHa1PG2NyXH52YC+oVxljdmNrML8DDmA7ikc4T3E3sA5Y7tz3dyDEGJOP7eh9EVujKQYOG0VUj7uBy4FC5+/6ds0OY0whcDpwDrYPYCswyWX/Umwn9SpnrUsFOTFGF6ZRKtiIyNfAHGPMi/4ui/I/DQRKBRkROQ7bvNXLWXtQQU6bhpQKIiLyKnaOwZ0aBFQNrREopVSQ0xqBUkoFuaNuQllSUpJJTk72dzGUUuqosnLlyv3GmLrzU4CjMBAkJyezYkVDowmVUkrVR0QaHCqsTUNKKRXkNBAopVSQ00CglFJB7qjrI6hPZWUlGRkZlJWV+bsobUJUVBQ9e/YkPFzXKlEqGLSJQJCRkUFcXBzJycm4pBVWHjDGkJeXR0ZGBikpKf4ujlKqFbSJpqGysjISExM1CHiBiJCYmKi1K6WCSJsIBIAGAS/S71Kp4NJmAoFSSrXI1q9g1/f+LoVfaCDwgkOHDvGf//yn2e8766yzOHTokPcLpJRqnh+fhzcvglfPgbXv+rs0rU4DgRc0FAiqqqoafd+CBQvo0KGDj0qllGqSMfDN3+HT/4FBZ0Pv4+H9G2B5cC3T0CZGDfnbPffcw/bt2xk5ciTh4eFERUWRkJDA5s2b+fnnnzn//PPZs2cPZWVl3HHHHcycORP4JV1GUVERU6dO5cQTT+T777+nR48efPTRR7Rr187Pv5lSbZjDAV/8AZb9B0ZcDuc+DY5KePca+O/voPQQnPQ7CII+szYXCB78eAMbswq8es4h3eP58zkNr2v+t7/9jfXr17NmzRq++eYbpk2bxvr162uHX86ePZuOHTtSWlrKcccdx0UXXURiYuJh59i6dStvvfUWL7zwApdeeinz5s3jiiuu8OrvoZRyqq6Cj2+HNW/CuJvgzL9CSAiEhsH01+HDm+Hrh6HsEJz+cJsPBm0uEASCsWPHHjYG/6mnnuKDDz4AYM+ePWzduvWIQJCSksLIkSMBGDNmDDt37myt4ioVXKrK4b1rYfMnMPE+OOV/Dr/Qh4bDBc9DVHv4/mkoy4ezn4SQUL8V2dfaXCBo7M69tcTExNQ+/+abb/jqq6/44YcfiI6OZuLEifWO0Y+MjKx9HhoaSmlpaauUVQWoylLYuxF6jvF3SdqW8iJ4+1eQ/g1M+TuMv7H+40JC4KxHoV0HWPwolBXAhbMgLLL+449yPu0sFpEpIrJFRLaJyD317O8tIotEZLWIrBWRs3xZHl+Ji4ujsLD+Vf/y8/NJSEggOjqazZs3s2zZslYunToqLXkSXjwVNi/wd0najpID8Pr5sOM7OP+5hoNADRE49Y9wxl9g44fw1gyoKG6NkrY6n9UIRCQUeAY4HcgAlovIfGPMRpfD/gi8Y4x5VkSGAAuAZF+VyVcSExOZMGECQ4cOpV27dnTp0qV235QpU3juuecYPHgwAwcOZPz48X4sqToqGANr59rn82+FHt9DXFf/luloV5gDr18Aedvg0tdg8Nnuv/eEW20z0ce323Nc/o6tKbQhvmwaGgtsM8akA4jIXOA8wDUQGCDe+bw9kOXD8vjUnDlz6t0eGRnJp59+Wu++mn6ApKQk1q9fX7v97rvv9nr51FEkYwUc3AkT7rTj2z+4Ea543zZXqOY7uBNeOx+K9sGv3oW+E5t/jtG/hsg4mHc9vHI2/Pp9iO3s5YL6jy//snoAe1xeZzi3uXoAuEJEMrC1gdvqO5GIzBSRFSKyIjc31xdlVSpwrHsHwqLs0MUpf4X0RXaIo2q+fZvgpTPt6J+rPvYsCNQ49ny4/G04sB1mnwmHdnupkO4xxmCM8cm5/X2LcRnwijGmJ3AW8LqIHFEmY8wsY0yqMSa1U6d6l9xUqm2oroT182DgVIiKhzHXwMBpsPBByF7r79IdXTJXwstT7fOrF3in473/afDrD6EkzwaY3C0tP2cTKqsdfLg6k7OeWsLirft98hm+DASZQC+X1z2d21xdB7wDYIz5AYgCknxYptbhqLbtvIHK0fiMZ+VH2xfZi8ywS+1rETvRqV1H2yxRUeLf8h0tdiyGV8+1bfvXfgZdhnjv3L3H2cDiqLKBJmu1987toqi8ihe/S+eUfyzizrfXUFFVfVTWCJYDA0QkRUQigBnA/DrH7AZOAxCRwdhAcHS3/RgH7NsIxQH6a5QXQc46KNzr75Ko+qx9G9olQP/Jv2yLSYQLnoP9W+CLP/qvbEeDrDXw7tXw2nnQoTdc+zl09MG6Gl2H2gATHgMvTrYT0LxUO9hXUMbfP9vM8f+3kEf+u4meHaN56apUvrzrFCYO9E2/hM86i40xVSJyK/A5EArMNsZsEJGHgBXGmPnA74AXROQubMfx1cZXIa+1VJXZO4Wqcn+XpH5VzjkMhVlgqiGuW5ufNXnUKC+CLQtgxAwIizh8X79JcMJtdoJT/8kw6Kgcae0bxsCOb+2Q2/RFEBkPJ9wOJ95pg6qvJPaD67+CJY/DylftLOWB0+DEu6DXcc0+3da9hbzwXTofrs6iyuFgytCu3HBSX0b19uHv4OTTCWXGmAXYTmDXbfe7PN8ITPBlGVpdpXMimKPSv+VoSE2zUHQiFO21zVjte2owCASb/wuVJb80C9V16p/sRCgdUmo5qmHTx7D0Sds8E9sFJj8IqdfYJqGWnNphKKqooqisiqLyKgprHyuP2CZcxYCTZjAu9z16bX2T0C3/hT4TbEDoP7nR/1vGGH7ccYBZi9P5evM+IsNCmH5cL64/KYU+iTENvs/b2tzMYr+rDQQNt8PHxsZSVFREVlYWt99+O++9994Rx0ycOJHHHnuM1NTUBs/z5JNPMnPmTKKjowGb1nrOnDmNZzStroSQMGjfy06ZL9pn/0Ml9IYj++lVa1r7NrTvDb3G1b8/LBIumg3PnxzcQ0qryiHtLVj6lB3B07EvnPMvGD4DwqM8OuW+wjIenL+RlbsOUlRuL/DuiIkIpdoYyiodwAlEM5rLwhYxc/endHnzYrKj+rGp73VUDT6PPp3a07tjNO0iQql2GD5bn8OsxdtJy8inY0wEd04ewK/H9yExtvVnL2sg8Laazrzqpv+QunfvXm8QcNeTTz7JFVdcURsIFixwYxZqdSWEhNu7lPgeIGG2mehANSQkt+l8KgGtaJ9t1jjxrsYv7p2OsUNKP7nLDik94dbWK6O/lRXAitmw7FkoyoFuI+GSV2HwOS36u/1sfTb3vr+Okopqpg3vRkJ0BLGRYcRFhREbGUas8zEuKoy4qPDabTERYYSGCMYYcgvL2ZlXwq68YnblDeX/8q4kOWsB5xS9y6kb72PP+ieYVT2Nd6tPoX18PIKQU1BGn8RoHj5/KBeP7km7CP/939NA4AX33HMPvXr14pabb4aqUh7453OEhYWzaMUmDh48SGVlJY888gjnnXfeYe/buXMnZ599NuvXr6e0tJRrrrmGtLQ0Bg0adFiuoZtuuonly5dTWlrKxRdfzIMPPshTTz1FVlYWkyZNIikpiUWLFtWmtU5KSuLxxx9n9uzZAFx//fXceeed7Ny5k6lnnM6J41L5ftX6X9Jdtw+F/D2/3F2F6J9Fq1v/vh1o0FCzkKsx19jVtBY+CCknQ7fhvi+fPxXuhR+fheWzoTzfzgW48HlIOaVFTZoFZZU8MH8D76/KZFiP9jwxfST9O8c2+zwiQuf4KDrHRzE2paPLnrHguJ+idZ/QcemTPLzvFe6N/ohvO1zI5+3O4cxzRnPGsV0JDfF/s2zb+x//6T12VIw3dR0GU//W4O7p06dz5513csvM68A4eOfjr/j8zX9z+32PEN++A/v372f8+PGce+65Da4H/OyzzxIdHc2mTZtYu3Yto0ePrt33l7/8hY4dO1JdXc1pp53G2rVruf3223n88cdZtGgRSUmHj7hduXIlL7/8Mj/++CPGGMaNG8cpp5xCQkICW9N38dZL/+aFk6Yenu5aQuwEmbxt0LFfy76vqnLbzPHDfyCpP1z4AoS3gbUV8jPgnSvhmCk2Y6U3rX0bug6HzoOaPrZmSOmzJ9ghpTO/gYho75YnEBxIt53jq9+0fW6Dz7UdwN1HtfjUP2zP4+5308gpKOP20wZw26n9CQ/1QTNbSAixI86FEefCrh+IXvIEU7e+xFR5GTLC4MNmnm/qP2wfiJe1vUDgB6NGjWLfvn1k7d5ObvrPJCQk0LVzInfddy+Lv1tKSEgImZmZ7N27l65d6+/gW7x4MbfffjsAw4cPZ/jwX+7y3nnnHWbNmkVVVRXZ2dls3LjxsP11LVmyhAsuuKA2C+qFF17Id999x7nnnENKr+6MHDESqJPuOrqjrV4f2AF5W22/QXOVFcDKV2yTRWE2dBoEmz6BNy6Gy96yE6SOVnnb7ZDE/D32RmP4dEjo451z798GWavgjEfcf0/NkNLXz7dDSs9+3DtlCQRZa2wH8MaPbO105OV2FFBiC29QgLLKah77fAsvLd1BcmIM7914fKuMygGgz/H2Z+8G2PChZwNKug7zerGgLQaCRu7cfemSSy7hvffmkZOxi+mXXsyb739K7r5cVq5cSXh4OMnJyfWmn27Kjh07eOyxx1i+fDkJCQlcffXVHp0HAEclkZERNt869aS7jmoPif3tnVhRrr1AJfVv+rxF+2y77fKXbNU95RQ4/1lbhV8/Dz74jV0L9or37QXsaJOzziYbMwYum2vHqS/6q22e8IZ17wACQy9q3vva0pBSY+wksCVPHD4EdPxNXhsdtT4zn9++s4af9xbx6/F9uPesQURH+OES2OVY+xNAgnDIgW9Mnz6dufM+5L0FC7nkoovJLyyic1Ii4eHhLFq0iF27djX6/pNPPrk2cd369etZu9amEygoKCAmJob27duzd+/ewxLYNZT++qSTTuLDDz+kpKSE4uJiPvjgA0466aRfOrCdgaBekbE2GBgDL09pPK3BgXTbafnEUPsfuN9EuGERXDXfXqREYNjFMGMO5G62szDz604uD3C7l8HL0yA00k4gGjgVxs60TTk565t+f1OMgbXv2Lb++O7Nf/+pf7J3ifNvtRk2jzaOanvn/8IkeO1cOxlz8oNw13o4/UGvBIFqh+GZRdu44D9LOVRSySvXHMfD5w/1TxAIUBoIvOTYIUMoLCykR/dudOvRk19dOJUVq1YxbNgwXnvtNQYNarzt96abbqKoqIjBgwdz//33M2aMzYsyYsQIRo0axaBBg7j88suZMOGXaRczZ85kypQpTJo06bBzjR49mquvvpqxY8cybtw4rr/+ekaNGvXLkNaQRgIB2Pbm2M724vfK2fZi6Co7za7r+vQYWP2GnQB16wqb3rfH6CPPd8yZtjZQkAWzp9hmlqPBtq9s1srYTjYIJA2w20+8yzZzLXyo5Z+RuRIO7oDhbnQS16dmSGlFiR1S6nC0vEytoarcTsL693G236VmFbA71tp+gBbOA6ixK6+YS5//gUc/38IZQ7ry+Z0n+2x27tFMjraJvKmpqWbFihWHbdu0aRODBw/2U4mcqsrt3Uz7XjZXec46OzwzkFLVFufaDs8uQxuvFeD8TrvF2jbo/EyY/oad7brkCdj+NUTEwXHXwvib3b9ry1oNb1wEEmrT+PqovdMrNnwA826wnbdXfGCDgavvHrejdq75FPqc4PnnLPg9rHoN7v65ZRe/FbNt7eyMvwT2kNKyAlj5sh1IUDME9MQ7bUewF4cuG2OYu3wPD3+ykdAQ4eHzhnLeyO4NDtYIBiKy0hhT78QkrRt5S81EsvB29kKHBN7s4pqmIXeHh3boBdd8Bm9cAG86269jOsNpf4bUa5u/OEf3UfZ8r58Pr0yDy9+1CbxaoiwfVrwMu5bC0Ith6IVNBrkmrXwVPrkTeo61aYfr+z3H3Qg/zYIv/wzXfeHZMMbqSjts9JgpLb8Ddh1SKgKjr7LNfIGibj9S34m2s7vvRK/Pal+Xkc9jX2zh259zOaFfIo9dMoLuHdrAqDUf0kDgLZUlgEBYO/uHHRLm1qSyVuVwmUzmrthOcNUn8NUDdrz6iMs9nr0J2AlR135mR+C8fr6tafQ/rfnnKdxrRyetmA3lBRDbFbZ+AV8/bDtQR10BER5M0V/6FHz5J9v5eunrDQ/LjIiGiffAx3fY/ECDpjX/s7YvgpL9njcLuaoZUvruVfD5ffDtP2Dcb2Dsb/zbQe86BLS6Aoaca5vWvDAE1JUxhh+25/Hst9v5but+4iLDuP/sIVx9QjIhATBOP9C1maahQYMG+bfal7fNXvhrxoHv22zvTL0w5M1r8rbZzrlOAxs9zBjD5s2bfdvcVrQPXr/QdiJf9KJd9MMdedvh+6dgzVuHjy3vOsIGgiVPwJ5lNm3zuBth7A12aGxTjLFB5Lt/wrEXwAWzjkz8Vld1FfxnvG3SuOn75jdtzLve9kP87uemP6s59vxkE7Bt+a+9MRl9pW0u6tDbe5/RlOw0W4aNH3p9CKgrh8Pwxca9PPvtdtL2HCIpNpLrT0rh8nG9iY9qYc2wjWnzTUNRUVHk5eWRmJjon2BgjG0ainQZJx8aFnh5/6urILTxC44xhry8PKKiWnDX747YznD1JzBnOrx3jb2rH31lw8dnrbYXlk3znReWX9k7f9cLy8Ap9mfXD3Yc+jd/tY+jr4Ljb7FNXfVxOGDB3bDiJXvs2U+4d1EPDYPT/mQ7O9PesrUQd5UX2SRzw6d7NwgA9BoLl82xNyPfP2V/r+Uv2hFcE+7w3dDFmiGgS5+0/Ug+GAJao2axlue+3c723GJ6d4zmkfOHcvGYnkSFa5qU5moTNYLKykoyMjI8H1/fUo5qKMi0KW8j4+y2kjzbgezJkEBfyc+EiHb2brkRUVFR9OzZk/DwVrijqiiGt38N2xfaCVUnuKxWWpte+AmbdTMy3vZNuHth2bsRlv4L1jvzOQ271F4IXWfvVlfChzfBunftvskPNq/pzBh48TQ7dPO2le7PoE57Gz6YaftM+hzv/ud5Ij/Dds6ufAUqi2HAmbZ5xluf66iGzZ/Yf6es1bYf6fib7b+Vl0b/1CipqOLt5Xt4YXE6WfllDOoax00T+zFtWDfCfDEzuA1prEbQJgKB3235DN6abhfB6D3ebvv8D7Zj7A/ZgZHiuaocHukMk/4Ip/ze36U5XFUFvH+DbUY46W6YdJ9NL7zkCcheY9MLj7/Z8/TCh3bDD8/Y0TmVJXDMVHsh7DbcTg77+TPbAX7Sbz0r/47FdsLc6Q/DhNvde88bF0Huz3BHWutlEC05YGsGPz5nb1R6jYMJd9rOak/KUFUOaXNtrSNvGySk2GA64rIG+5GMMR7V2g+VVPDaD7t4eekODpZUMja5IzdN7MfEgZ2CeiRQc7T5piG/y04DxA7LrBHTCapK7R1vIIzeKHKuSBaIOezDIuDi2fBxHHz3mL1zLdnvlfTCgG0bn/p3OPl/7Eifn56H2WdAdJK9IE77Jxx3vefnTzkZ+p1m+xdGX9n0aKqifbbpZMKdrZtGOrqjzZF0/K12/sf3T8Pcy2wqkB7NXM/XOGxnd1EOdBsBl7xS7xDQ/NJKlqXn8f22/SzZtp/0/cVEhoUQHRFGu/BQ2kWEHvYYHVHneXgoh0ormbcyg+KKak4b1JmbJvYjNdmNfh/lNg0E3pCdZicbuV7wY5yJ4Er2B0YgqJl1GtfNv+VoSEioHfUS28UOBR33zxanFz5CTCJMutfeta96HdLm2AAx7OKWn3vyn+06AUv/ZZ83pibT6PDpLf9cT0REw7iZtoa14QNbQ9ixuPnn6TLkiCGgZZXVrNx1kKXb9rN0237WZebjMNAuPJRxfTsyZWhXqqoNJRXVlFRUU1ZZTUlFFSUV1RwqqSDrkOv2akorqwkNEc4Z3o0bJ/ZjUNejOF9VANNA4A3ZaUe2t8Y4JyAV77d5/v2tMNs+BmKNoIaI7Xz1tYgYGH+j/fGWbiPsPIZlz9oUFPGNBNx179jJdO5kGvWl0HA7dLUFw1erHYZ1Gfks3baf77fvZ/nOg1RUOQgLEUb26sBtpw5gQv8kRvbqQERY82s/Doeh2hjfZAZVtTQQtFTxfijIsBcCVzU1gkBZxD7QawRtwal/sP0c3/4dznmy/mPyttu0Es3JNBpgHA7DJ+uy+SQti2XpeRSU2dFxg7rG8evxfZjQP5GxKYnERrb88hISIoSgfQC+poGgpbLT7OMRgaCmRhAggaAgy04mc2dMvfJMx752pMzyl2w7fH2ZW9d6mGk0ABhjWLhpH499sYXNOYX06NCOqUO7MWFAEif0SyTJD0ssKu/waSAQkSnAv4BQ4EVjzN/q7H8CqMmYFg10NsZ08GWZvK4mENTNmxMdgDWCuG6BMYKpLTv593YW7dcPw6WvHr7PGJu1NOWkwBpW7IYftufx6OebWbX7EMmJ0Tx12SjOHtZNZ+22ET4LBCISCjwDnA5kAMtFZL4xZmPNMcaYu1yOvw3w7rzz1pCzFjr0sXMIXEVEQ0SsbToKBIXZgd0/0FbEdrazeL/9O2SuOjwba02m0ZPv9l/5mmldRj7/+Hwz323dT9f4KP56wTAuSe2pbfZtjC9rBGOBbcaYdAARmQucB2xs4PjLgCaGWwSg7LQjm4VqxCQFVo2gidQSykuOv9WO1//qAbs2Q42179jU3oPP8VvR3LVtXxGPf7mFBetySIgO5w9nDebXx/fRWbttlC8DQQ9gj8vrDKDeVJMi0gdIAb5uYP9MYCZA796tmC+lKWX5NqnWyF/Vvz+mU2AFgn6Tmj5OtVxUvG0i+uweO1+g36nOTKPzbAoML8+29abMQ6U8+eXPzFuVQbvwUG4/bQA3nJRCnObtadMCpbN4BvCeMabehXKNMbOAWWBnFrdmwRqVs84+dhtZ//6YTnZWq79VFNvUv9o01HpSr7XZUb/8M6RMtCkySvb7b+5AE/YXlfPMom28uWw3CFwzIYWbJ/YjUTuAg4IvA0Em4Jrlq6dzW31mALf4sCy+UTtiqIGF5GOSbLuwv+nQ0dYXFgmT/mDXa97wvk1jEdUB+p/us4+sqHKQvr+I4vIqyqscVNT8VNvHuttqXueXVvLRmkzKKqu5ZEwv7pg8QPP3BxlfBoLlwAARScEGgBnA5XUPEpFBQALwgw/L4hvZafbi2tAqZDGdbGexw9G6qQTqqg0EWiNoVcMusesbfP2wTSvhxUyj1Q5Dem4RaRn5rM04RNqeQ2zKLqSiunlLVUaEhhAZFsKkQZ357enH0K9TAMyCV63OZ4HAGFMlIrcCn2OHj842xmwQkYeAFcaYml60GcBcc7RlvwO7sHtDHcVgA4GphrJD/h2/XzurWGsErSok1KabmOOcuevhDF5jDBkHS1mbkU+a86K/PjOf4grbkhobGcbQHvFcMyGZY3u0p327cCJCQ4gIsxf5iLCQI187t2nCNgU+7iMwxiwAFtTZdn+d1w/4sgw+U1EC+7fYFZca4jqpLCACgdYIWt2AM6jsNYGqg3tYVtoXx6a9VDkMDoehymGodv0x5rB9+SUVrM3MZ21GPgeKKwB7Bz+4ezwXjenJ8J4dGNmrPX2TYnU8v2qRQOksPvrs3WAThzVaI3CZVObPoZuFORAeffjCOcrnKqsdvPr9Tmbtnkl1eQl5rzSvvyhEYEDnOE4b1JnhvTowomd7BnWN9yhnj1KN0UDgqew19rGppiHw/xDSmslk2gzQan7Ynsef56/n571FnHJMd35zcl+iIkIJCxFCnT9hIUKICGEhIYSEcNhjqAiR4SE6bl+1Cg0EnspOg+hEiO/R8DGuGUj9qSa9hPK57PxS/rpgMx+nZdEzoR0vXJnK5MGdtS1eBTQNBJ6qmVHc2H/wdh0BCYBAkA3dRzd9nPJYRZWDl5bs4Omvt1LtMNw5eQA3ntJP7+jVUUEDgSeqKmDfJrsgemNCw2wOIn82DRnjrBFoR7GvLP45lwfmbyB9fzGnD+nC/WcPoVfHaH8XSym3aSDwRO4mcFQ23j9Qw99pJsoL7Dq92jTkdRkHS3jkk018tiGH5MRoXr7mOCYNbGBOiVIBTAOBJxpag6A+NZPK/EUnk9WrrLKa91ZmYIyhY0wkHWMiSIqNoGNMBB2iIwhtZDhmWWU1sxan859vtiEIvz9zINeflEJkmDYDqaOTBgJPZKfZoZgJKU0fG5Nkh5r6i04mO8L23CJunbOaTdkF9e4PEegQHUFijA0MibERJDqDRWxkGG/8uItdeSWcNawrf5g2hB6ajkEd5TQQeCI7DboOdy9thL+bhgp0MlkNYwzzVmVy/0friQwL4cUrUxneqz0Hiis4UFTB/uIKDhSVc6C45nkFB4or2JxTyIHiPA6VVALQr1MMb1w3jhMHJPn5N1LKOzQQNFd1FeSst9kl3RHTyaaYqKrwWp6ZZtFZxQAUlVfxpw/X88HqTMaldORfM0bRtX0UAJ3jotw6R1W1g0OllSQ00XSk1NFGA0Fz5W2FqtKGM47WVTO7uCQP4v3QPFOYA5HtISKm9T87QKzLyOe2t1ax+0AJd00+hltP7e/RhTwsNETX5VVtkgaC5mpORzEcPrvYL4EgeJeoNMYwe+lO/vbpJpJiI3nrhvGM65vo72IpFXA0EDRX9loIaweJA9w73t9pJgpz/BOA/OxAcQW/fzeNhZv3MXlwFx69eDgJMX5omlPqKKCBoLmy06DrUDtZzB3+TjNRmAPJE/zz2R4or6qmoLSKpNgIj9MyLEvP4465qzlYXMkD5wzhqhOSNcWDUo3QQNAcDgfkrG1eXnnXDKStzZijpmloc04Bby/fwwerMzlUUklCdDiDusYzqFscg7vGM7hbPAO6xDaasqHaYXhq4Vae/noryYkxvHTVcQztEbjrAysVKDQQNMfBHXamrrv9A2AXKg8J908gKDlgZ0AH6ByCwrJK5qdl8c7yPaRl5BMRGsLpx3ZhVK8ObM8tYlN2IXN/2kNppV2AJUQgJSmGQd3iGdw1rjZQ9OjQjpyCMu6Yu4afdhzgwtE9ePi8ocRE6p+3Uu7Q/ynN0dyOYrBJ6fw1u7gwyz4GUI3AGMPynQd5e/ke/rsui7JKBwO7xHH/2UO4YFSPI9rxHQ7DrgMlbM4uYFNOIZuzC1iXkc9/12bXHhMXGYYBHMbwz0tGcNGYnq38Wyl1dNNA0BzZafbuvtPg5r0vJsk/NYIAWrR+X2EZ81Zm8u6KPaTvLyY2MowLRvVk+nG9GNGzfYNt+CEhQkpSDClJMUwd9svvUVRexZacQjbnFLApu4CC0irunDyAvrrmrlLNpoGgObLToPPg5k8Mi+kEJf6oEfh3MllVtYNvtuTy9oo9fL15H9UOw3HJCdw0sR/ThncjOsLzP7/YyDDG9ElgTJ8EL5ZYqeCkgcBdxtiO4oFnNf+9MZ3sRLTWVlMjiO3Sqh9bVlnNuyv28Ny36WQeKiUpNoLrT0zhktRe9O+sd+xKBRqfBgIRmQL8CwgFXjTG/K2eYy4FHgAMkGaMudyXZfJYQaadHdyc/oEaMUl+6iPItquohbXObNiSiirm/Lib5xenk1tYzujeHfjT2YM5bXAXwkN1nV2lApXPAoGIhALPAKcDGcByEZlvjNnocswA4F5ggjHmoIgEbjL32o7ikc1/b0ySXROgorh1Uz0U5kBcd99/TFklr/2wi5eW7OBAcQXH903kX9NHcny/RB2/r9RRwJc1grHANmNMOoCIzAXOAza6HHMD8Iwx5iCAMWafD8vTMtlpICHQ5djmv9d1dnGrBgLfziE4VFLB7KU7eWXpDgrKqpg4sBO3TupPanJHn32mUsr7fBkIegB7XF5nAOPqHHMMgIgsxTYfPWCM+azuiURkJjAToHfv3j4pbJOy0yDpGIjwYAlC19nFCcleLVajCnOgy1Cvnza3sJwXl6Tzxg+7KK6o5sxju3DrpAEM66mTt5Q6Gvm7szgMGABMBHoCi0VkmDHmkOtBxphZwCyA1NRU08pltLLTIOVkz97rj9nFjmoo2uvVoaPZ+aU8/206b/20m8pqB2cP784tk/ozsGuc1z5DKdX6fBkIMoFeLq97Ore5ygB+NMZUAjtE5GdsYFjuw3I1X9E+28ziSUcx+CfxXNE+MI4WNw1l55eyfOdBvvs5l4/WZFFtDBeM6sHNE/vpmH2l2ghfBoLlwAARScEGgBlA3RFBHwKXAS+LSBK2qSjdh2XyTPZa++hpIIj2Q43AgyUqHQ7Dttwilu88wIqdB1m+8wAZB0sBiIkI5eLUntx0Sj96dfSgeUwpFbB8FgiMMVUicivwObb9f7YxZoOIPASsMMbMd+47Q0Q2AtXA740xeb4qk8ey19jHrsM8e39ENETEtu4QUjcWra+ocrAuM9954T/Ail0Ha5djTIqNZGxKAtdOSOG45I4M7hZHmA4BVapN8mkfgTFmAbCgzrb7XZ4b4LfOn8CVnQYd+9oEcp5q7TQTDdQI8ksqeWnpDpal55G25xDlVQ4A+ibFcOaQrqQmJ3Bcckf6JEbr0E+lgoS/O4uPDtlp0GN0y87R2ovYF+bY4a41/RPA7rwSrnnlJ3bsL2ZYj/ZcMb4PxyV3JDU5QZdgVCqIaSBoSulBOLQLxlzdsvPEdIJDu71SJLcUZtvUEs4FdFbuOsjM11ZQ5TDMuWE843XJRqWUkzb6NqWlHcU1Wr1pKKe2f+DjtCwue2EZcVFhfHDzCRoElFKH0RpBU3K8FQicaxI4HBDSCvG3MAfToSf/WbSNRz/fQmqfBGZdmUpHXbdXKVWHBoKmZKdBfM9fJoV5KqYTmGooOwTRvk/BYAqzWFbZl0fTtnDuiO784+LhjS7zqJQKXhoImpKd1vLaAByeZsLHgSC/sIj2JXl8nx/O7af2567Tj9ERQEqpBjXZRiEi54hIcPYllBfB/q1eCgSNTypb/HMuc3/aTV5ReYs+Zs+BEm56/lMATkkdzm/PGKhBQCnVKHdqBNOBJ0VkHnZS2GYflylw7NsEGM8nkrlqJM3Etz/nct0ry6lyGP7w4XqO75vIWcO6ceaxXUhsxrDO1bsPcsNrKxhUZSeTpQ4d0vJyK6XavCbv9I0xVwCjgO3AKyLyg4jMFJG2n2ks3zncM6FPy8/VQCBYl5HPTW+sZECXOObddDy/ObkvGQdLuO+DdYz960J+9eIy3vxxV5M1hU/XZTNj1jKiI8J49Eznsg4BtGi9UipwudVHYIwpEJH3gHbAncAFwO9F5CljzNM+LJ9/5Ttz5MX3aPm52jn7BVzSTNRM8EqIjuDVa46jc3wUY/p05PdnDmRjdgEL1mWzYF0Of/hgPX/6cD3jnTWFKUO71k4AM8bw/OJ0/vbpZkb37sALV6aSuOEV+wEBsGi9UirwNRkIRORc4BqgP/AaMNYYs09EorGLzLTdQFCQZXMEtSS1RI3QMBsMnDWCvKJyrnr5J6ochrnXjqVzfFTtoSLCsd3bc2z39tx9xkA2ZRc6g0I2f/xwPfd/tJ5xKYlMG96NDVn5vPXTHs4e3o3HLhlhRwYVZkNIuF2mUimlmuBOjeAi4AljzGLXjcaYEhG5zjfFChAFGbY24K3OVmeaiZKKKq59dQVZh0qZc8O4Rhd0FxGGdI9nSPd4fnfGMWzOsUHhv86gAHDLpH787vSBhIQ4y1mYY2sD2kmslHKDO4HgASC75oWItAO6GGN2GmMW+qpgASE/E9p7oVmoRkwnTHEut81ZzbqMQzx7xRjG9HF/KKmIMLhbPIO7xfPb049hy95CSiuqGdU74fADfbxEpVKqbXFnWOi7gMPldbVzW9tXkOmd/gEnE5NE7t5MFm7ex0PnDeXMYz2/WIsIg7rGHxkEAAo0ECil3OdOIAgzxlTUvHA+b/t5Cqoq7Cpf7Xt67ZRpB8OJKMvj1kn9uWK8F0YiNaSmaUgppdzgTiDIdXYYAyAi5wGtuMKKnxRmAQbiu3vldHN/2s3Xuw0dpJjfnZbslXPWq6IYyvO1RqCUcps7geBG4D4R2S0ie4D/BX7j22IFAC8OHV24aS9/+HA9HTvboCIlB1p8zgbVrkymNQKllHua7Cw2xmwHxotIrPN1kc9LFQgKsuxjC5uGVu8+yC1zVjGkWzwzThkN72OHkMb76ELtxhKVSinlyq0JZSIyDTgWiKrJW2OMeciH5fK/ggz72IIaQXpuEde9uoLOcVHMvvo4og6ssjt8uS6BB4vWK6WCmzsTyp4DooFJwIvAxcBPPi6X/+Vn2olkkQ2P8W9MbqGdMAbw6rVj6RQXCeXONBMled4q5ZFqagS+qnEopdocd/oITjDGXAkcNMY8CBwPHOPOyUVkiohsEZFtInJPPfuvFpFcEVnj/Lm+ecX3oYJMuw6BB4rKq7jmlZ/YX1jB7KuPIyUpxu5oIgOpVxRmQ3g0RMb77jOUUm2KO01DZc7HEhHpDuQBTd5uikgo8AxwOpABLBeR+caYjXUOfdsYc2szytw68jM8mkxmjOG2OavYlF3IC1eOYWSvDr/sjGpvUz/4NBA4l6jUWcVKKTe5UyP4WEQ6AI8Cq4CdwBw33jcW2GaMSXfOPZgLnOdhOVtfQaZHQ0e/357Hoi253Dt1EKcO6nL4TpHaNBM+U5it/QNKqWZpNBA4F6RZaIw5ZIyZB/QBBhlj7nfj3D2APS6vM5zb6rpIRNaKyHsi0quBcswUkRUisiI3txUWgK8ste34HjQNPb84naTYyIYnjMUkHZaB1Os0vYRSqpkaDQTGGAe2eafmdbkxJt+Ln/8xkGyMGQ58CbzaQDlmGWNSjTGpnTp18uLHN6B26GjzmoY2ZRew+OdcrpmQ3PD6wL6sERijs4qVUs3mTtPQQhG5SJq/3mEm4HqH39O5rZYxJs8YU7PiyovAmGZ+hm8UeDaZ7IXF6URHhHLFuEbSR8Qk+S4QlBdAZYnWCJRSzeJOIPgNNslcuYgUiEihiBS48b7lwAARSRGRCGAGMN/1ABFxvXU9F9jkZrl9q2ZWcTMmk2UdKmV+WhbTj+tF++jwhg+M6eS7piGdVayU8oA7M4s9WpLSGFMlIrcCnwOh2PWON4jIQ8AKY8x84HZnHqMq4ABwtSef5XW1k8nc7yx+eekODHDdiSmNHxiTZO/aK4ohIsbzMtandjKZ1giUUu5zZ0LZyfVtr7tQTQPHLAAW1Nl2v8vze4F7my5mK8vPtKt7hbdz6/CCskre+mkP04Z1o2dCdOMHu65d7PVAoDUCpVTzuTOP4Pcuz6Oww0JXAqf6pESBoJnrEMz5cTdF5VXMPLlv0wfXBoL9kJDsWfkaojUCpZQH3GkaOsf1tXOI55O+KlBAyM+EDr3dOrSiysHLS3cwoX8iQ3u4sbaxL2cXF2RDZHvv1zSUUm2aO53FdWUAg71dkIBS4P6s4o/WZLK3oJyZJ/dz79yuTUPepnMIlFIecKeP4GnAOF+GACOxM4zbpvIiKMt3q2nIGMML36UzqGscJw9Icu/80T6sEdSkl1BKqWZwp49ghcvzKuAtY8xSH5XH/5qxDsE3W3L5eW8Rj186ArenWUREQ0Ssb4aQFuZAnxO8f16lVJvmTiB4DygzxlSDTSYnItHGmBLfFs1PmrEOwfOLt9OtfRTnjGhmTiJfpJkwRpuGlFIecWtmMeA6jrId8JVvihMAaieTNR4I1mYcYln6Aa6dkEJ4aDO7WnyRZqLkADgqdeioUqrZ3LmCRbkuT+l83sRg+aNYQSYgENf4Xf7zi9OJiwxjxth68+Q1zhezi3XoqFLKQ+4EgmIRGV3zQkTGAKW+K5Kf5WdAbGcIi2jwkN15JXy6LpvLx/cmLqqRdBIN8UW+odqVyZqfOlspFdzc6SO4E3hXRLIAAboC031ZKL9yYx2Cl5akExoiXDuhiXQSDYnpBCX7weGAEE9G8NZDawRKKQ+5M6FsuYgMAgY6N20xxlT6tlh+VJAFif0b3H2wuIJ3VmRw3sgedImP8uwzYjqBowrKDkF0R8/OUVdNIIjt0vhxSilVR5O3oyJyCxBjjFlvjFkPxIrIzb4vmp/kZzY6dPT1Zbsorax2L51EQ1zTTHhLYbbNjxQW6b1zKqWCgjvtEjcYYw7VvDDGHARu8FmJ/KksHyoKGxw6WlZZzavf72TSwE4c08WjpKyWL9JM6II0SikPuRMIQl0XpXEuSt9wT+rRrImho/NWZZBXXOF+OomG+GJ2sc4hUEp5yJ1A8BnwtoicJiKnAW8Bn/q2WH5SuzLZkU1D1Q7Di9/tYHjP9ozv28J2fV/kG9L0EkopD7kzauh/gZnAjc7Xa7Ejh9qefOes4npqBF9u3MuO/cX8+/JR7qeTaEh0on30Vh+BoxqK9mrTkFLKI03WCJwL2P8I7MSuRXAqgbKkpLcVZIKEQOyRcW7W4u306tiOKcd6IQaGhkG7jt6rERTngnFojUAp5ZEGawQicgxwmfNnP/A2gDFmUusUzQ/yM20QCD38a1mx8wCrdh/iwXOPJay56SQa4s00E7VzCHQymVKq+RprGtoMfAecbYzZBiAid7VKqfylILPeZqHnF6fTITqcS1LdX8y+Sd5MM1Ggk8mUUp5r7Pb2QiAbWCQiLzg7ilvYOB7g6lmicntuEV9t2suV4/sQHeFOl4qbvJlmorZGoH0ESqnmazAQGGM+NMbMAAYBi7CpJjqLyLMicoY7JxeRKSKyRUS2icg9jRx3kYgYEUltZvm9x5h6J5O9+F06EaEhXHlCsnc/rybNhDcU5ti+jZrRSEop1QzudBYXG2PmONcu7gmsxo4kapRzvsEzwFRgCHCZiAyp57g44A5sh7T/lB6EqtLDagTZ+aXMW5XJRWN6khTr5Rm7MZ3sZ1Z7IVtHYTbEdD6ib0MppdzRrJ5PY8xBY8wsY8xpbhw+FthmjEk3xlQAc4Hz6jnuYeDvQFlzyuJ1dYaOVjsMv307jVARbmzpBLL61MwuLslr+bl0DoFSqgW8NASmXj2APS6vM5zbajnTW/cyxvy3sROJyEwRWSEiK3JzfbDWLxwxmeyZRdv4IT2PB887lt6JPlh+wZuTyjS9hFKqBXwZCBolIiHA48DvmjrWWQtJNcakdurko3bwmhpBfHd+TM/jya9+5vyR3blkjBdHCrnyaiDQ9BJKKc/5MhBkAq7Ld/V0bqsRBwwFvhGRncB4YL7fOowLsiAkjIPSgTvmrqF3x2geuWBYy2cRN8RbGUirKmyns9YIlFIe8mUgWA4MEJEUEYkAZgDza3YaY/KNMUnGmGRjTDKwDDjXGLPCh2VqWEEmJq4bd89bz4HiCv59+WhiI33Y+eqtDKRFe+1jvAYCpZRnfBYIjDFVwK3A59iUFO8YYzaIyEMicq6vPtdj+ZnslSQWbt7HvWcNYmiP9r79vKj2EBLe8kCgcwiUUi3k0/GGxpgFwII62+5v4NiJvixLU8oP7GZ5fk8mD+7C1d6eM1AfEe+kmdAlKpVSLeS3zuJAUlhajhRmcSi8M49ePNx3/QJ1xSS1vI+gZtF6rREopTwU9IHAGMPf5y0hgipOGjOShJhWXHPHG2kmCrNtE1M7L619rJQKOkEfCN5dkUHaho0AJKcMaN0P90rTkHMyWUjQ/1MqpTwU1FePbfsKuX/+eiZ1c6Z5aGCJSp/xRgZSnUOglGqhoA0EZZXV3PLmamIiwrhhhLM5qJ4lKn0qJgkqS6Ci2PNzaHoJpVQLBW0gePiTjWzZW8g/Lx1BXPleCI38ZWx/a/HG7OLCbO0oVkq1SFAGgv+uzebNH3fzm5P7MnFgZ+c6BN3tkM7WVBsIPEw8V1ECZfkaCJRSLRJ0gWDPgRLueX8tI3t14O4zB9qN9axD0CpaOrtYJ5MppbwgqAJBZbWD295aDcDTl40ivGb94XpWJmsVLW0aqp1DoH0ESinPBdVKJo99sYU1ew7xn1+NpldHZ2ppR7VNONfaI4YAorVGoJTyv6CpEXyzZR/Pf5vOr8b15qxhLhfOon1gqm0fQWuLiIaIWM+HkGqNQCnlBUETCMoqq0ntk8Cfzq6zWmadBWlaXUtmF29faGsVUT5OkKeUatOCpmloytBunHls1yPzCNVZorLVeTq7OP1b2P41nPGX1h/tpJRqU4KmRgDUn0yutkbgz0DQzKYhY+CrB2wt5rjrfVIspVTwCKpAUK/8TAiPhnYJ/vl8T5qGNn4EWatg0n0QHuWbcimlgoYGgoIMWxvwV/NKTCe71KTD4d7x1ZWw8CHoNBhGzPBt2ZRSQUEDQX6m//oHwAYCRxWUHXLv+NWvw4HtMPnPEBLq06IppYKDBoKCTP+NGILmLWJfUQLf/B16jYdjpvi2XEqpoBHcgaC60o7F98ccghrRifbRnX6CH5+Fohw4/UEdKaSU8prgDgSFOYDxf9MQNB0ISg7Akn/BMVOh93jfl0spFTR8GghEZIqIbBGRbSJyTz37bxSRdSKyRkSWiMiQ+s7jM/6eTAbuB4Ilj0N5AZx2v+/LpJQKKj4LBCISCjwDTAWGAJfVc6GfY4wZZowZCfwDeNxX5amXvyeTwS9NQyWNpKLOz4AfZ8GIy6BL68ZKpVTb58sawVhgmzEm3RhTAcwFznM9wBhT4PIyBjA+LM+R/D2ZDCA0zC4831iNYNH/AQYm3dtqxVJKBQ9fppjoAexxeZ0BjKt7kIjcAvwWiABOre9EIjITmAnQu3dv75UwPxMi4yEq3nvn9ERjaSb2bYa0OTDuJujgxd9dKaWc/N5ZbIx5xhjTD/hf4I8NHDPLGJNqjEnt1KmT9z7cX+sQ1NVYmomFD9kMpSf9rnXLpJQKGr4MBJlAL5fXPZ3bGjIXON+H5TlSzRKV/tZQmondP8KW/8KE2yEmsfXLpZQKCr4MBMuBASKSIiIRwAxgvusBIjLA5eU0YKsPy3Mkf88qrlFf01BNYrnYLjD+Zr8USykVHHzWR2CMqRKRW4HPgVBgtjFmg4g8BKwwxswHbhWRyUAlcBC4ylflOUJVORTv8+/Q0RoxnaD0oJ3gFhput239AnZ/D9P+CREx/i2fUqpN8+l6BMaYBcCCOtvud3l+hy8/v1EFWfYxIGoEziUrS/LsamOOavjqQejYF0a3XmxUSgUnv3cW+00gDB2tUXdS2bp3Yd8GOPWPv9QQlFLKR4I3EOQ7A0H7AGkaAhsIqsrh679AtxEw5AL/lkspFRSCZqnKIxQ4ZxUHVI1gPyx/CfJ3w7n/gpDgjdNKqdYTxIEgy65KFhHt75L80kdwIB1+mgV9J0K/eufWKaWU1wVvIMgPkMlkAFHtISQcfnjGJpab/IC/S6SUCiLB2/ZQs0RlIBCxzUPlBXDsBdB9lL9LpJQKIsEbCAJlMlmNmEQICYNT/+TvkiilgkxwNg1VlEDpgcCpEQCMuhIclZDYz98lUUoFmeAMBLWTyQJg6GiNcTP9XQKlVJAKzqahQBo6qpRSfhacgaB2MpkGAqWUCs5AUNM0FBcAKaiVUsrPgjQQZEB0EoRH+bskSinld8EZCAJt6KhSSvlRcAaCgszAWIdAKaUCQHAGAq0RKKVUreALBOWFUJ6vQ0eVUsop+AJBIK1DoJRSASD4AkEgrUymlFIBIIgDgc4hUEop8HEgEJEpIrJFRLaJyD317P+tiGwUkbUislBE+viyPICzaUg0ECillJPPAoGIhALPAFOBIcBlIjKkzmGrgVRjzHDgPeAfvipPrYIMiO2ii8IrpZSTL2sEY4Ftxph0Y0wFMBc4z/UAY8wiY0yJ8+UywPc9uDp0VCmlDuPLQNAD2OPyOsO5rSHXAZ/Wt0NEZorIChFZkZub27JSFQTQEpVKKRUAAqKzWESuAFKBR+vbb4yZZYxJNcakdurUyfMPMsZZI9Cho0opVcOXC9NkAr1cXvd0bjuMiEwG/gCcYowp92F5oCwfKou1RqCUUi58WSNYDgwQkRQRiQBmAPNdDxCRUcDzwLnGmH0+LItVoOsQKKVUXT4LBMaYKuBW4HNgE/COMWaDiDwkIuc6D3sUiAXeFZE1IjK/gdN5R75OJlNKqbp8umaxMWYBsKDOtvtdnk/25ecfQZeoVEqpIwREZ3Gryc8ECYW4rv4uiVJKBYzgCgQFmRDXDUJC/V0SpZQKGMEVCPIztKNYKaXqCK5AUJCl/QNKKVVH8AQCY2zTkNYIlFLqMMETCEoOQFWZ1giUUqqO4AkEOnRUKaXqFTyBIF9nFSulVH2CJxDUrkymCeeUUspV8ASC+O4wcBrEtCB7qVJKtUE+TTERUAZNsz9KKaUOEzw1AqWUUvXSQKCUUkFOA4FSSgU5DQRKKRXkNBAopVSQ00CglFJBTgOBUkoFOQ0ESikV5MQY4+8yNIuI5AK7PHx7ErDfi8U5Wun38Av9Liz9Hqy2/D30McbUm1rhqAsELSEiK4wxqf4uh7/p9/AL/S4s/R6sYP0etGlIKaWCnAYCpZQKcsEWCGb5uwABQr+HX+h3Yen3YAXl9xBUfQRKKaWOFGw1AqWUUnVoIFBKqSAXNIFARKaIyBYR2SYi9/i7PP4iIjtFZJ2IrBGRFf4uT2sRkdkisk9E1rts6ygiX4rIVudjgj/L2Foa+C4eEJFM59/FGhE5y59l9DUR6SUii0Rko4hsEJE7nNuD8m8iKAKBiIQCzwBTgSHAZSIyxL+l8qtJxpiRQTZe+hVgSp1t9wALjTEDgIXO18HgFY78LgCecP5djDTGLGjlMrW2KuB3xpghwHjgFuc1ISj/JoIiEABjgW3GmHRjTAUwFzjPz2VSrcgYsxg4UGfzecCrzuevAue3Zpn8pYHvIqgYY7KNMauczwuBTUAPgvRvIlgCQQ9gj8vrDOe2YGSAL0RkpYjM9Hdh/KyLMSbb+TwH6OLPwgSAW0VkrbPpKCiaRABEJBkYBfxIkP5NBEsgUL840RgzGttMdouInOzvAgUCY8dRB/NY6meBfsBIIBv4p19L00pEJBaYB9xpjClw3RdMfxPBEggygV4ur3s6twUdY0ym83Ef8AG22SxY7RWRbgDOx31+Lo/fGGP2GmOqjTEO4AWC4O9CRMKxQeBNY8z7zs1B+TcRLIFgOTBARFJEJAKYAcz3c5lanYjEiEhczXPgDGB94+9q0+YDVzmfXwV85Mey+FXNxc/pAtr434WICPASsMkY87jLrqD8mwiamcXO4XBPAqHAbGPMX/xbotYnIn2xtQCAMGBOsHwPIvIWMBGbZngv8GfgQ+AdoDc2tfmlxpg234nawHcxEdssZICdwG9c2srbHBE5EfgOWAc4nJvvw/YTBN/fRLAEAqWUUvULlqYhpZRSDdBAoJRSQU4DgVJKBTkNBEopFeQ0ECilVJDTQKBUHSJS7ZKFc403s9WKSLJr1k+lAkGYvwugVAAqNcaM9HchlGotWiNQyk3OtRz+4VzP4ScR6e/cniwiXzsTti0Ukd7O7V1E5AMRSXP+nOA8VaiIvODMg/+FiLTz2y+lFBoIlKpPuzpNQ9Nd9uUbY4YB/8bOVAd4GnjVGDMceBN4yrn9KeBbY8wIYDSwwbl9APCMMeZY4BBwkU9/G6WaoDOLlapDRIqMMbH1bN8JnGqMSXcmLMsxxiSKyH6gmzGm0rk92xiTJCK5QE9jTLnLOZKBL50LnyAi/wuEG2MeaYVfTal6aY1AqeYxDTxvjnKX59VoX53yMw0ESjXPdJfHH5zPv8dmtAX4FTaZGdilDm8Cu1yqiLRvrUIq1Rx6J6LUkdqJyBqX158ZY2qGkCaIyFrsXf1lzm23AS+LyO+BXOAa5/Y7gFkich32zv8m7KIvSgUU7SNQyk3OPoJUY8x+f5dFKW/SpiGllApyWiNQSqkgpzUCpZQKchoIlFIqyGkgUEqpIKeBQCmlgpwGAqWUCnL/D9cr/oJxoI7ZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/eUlEQVR4nO3dd3zU9f3A8df7Lgkhg2wSQkLCXmEEwpCNCCLujXXXUamz1rZq+6u71WqtexdH6yxuRQGV4QIZskEZAkkYgQABEkLW5/fH5wJHuISMu1ySez8fjzzu7rvunSN83/fZYoxBKaWUqsrh7wCUUko1TZoglFJKeaQJQimllEeaIJRSSnmkCUIppZRHmiCUUkp5pAlCqQYQkXQRMSISVItjrxCRbxp6HaUaiyYIFTBEZJOIlIhIfJXtP7puzul+Ck2pJkkThAo0vwAXVb4QkT5AmP/CUarp0gShAs1/gMvcXl8OvOZ+gIhEichrIrJTRDaLyF9ExOHa5xSRR0Rkl4hsBE71cO6/RWSbiOSKyP0i4qxrkCKSLCIfichuEVkvIte47RssIotEZJ+I7BCRR13bQ0XkvyKSLyJ7RWShiCTW9b2VqqQJQgWa+UAbEenpunFPBv5b5ZgngSigEzAam1CudO27BjgNyASygPOqnPsKUAZ0cR0zAbi6HnG+BeQAya73+JuInOja9zjwuDGmDdAZeMe1/XJX3KlAHHAdcLAe760UoAlCBabKUsR4YA2QW7nDLWncYYzZb4zZBPwTuNR1yAXAY8aYbGPMbuDvbucmApOAW4wxhcaYPOBfruvVmoikAsOBPxljio0xS4GXOFLyKQW6iEi8MeaAMWa+2/Y4oIsxptwYs9gYs68u762UO00QKhD9B/gVcAVVqpeAeCAY2Oy2bTPQ3vU8Gciusq9Smuvcba4qnr3A80DbOsaXDOw2xuyvJoargG7AWlc10mluv9cM4C0R2Soi/xCR4Dq+t1KHaYJQAccYsxnbWD0JeK/K7l3Yb+Jpbts6cKSUsQ1bheO+r1I2cAiIN8ZEu37aGGN61zHErUCsiER6isEYs84YcxE28TwETBORcGNMqTHmHmNML2AYtirsMpSqJ00QKlBdBZxojCl032iMKcfW6T8gIpEikgbcypF2ineAm0QkRURigNvdzt0GzAT+KSJtRMQhIp1FZHRdAjPGZAPfAX93NTz3dcX7XwARuUREEowxFcBe12kVIjJWRPq4qsn2YRNdRV3eWyl3miBUQDLGbDDGLKpm941AIbAR+AZ4A5jq2vcithpnGbCEY0sglwEhwGpgDzANaFePEC8C0rGlifeBu4wxX7j2TQRWicgBbIP1ZGPMQSDJ9X77sG0rc7HVTkrVi+iCQUoppTzREoRSSimPNEEopZTySBOEUkopjzRBKKWU8qhFTS0cHx9v0tPT/R2GUko1G4sXL95ljEnwtK9FJYj09HQWLaqu56JSSqmqRGRzdfu0ikkppZRHmiCUUkp5pAlCKaWURy2qDcKT0tJScnJyKC4u9ncoLUJoaCgpKSkEB+skoUq1dC0+QeTk5BAZGUl6ejoi4u9wmjVjDPn5+eTk5NCxY0d/h6OU8rEWX8VUXFxMXFycJgcvEBHi4uK0NKZUgGjxCQLQ5OBF+lkqFTgCIkHUpKLCsHP/IQ4Ul/k7FKWUalICPkEgsOvAIXYeOOSTy+/du5dnnnmmzudNmjSJvXv3ej8gpZSqpYBPEA4RYsND2F9cyqHScq9fv7oEUVZWc4ll+vTpREdHez0epZSqrYBPEACx4SGICPmFJV6/9u23386GDRvo378/gwYNYuTIkZxxxhn06tULgLPOOouBAwfSu3dvXnjhhcPnpaens2vXLjZt2kTPnj255ppr6N27NxMmTODgwYNej1Mppapq8d1c3d3z8SpWb93ncd+hsgrKKyoIC6nbR9IruQ13nV79mvQPPvggK1euZOnSpcyZM4dTTz2VlStXHu4mOnXqVGJjYzl48CCDBg3i3HPPJS4u7qhrrFu3jjfffJMXX3yRCy64gHfffZdLLrmkTnEqpVRdaQnCJdgpGANl5b5d433w4MFHjSF44okn6NevH0OHDiU7O5t169Ydc07Hjh3p378/AAMHDmTTpk0+jVEppSDAShA1fdM3xrA+7wAG6No2wmfdOcPDww8/nzNnDl988QXff/89YWFhjBkzxuMYg1atWh1+7nQ6tYpJKdUotAThIiLERbSiuLScwkPea6yOjIxk//79HvcVFBQQExNDWFgYa9euZf78+V57X6WUaqiAKkEcT3TrYLYVCPmFh4gI9c5HExcXx/Dhw8nIyKB169YkJiYe3jdx4kSee+45evbsSffu3Rk6dKhX3lMppbxBjDH+jsFrsrKyTNUFg9asWUPPnj1rfY1tBQfZtb+E7kmRhARpAcuTun6mSqmmS0QWG2OyPO3TO2AVceEhgGF3oW8GzimlVHOhCaKKkCAnkaHB7C4spaKi5ZSulFKqrjRBeBAXEUJZRQUFB0v9HYpSSvmNJggPIloF0SrI6ZOR1Uop1VxogvDAdnkNoaikjKISneVVKRWYNEFUIyYsBKcI+Qe0FKGUCkw+SxAikiois0VktYisEpGbPRwjIvKEiKwXkeUiMsBt3+Uiss71c7mv4qyO0yFEh4ew92AppT6efsNdREQEAFu3buW8887zeMyYMWOo2p23qscee4yioqLDr3X6cKVUXfmyBFEG/N4Y0wsYClwvIr2qHHMK0NX1cy3wLICIxAJ3AUOAwcBdIhLjw1g9igsPwRjDHj+0RSQnJzNt2rR6n181Qej04UqpuvJZgjDGbDPGLHE93w+sAdpXOexM4DVjzQeiRaQdcDIwyxiz2xizB5gFTPRVrNUJDXYS0SqI/MIS6jug8Pbbb+fpp58+/Pruu+/m/vvvZ9y4cQwYMIA+ffrw4YcfHnPepk2byMjIAODgwYNMnjyZnj17cvbZZx81F9OUKVPIysqid+/e3HXXXYCdAHDr1q2MHTuWsWPHAkemDwd49NFHycjIICMjg8cee+zw++m04kopd40y1YaIpAOZwIIqu9oD2W6vc1zbqtvu6drXYksfdOjQoeZAPrsdtq+ofeBAh4oKiksrKA92EOTwkE+T+sApD1Z7/oUXXsgtt9zC9ddfD8A777zDjBkzuOmmm2jTpg27du1i6NChnHHGGdVOEPjss88SFhbGmjVrWL58OQMGHK6J44EHHiA2Npby8nLGjRvH8uXLuemmm3j00UeZPXs28fHxR11r8eLFvPzyyyxYsABjDEOGDGH06NHExMTotOJKqaP4vJFaRCKAd4FbjDGeF2NoAGPMC8aYLGNMVkJCgrcvj9MhOARKy+tXgsjMzCQvL4+tW7eybNkyYmJiSEpK4s4776Rv376cdNJJ5ObmsmPHjmqvMW/evMM36r59+9K3b9/D+9555x0GDBhAZmYmq1atYvXq1TXG880333D22WcTHh5OREQE55xzDl9//TWg04orpY7m0xKEiARjk8Prxpj3PBySC6S6vU5xbcsFxlTZPqfBAdXwTb86AhzYX8z2gmK6JUYSGuys8zXOP/98pk2bxvbt27nwwgt5/fXX2blzJ4sXLyY4OJj09HSP03wfzy+//MIjjzzCwoULiYmJ4YorrqjXdSrptOJKKXe+7MUkwL+BNcaYR6s57CPgMldvpqFAgTFmGzADmCAiMa7G6QmubX4RGxaCQ4T8A/Wbn+nCCy/krbfeYtq0aZx//vkUFBTQtm1bgoODmT17Nps3b67x/FGjRvHGG28AsHLlSpYvXw7Avn37CA8PJyoqih07dvDZZ58dPqe6acZHjhzJBx98QFFREYWFhbz//vuMHDmyXr+XUqpl82UJYjhwKbBCRJa6tt0JdAAwxjwHTAcmAeuBIuBK177dInIfsNB13r3GmN0+jLVGQU4H0a2D2VNUSmJUhee2iBr07t2b/fv30759e9q1a8fFF1/M6aefTp8+fcjKyqJHjx41nj9lyhSuvPJKevbsSc+ePRk4cCAA/fr1IzMzkx49epCamsrw4cMPn3PttdcyceJEkpOTmT179uHtAwYM4IorrmDw4MEAXH311WRmZmp1klLqGDrddy0dLCljXd4B2kW1JiGy1fFPaMF0um+lWg6d7tsLWocEERYSxO7CQ/Xu8qqUUs2JJog6iI8I4VBZBQcO6fxMSqmWLyAShLe+8bdpHUyQ0xHQ8zNp6UmpwNHiE0RoaCj5+fleubE5RIgLD2FfcSmHysq9EF3zYowhPz+f0NBQf4eilGoEjTKS2p9SUlLIyclh586dXrleeYUhr6CYorwgoloHe+WazUloaCgpKSn+DkMp1QhafIIIDg6mY8eOXr3mc28sYd7P25h/5zjCQlr8R6iUClAtvorJFy4fls6+4jI+XLrV36EopZTPaIKoh6y0GHq1a8NzczdQXBp4bRFKqcCgCaIeRIQ/n9qTzflFPPHlOn+Ho5RSPqEJop6Gd4nnvIEpvDBvI2u2eX2SWqWU8jtNEA3w50k9iWodzO3vLqe8QscHKKVaFk0QDRATHsJfT+/FspwCXv1uk7/DUUopr9IE0UBn9EtmbPcEHpn5Ezl7io5/glJKNROaIBpIRLj/7D4A/OWDlToVhVKqxdAE4QXto1tz24TuzPlpJx8t07ERSqmWQROEl1w+LJ1+qdHc+/Fq9hQG7mR+SqmWQxOElzgdwoPn9KHgYCkPTF/j73CUUqrBNEF4Uc92bbh2VCemLc7h2/W7/B2OUko1iCYIL7tpXFfS48K48/0VOg2HUqpZ0wThZaHBTv52Th825xfx2Bc6DYdSqvnSBOEDwzrHc2FWKi9+vZGVuQX+DkcpperFZwlCRKaKSJ6IrKxm/x9EZKnrZ6WIlItIrGvfJhFZ4dq3yFcx+tKdk3oSExbCHe+toKy8wt/hKKVUnfmyBPEKMLG6ncaYh40x/Y0x/YE7gLnGmN1uh4x17c/yYYw+ExUWzN1n9GJFbgGv6DQcSqlmyGcJwhgzD9h93AOti4A3fRWLv5zapx3jerTlnzN/Jnu3TsOhlGpe/N4GISJh2JLGu26bDTBTRBaLyLXHOf9aEVkkIou8te60t4gI952VgUPgzzoNh1KqmfF7ggBOB76tUr00whgzADgFuF5ERlV3sjHmBWNMljEmKyEhwdex1llydGv+OLEH837eqUuUKqWalaaQICZTpXrJGJPreswD3gcG+yEur7lkaBqZHaK595PV7NZpOJRSzYRfE4SIRAGjgQ/dtoWLSGTlc2AC4LEnVHNhp+Hoy/7iUu79eJVWNSmlmoUgX11YRN4ExgDxIpID3AUEAxhjnnMddjYw0xhT6HZqIvC+iFTG94Yx5nNfxdlYuidF8tsxXXj8y3WEtwrinjN6E+RsCgU4pZTyzGcJwhhzUS2OeQXbHdZ920agn2+i8q+bx3XlUFkFz83dwI59xTx50QBahzj9HZZSSnmkX2EbkcMh3H5KD+49szdfrs3johfnk3/gkL/DUkopjzRB+MFlJ6Tz7MUDWbNtH+c99z2b8wuPf5JSSjUyTRB+MjEjidevHsKeohLOffY7lufs9XdISil1FE0QfpSVHsu064YRGuxk8gvzmf1Tnr9DUkqpwzRB+FmXthG899thdIwP5+pXF/HOwmx/h6SUUoAmiCahbWQob//mBIZ1juOP7y7nsS9+1rESSim/0wTRRES0CmLqFYM4d0AKj32xrl7ThFdUGNbn7eedhdnc/8lqVm/d56NolVKBwGfjIFTdBTsdPHJ+X5KjQ3nyq/Xs2FfM0xcPICzE8z9TwcFSlmXvZcmWPSzZspelW/awr7js8P73f8zlg+uHkxob1li/glKqBZGWVJWRlZVlFi1qlusLHeP1BZv5vw9WkpHchlcnhRLVMYv1Ow+wZPMeftxik8L6nQcwBkSge2IkmR2iyewQw4AO0YBwzjPf0rZNKO9OGUZU62B//0pKqSZIRBZXt+6OJogmbNbqHbzy5uu87ryHK7mL2cXdAYgOCyYztTIZxNAvNYrI0GMTwHcbdnHZv39gaKc4Xr5yEME6tYdSqoqaEoRWMTVh43sl0m1Ua/gWLk7M4dTBF5DZIZpO8eG45qqq0bDO8fz9nD78Ydpy/vrhSv52dp9anaeUUqAJoslLc9plMk6K3AwDU+p8/vlZqWzKL+Tp2RtIjwvnN6M7eztEpVQLpQmiqStwjYvIWQQVFeCoezXR78d3Z3N+EQ9+vpa0uDAmZrTzcpBKqZZIK6WbuoIc+1i8F3ZvqNclHA7hkfP70T81mlveXsqy7L1eC08p1XJpgmjqCrIhsY99nv1DvS8TGuzkxcuySIhsxVWvLiJ370EvBaiUaqk0QTRlFRVQkAudx0KrNpCzsEGXi49oxctXDOJQWTm/fnkh+4tLvRSoUqol0gTRlB3YARWlEJMG7QfadogG6tI2kucuGciGnQe44Y0f6zxaWykVODRBNGWV7Q9RqZA6GPJWwaH9Db7s8C7x3H9WBnN/3snduka2Uqoa2oupKavswRSVAuIAUwFbf4SOoxp86cmDO/BLfiHPz91Ielw4V4/s1OBrKqVaFi1BNGWHSxAptooJGtwO4e5PJ/fglIwkHpi+hpmrtnvtukqplkETRFNWkGMbp0OjICwW4rpCtvcShMMhPHpBf/qmRHPzW0tZmVvgtWsrpZo/n1UxichU4DQgzxiT4WH/GOBD4BfXpveMMfe69k0EHgecwEvGmAd9FWeTVpBt2x8qpQyCdTM5PEOfF7QOcfLiZQM5++nv+PUrC3nm4gGIQOGhcopKyg4/FpWUU1hSTtGhMvvoti+xTSh/mtiDpKhQr8SklGoafNkG8QrwFPBaDcd8bYw5zX2DiDiBp4HxQA6wUEQ+Msas9lWgTVZBtq1eqpSSBcvegD2bILaj196mbWQoU68YxHnPfsd5z31f47HhIU7CWgXZx5AgwkKcfLZyG1+u2cF9Z2VwRr9kne9JqRbCZwnCGDNPRNLrcepgYL0xZiOAiLwFnAkEYILIgZTBR16nup7nLPRqggDonhTJJzeNYM22fbQOOZIAwlsdeQwNcuJwHHvz/2VXIb9/Zyk3v7WUGau2c/9ZfYgND/FqfEqpxufvXkwniMgyYCtwmzFmFdAecF+YOQcYUt0FRORa4FqADh06+DDURnboABzcc3QJIqEnBIfbBNH3Aq+/ZVpcOGlx4XU+r2N8OP+7bhjPz9vAv2b9zA+/7OGhc/swrmei12NUSjUefzZSLwHSjDH9gCeBD+pzEWPMC8aYLGNMVkJCgjfj86/KHkzRbknPGQTtBzRoyg1fcTqE347pwofXjyA+IoSrXl3EH6ct09HaSjVjfksQxph9xpgDrufTgWARiQdyAbeWWVJc2wKLexdXdymDYMdKKClq/JhqoVdyGz68YTi/HdOZaYtzmPjY13y/Id/fYSml6sFvCUJEksTVmikig12x5AMLga4i0lFEQoDJwEf+itNv3AfJuUsZBBVlsG1Z48dUS62CnPxxYg/+d90wQoIcXPTifO79eDXFpeX+Dk0pVQc+SxAi8ibwPdBdRHJE5CoRuU5ErnMdch6w0tUG8QQw2VhlwA3ADGAN8I6rbSKwFOSAOCEi6ejtKYPsY07Tq2aqamBaDJ/eNILLT0hj6re/cOoTX+tU40o1I77sxXTRcfY/he0G62nfdGC6L+JqNgpyoE2ybXdwF5EAMeleHVHtS2EhQdxzZgbjeyXxh2nLOOfZ77h+TGduHNdV18hWqonT/6FNVdVBcu5SBtkR1c1okr0RXeP5/JZRnNk/mSe+Ws+ZT33LV2t3UFHRfH4HpQKNJoimquogOXcpg+HA9iMN2c1EVOtgHr2gP89fOpCCg6X8+pVFTHx8HtMW51BSptOOK9XUaIJoiirKYd/WGhJEln1sJtVMVZ3cO4k5fxjDYxf2xyHCbf9bxqh/zObFeRu1W6xSTYgmiKbowA7bU6m6BJGYAUGhXllAyF+CnQ7OymzPZzeP5NVfD6ZTQjgPTF/DsAe/4qHP15K3r9jfISoV8Pw9klp5stfVxTW6mpHhQSGQnNksejIdj4gwulsCo7slsDxnL8/P28jzczfw769/4ZwB7blmVCc6J0T4O0ylApImiKaoujEQ7lKyYMHzUHYIglo1Tlw+1jclmqd/NYDN+YW89PUvvLMom7cXZTO+ZyK/Gd2ZgWkx/g5RqYCiVUxNUWXjc5v21R+TMgjKS2D7isaJqRGlxYVz31kZfHf7idx4Yld+2LSbc5/9jvOf+453Fmazu7DE3yEqFRBqVYIQkXDgoDGmQkS6AT2Az4wx2qLoCwU5dpGg0DbVH1M5y2v2D0carVuYuIhW3Dq+G9eN7sQ7C7OZ+u0m/vjuchzvweCOsZzcO4mTeyeRHN3a36Eq1SLVtoppHjBSRGKAmdjpMC4ELvZVYAGtIKf6MRCV2rSDNinNtidTXYSFBHHF8I5cPiydVVv3MWPVdmas2s49H6/mno9X0zcl6nCy6NJW2yuU8pbaJggxxhSJyFXAM8aYf4jIUh/GFdhqGiTnLiWrWfdkqisRIaN9FBnto/j9hO5s3HmAGat28Pmq7Tw84ycenvETXdpGcHLvRCb2bkdG+za6eJFSDVDrBCEiJ2BLDFe5tjl9E5KiIBs6nHD841IHw+oPYP92iEw67uEtTaeECKaMiWDKmM5sKzjIzFU7+Hzldp6bu5GnZ2+gfXRrJvROZPKgDnRPivR3uEo1O7VNELcAdwDvG2NWiUgnYLbPogpkxfuguKDmHkyVDk/ctxB6nu7buJq4dlGtuXxYOpcPS2d3YQlfrNnBzFXbeX3BFl79bhMXDe7AreO7ERfRMnp8KdUYapUgjDFzgbkAIuIAdhljbvJlYAFrn2vpi9okiKS+4AjWBFFFbHgIF2SlckFWKnuLSnjsi3X8Z/5mPlq2lVtO6sZlJ6TpRIFK1UKt/peIyBsi0sbVm2klsFpE/uDb0AKUp5XkqhMcCu362Yn7lEfRYSHcfUZvPr95JJkdYrjvk9Wc/Ng8Zq/N83doSjV5tf0a1csYsw84C/gM6Ahc6qugAtreLfaxNiUIsNVMW3+Ecu1xXJOuiZG8euUgpl6RBQaufGUhl0/9gfV5+/0dmlJNVm0TRLCIBGMTxEeu8Q86T7MvFOSAIwgiEmt3fOogKDsIOwJvTaW6EhFO7JHI57eM4i+n9mTJlj1MfOxr7vl4FQVFmmCVqqq2CeJ5YBMQDswTkTRgn6+CCmiVCwU5atlJzL2hWtVKSJCDq0d2Ys5tY7hgUCqvfreJMY/M5j/fb6KsXKcdV6pSrRKEMeYJY0x7Y8wk17Kgm4GxPo4tMNVmkJy7qFRb2tAEUWdxEa3429l9+OTGkXRPiuT/PlzFqU98w7frd/k7NKWahNpOtREF3AWMcm2aC9wLFPgorsZTWgyLpkJib+g02t/R2DEQacNrf7yILUVogqi3XsltePOaocxYtZ0Hpq/h4pcW0C8liqSoUGLDQ1w/rYgND7aPYSHERoQQFx5CaLAOB1ItV23HQUzF9l66wPX6UuBl4BxfBNWonCHw9T+h84n+TxDlZTUvFFSdlEGw9hMo3AXh8b6JrYUTESZmtGNM97a88t0m5vyUx6ZdRSzevJc9RSWUV7M0autg5+EkkhLTmouHpDG8S5yO4FYtQm0TRGdjzLlur+9pMVNtOBzQZRys/8Ku5Fbbun9fOLAdTHn9EgTYaTe6T/R+XAEkNNjJdaM7c93ozoe3VVQY9heXkV94iD1FJeQfKGF3YQm7i0rY7fZ80eY9fLZyO33aRzFlTGdO7p2E06GJQjVftU0QB0VkhDHmGwARGQ4crOkEEZkKnAbkGWMyPOy/GPgTIMB+YIoxZplr3ybXtnKgzBjj2+lKu4yH5W/b7qL+nBm1cgxEXdogwC4eJE67gJAmCK9zOISosGCiwoJrPO5QWTnvL8nl+Xkb+e3rS+gYH85vRnXi7AHtaRWkVVGq+altL6brgKdFZJPr5v0U8JvjnPMKUNPd6hdgtDGmD3Af8EKV/WONMf19nhzAliAQWDfL529Vo8OD5OqYIELCIClD2yH8rFWQk8mDO/DFraN55uIBRLQK4vb3VjDyodk8P3eDrretmp3a9mJaZozpB/QF+hpjMoETj3POPGB3Dfu/M8bscb2cD9SxXsWLwmJtyWG9nxNE5SC5mhYKqk7KIMhdYqvJlF85HcKkPu346Ibh/PeqIXRNjODvn61l2INf8fCMtezcf8jfISpVK3WakMYYs881ohrgVi/GcRV2hPbhtwJmishiEbm2phNF5FoRWSQii3bu3Fn/CLqMtzfYQj92cSzIgdYx0KoeaxqkDIaSA5C3xvtxqXoREUZ0jef1q4fy0Q3DGdk1nmfmbGDEQ1/xfx+sZEt+kb9DVKpGDVmT2iutbyIyFpsgRrhtHmGMyRWRtsAsEVnrKpEcwxjzAq7qqaysrPqP7u56Esz5G2z4CvpecPzjfaEgp+4N1JUq205yFtrqJtWk9E2J5pmLB7Jx5wFemLeRtxZu4fUFm5nUpx0DOsSQHN2a5OhQ2kW1Ji48BIc2bqsmoCEJosFTbYhIX+Al4BRjTP7hCxuT63rME5H3gcHYVe18p10mhMXbdgh/JoiYtPqdG9sJwuJsT6asK70bl/KaTgkRPHhuX245qRtTv/2FNxds4ZPl2446JsTpICkqlHZRoSRHt6ZdVCjtoluTHGUTSHJ0KFGtg7UrrfK5GhOEiOzHcyIQoEELAYtIB+A94FJjzM9u28MBhzFmv+v5BOygPN86qrtrhX3d2AqyIX3E8Y/z5PCAuR+8G5PyiaSoUO6c1JM7TunB7sISthUUs3XvQftYcJBte4vZVnCQH37ZzY59xZRVGYeR2KYVA9NiGNAhhoFpMfROjiIkSKcwV95VY4IwxtR7GS4ReRMYA8SLSA52JHaw67rPAX8F4oBnXN+EKruzJgLvu7YFAW8YYz6vbxx1clR314GN8paHFRfAoX31r2ICW8308+dwcI9ty1BNnogQF9GKuIhWZLSP8nhMeYVh14FDhxNI7p6DrNxawOLNe5i+YjsArYIc9EuJZkBajCtxROviSKrBGlLFVCNjzEXH2X81cLWH7RuBfr6Kq0adTwTE9mZq7ARxeAxEQxKEa8Bc7mLoclLDY1JNgtMhJLYJJbFNKJlV9u3YV8zizXsO//z7m408N9eWNjrGhx8uYQxMi6Fr2wht21B14rME0SyFx0H7gbBuJoy5vXHfu76D5Ny1HwiIXUBIE0RASGwTyqQ+7ZjUpx0AxaXlrMgtOJww5vyUx7tL7N9WbHgIY7olcGLPtozsmkBU65oH/imlCaKqrhNgzt8bf16jgmz7WNdBcu5aRULbXjpgLoCFBjsZlB7LoPRYAIwxbM4vYtHmPXyzbidf/ZTHez/m4nQIg9JjOLFHW07skUjnhHBt9FbH0ARRlb+6u+7NtutLh7dt2HVSB8Gq9/3X0K6aFBEhPT6c9PhwzhuYQnmF4ccte/hqbR5frc3jb9PX8rfpa+kQG+ZKFm0Z0ilWpwZRgCaIY/mru2tBDkS1b/hNPWUQLH4F8tdBQnevhKZaDqdDyEqPJSs9lj9O7EHu3oPMdiWLN3/YwivfbSIsxMmILvGM69mWsd3b0rZNqL/DVn6iCaIqf3V3retCQdVxX2FOE4Q6jvbRrblkaBqXDE3jYEk532/cZUsXa/KYuXoHIjAoLZZT+7bjlD5JtI3UZBFINEF44o/urgU50HHU8Y87nriuEBplE0TmJQ2/ngoYrUOcnNgjkRN7JGLONPy0Yz8zVu5g+opt3PXRKu7+eBVDOsZyat9kJvZOIiFSu9G2dJogPGns7q7lpbC/HgsFeeJwQPss25NJqXoSEXoktaFHUhtuPqkrP+/Yz6fLt/HJ8q383wcruevDlQztFMepfdsxsXeSjrloobQV05PD3V0baXbX/dvAVHgnQYCtZspbDYf2e+d6KuB1S4zkd+O78cWto5lxyyhuGNuF7QXF/Pn9lQz+25dc8tIC3vphC3sKS/wdqvIiLUFUp+t4mPMgFObbhOFL3hgk5y51EGDs7LT+XkZVtSgiQvekSLonded347uxZtt+Pl2xlU+Xb+P291bw5w9WMrxLPBcNSmV8r0SCnPodtDnTf73qdBkPGNjwpe/fyxuD5Ny1d1WL6XgI5UMiQq/kNvzh5B7Mvm0Mn9w4gmtHdWJD3gGmvL6EEQ/N5okv15G3v9jfoap60hJEdZIbsbtr5SA5b5UgWsdAbGfbyK5UIxARMtpHkdE+itsmdOertXm89v0mHp31M09+tY6JGe247IQ0stJidEBeM6IJojqN2d11b7adqjskzHvXTM6ELfO9dz2lasnpEMb3SmR8r0Q27jzAf+dv4X+Ls/l42VZ6JEVy2QnpnJWZTFiI3n6aOq1iqkmX8VCU7/tv4g1ZKKg6yZmwLwcO5Hn3ukrVQaeECP56ei8W3DmOv5/TBxHhzvdXMORvX3LPx6vYuPOAv0NUNdAUXpPG6u5akANxnb17zWTXvJ9bl0K3Cd69tlJ1FBYSxEWDOzB5UCqLN+/hte8389/5m3n5202M7BrPZSek0yMpEqdDcIjgcIBTxL52CE45drtWVfmeJoiauHd39dXsrsbYNohOY7x73XZ9AbGlH00QqokQOTLVR97+nrz9QzavL9jCNa8tqse1oH9qNPedmVHtWhqqYTRBHI+vu7sW74WSA96vYmoVCfHdtKFaNVltI0O5cVxXpozpzNfrd7Fr/yEqjKG8AsqNwRhDeYX9qdxeYQwVFYZyYygpq+CdRTmc8dQ3/Hp4R343vhvhrfSW5k36aR5Pl/F2+u8NX0Hf871/fW+PgXCXnAkb53j/ukp5UZDTwdju9ZvF+DejO/OPz9fy0je/MH3FNu49M4OTeiV6OcLApY3Ux5OcaXsYrffRqGpvj4Fwl5wJB7bDvm3ev7ZSTUBU62AeOLsP7045gcjQYK5+bRHX/Wcx2wt07IU3aII4HocDOrt1d/U2X5cgQKuZVIs3MC2WT24awR8ndmf2T3mc9OhcXv1uE+UVxt+hNWuaIGqj6wTfdXctyAZnKwhP8P61k/qAODRBqIAQ7HTw2zFdmPW70WR2iOauj1ZxzjPfsmprgb9Da7Y0QdSGe3dXb9ub7Z2FgjwJCYOEnpogVEDpEBfGa78ezBMXZZK79yBnPPUtD3y6msJDZf4OrdnxaYIQkakikiciK6vZLyLyhIisF5HlIjLAbd/lIrLO9XO5L+M8Ll/O7uqLQXLukjNtgjBa1FaBQ0Q4o18yX946hguyUnnx61+Y8K95fLlmh79Da1Z8XYJ4BZhYw/5TgK6un2uBZwFEJBa4CxgCDAbuEpEYn0Z6PF3HQ+5i293Vm7y1klx1kvtD0a4jbR1KBZCosGD+fk4fpl13AuGtnFz1qm3Ezt5d5O/QmgWfJghjzDxgdw2HnAm8Zqz5QLSItANOBmYZY3YbY/YAs6g50fje4dldv/LeNctL7VoQPk0QrkKZVjOpAJaVHssnN47kDyd3Z87PeYx7dC7/nPkTRSVa7VQTf7dBtAey3V7nuLZVt/0YInKtiCwSkUU7d+70WaA+6e66Lxcwvq1iSuwNjiBNECrghQQ5uH5sF776/RhOyUjiya/Wc+Ijc/ngx1yMVsF65O8E0WDGmBeMMVnGmKyEBB/0BKp0uLvrl97r7urLLq6VgkOhbS9NEEq5JEe35vHJmbw75QTatmnFLW8v5Zxnv2Np9l5/h9bk+DtB5ALu9Ssprm3VbfevruNtff42L91sfTlIzp02VCt1jIFpsXzw2+E8fF5fcvYc5Kynv+X37ywjb58Osqvk7wTxEXCZqzfTUKDAGLMNmAFMEJEYV+P0BNc2/+o8DhBY94V3rnd4oSCPtWfek5xp53zas8m376NUM+NwCOdnpTL7tjFMGdOZj5dtZewjc3h69nqKS8v9HZ7f+bqb65vA90B3EckRkatE5DoRuc51yHRgI7AeeBH4LYAxZjdwH7DQ9XOva5t/He7uOtM71yvIsQPkglt753rVSe5vH7WaSSmPIloF8aeJPZh16yiGd4nn4Rk/Mf5fc/l85faAbp/w6WR9xpiLjrPfANdXs28qMNUXcTWIN2d33Zvt2/aHSm17gTPEJoiMc3z/fko1U2lx4bxwWRbfrNvFvZ+s4rr/LmZY5zjunNSTtLgwghwOnA4hyLVORUuns7nWlTdndy3IgYRu3omrJkGtbG8mLUEoVSsjusYz/aaRvPnDFv4562dOe/KbY44RgSCHuBLGkcQR5LSvg5xCYmQoHeLCSI8Lo0NcOGmxYaTHhRMVFuyH36ruNEHUlXt314YkCGNsgugyznux1SQ5E1ZM8/362kq1EEFOB5eekM7p/ZL5ZPk2ikvLKXOtT1FWbiivqDj8urTK67IKu17F9oJi5v28k2n7Dx117ajWwaTFhdHBlTA6xIXZ5BEfTtvIVk1mtTxNEHVVtbtrfW+2B/dAaaHvezBVSs6ERVNhzy/eX95UqRYsOiyES4amNegaB0vK2bK7iE35hWzJL2Lz7kI25xexPKeAz1ZuP2rW2a5tI7hpXFcm9WmH08/VWJog6qPreFjxju3u2r6ea1Uf7sHUCG0QcPTU35oglGpUrUOcdE+KpHtS5DH7Sssr2Lr3IJvyi9i48wBvLNjCjW/+yONfruPGE7twWt9kvyUKrWuoD290d22MQXLuEnpAUKi2QyjVxAQ7HaTFhTO6WwJXDu/IjFtG8dSvMnEI3PzWUib8ay4fLs31y9oWmiDqo7K766r36z+qurEGyVVyBtv1ITRBKNWkORzCaX2T+fzmUTz9qwEEORzc/NZSxv9rLu//mENZuQ8WLqsulkZ7p5ZmyG9g5xpY81H9zi/Itt/ow+O9G1dNkjNh2zKo0AFASjV1Dodwat92fHbzSJ69eAAhTge/e3sZ4/81j/eWNE6i0ARRXxnnQnx32+W1PjfcynUgGrO3QnImlByA/PWN955KqQZxOIRT+rRj+k0jee6SAYQGO7n1nWWc9Ohcpi32baLQBFFfDieMuR12roWV79X9/MYaJOdO16hWqtlyOISJGe349MYRPH/pQMJCgrjtf8sY9+hc/rco2yeJQhNEQ/Q6CxIzYO6DUF7HeeV9vZKcJ/HdIDhME4RSzZjDIZzcO4lPbxrBC5cOJKJVEI99sQ5ftGFrN9eGcDhgzB3w9sW222v/X9XuvLJDcGB74zVQV3I4oV0/TRBKtQAiwoTeSYzvlci2gmJCgrz/fV9LEA3V41R70537kF0hrjb2bbWPjZ0gwNVQvbzuJR6lVJMkIiRH+2bCT00QDSUCY/9sp9Je+kbtzmnsQXLukjOh7CDs+qnx31sp1axogvCGrhOgfRbMe9hWHx1PYw+Sc6cN1UqpWtIE4Q0iMPZOWzJY8trxj69MEG18vFCQJ7GdISRSE4RS6rg0QXhL5xOhwwnw9T+h9DhLFhZkQ3hbu150Y3M47AJCmiCUUsehCcJbKksR+7fB4pdrPrYgB6L90EBdKbk/bF8JZSX+i0Ep1eRpgvCmjqMgfSR8/SiUFFV/nD8GyblLzoTyQ3aqEKWUqoYmCG8b+2cozIOFL3neX7lQkD+6uFbShmqlVC1ogvC2tBNse8S3j8Gh/cfuL9ptu5n6swQR0xFCozRBKKVqpAnCF8b+BYryYcHzx+47PAbCjyUIEVuK0AShlKqBTxOEiEwUkZ9EZL2I3O5h/79EZKnr52cR2eu2r9xtXz3n1PaTlIHQbSJ89yQUFxy9z5+D5NwlZ8KO1cfvcaWUClg+SxAi4gSeBk4BegEXiUgv92OMMb8zxvQ3xvQHngTcp0U9WLnPGHOGr+L0mTF3QPFemP/s0dsbe6Gg6iRnQkUp5K3ybxxKqSbLlyWIwcB6Y8xGY0wJ8BZwZg3HXwS86cN4Gldyf+hxGnz/tG13qFSQA0GtISzWb6EB2lCtlDouXyaI9kC22+sc17ZjiEga0BH4ym1zqIgsEpH5InJWdW8iIte6jlu0c+dOL4TtRWPugEP7bJKoVJDd+AsFeRKVCmFxmiCUUtVqKo3Uk4Fpxhj3pdnSjDFZwK+Ax0Sks6cTjTEvGGOyjDFZCQkJjRFr7SVlQO+zYcFzUJhvt/l7kFylww3VS/0diVKqifJlgsgF3O+EKa5tnkymSvWSMSbX9bgRmANkej/ERjDmDigptN1ewf+D5NwlZ0LempoH9SmlApYvE8RCoKuIdBSREGwSOKY3koj0AGKA7922xYhIK9fzeGA4sNqHsfpOQnfocz788CLs3WIH0fm7gbpSciaYctix0t+RKKWaIJ8lCGNMGXADMANYA7xjjFklIveKiHuvpMnAW8YY9wXzegKLRGQZMBt40BjTPBME2LWry0tg+h/s66ZUggBth1BKeeTTJUeNMdOB6VW2/bXK67s9nPcd0MeXsTWquM7QbzIsfd2+bioliMh2dlZZTRBKKQ+aSiN1yzf6j+Bw5eOmUoLQEdVKqRpogmgsMekw4DIIDoM2yf6O5ojkTNj5Exw64O9IlFJNjCaIxjTxIbjuGwhq5e9IjkjOBAxsX+7vSJRSTYwmiMYUFGLbI5qS5P72UauZlFJVaIIIdJFJEJmsCUIpdQxNEEobqpVSHmmCUDZB5K+H4n3+jkQp1YRoglBHBsxtW+bfOAJNQS58fLPtRaZUE6QJQmlDtT9sWw4vjYPFr8Dr50PhLn9HpNQxNEEoCI+HqA6aIBrLulnw8ikgTjjrWTiwA96+FMpK/B1ZwxgDy/8HL46Dle8d/3jV5GmCUFZy/7oniN2/wKKX7dKqB/J8EtZh+Rvgm8dgezOfWHDRy/DGhRDbCa7+Avr/Cs58GrZ8B5/+zt5km6PdG+E/Z8N7V8OudTDtSnh/irZrNXM+nYtJNSPJmbDmIzi4B1rHeD6maDf8Mg82zoaNc2DPpiP7vrgHMs6BIb+B9gO9E1NFBWz8ChY8D+tm2m1f3QcjboVRtzWtAYfHU1EBX95jp33vOgHOexlaRdh9fc6z7RDz/gEJPWHYDX4NtU7KSuD7J2HuP8ARDJMesTMGfP1PmPcwbP4Wzn0JUgf7O9JjGWMX9CrcBYU73X52Vdm2CyrK4KS7oedp/o66UYlprt9YPMjKyjKLFi3ydxjN04bZ8J+z4NIPoPNYu63sEGQvsPs2znYtLmQgJBI6joROY6HTGBAHLHwRfnwdSvZDyiAYch30PMMODqyrQ/th2Vs2MeSvsxMKZv0aep9lSxHL34KEHvabd0qWlz4AHyothg+mwKr37O9xysPgrPLdrKIC/nc5rPkYfvU2dDvZP7HWxZb58PEtsHMN9DrTzhTQpt3R+9+7xjbGj/4jjLzt2N+7IQp32d53JYVQWmTXNSktdD0Wed5eUngkKRTtsrMsexIaBeEJEBZvq2D3bLLT4g+ZAuPvrd/fdW1UVICp8O7ndBwisti1ONux+zRBKMCWHB5Kh6yrILajTQqbv4Oyg7auPGWQTRydxkL7AeAMPvYaxftg2Zv2xr57A0Qk2utlXQkRbY8fQ/4GWPgS/Phf+584eQAMnWJvPu6lhZ9nwie3wP5tcML1MOZOCAnz1ifhXUW74c2LIHu+vbEMu6n65WZLCm3bRP5GuGomJPZq3Fhr6+Ae+OJu28AelWpLDd0nej62eB989kf7d5EyGM55wf59NcTWH2H+c7DyXagorf44Z4id+ywk3PUYBsHh0CrS3vzD412Plc/jjySFqgmg7BDMugsWPGtL2+e93PDfw50xsGIazPyzTWTpI478f4vv6tMlijVBqNp5vD/s+cU+j+9m/zg7j4W04RDapvbXqaiADV/aRLF+lv2P2vscGHLtsdVPxsAGt2okh9Mu0zrkuppLB8UFMOuv9iYV2wnOeArSh9f1N/at/A22h1JBDpzzvP29jqcgF1480d6grpltb1pNhTH2pvz5Hfbb99Df2hUTK6vKarLyXfj4d/bb8aSH7fT3dbnplZfB2o9tYsieb2/0mRdD15Pt+3tKBL74Fr7mY/jgevv8zKeg1xk1H18bO3+G6b+31bfJA6BdP1cVruv/Ypv2tqRe+VObL1t1oAlC1c7m7+0fZcfRENXeO9fctR5+eMGuhVFy4Ej1U5dx9hvTDy/Arp/tN7esX8PAK4+upjiejXPhoxth72YYdLWtJ24V6Z3YGyL7B3hzsr2pXvQWdBhS+3NzFsMrk+w31cs+9E5bS0kRFOXbNUDqc+Pcswk+/T2s/8LGdfrj9kZWF3uz4f3rYPM3Nlme9q/q27sqFe2GJa/CDy/BvhyITrPtXJmX2Gogf9izCf53JWxdAoN/AxPuq9+/UUmRbav59nGb1MbdBQOvsF+SKt9ng6u975e5tuQGkJjhShZjIW1Yg0vPmiCU/1WtfqqUnGnrdXufVf8bYUkhfHU/zH/WrrVx+mPQ5aS6XaNoN+QstG0u2T/YZJbU1/buatcfEnvXPr5VH8B719oke/G0+k3QuGIavHsV9L/EflOtbxVDeam9wc7+u/3WL0473XxUKkSnuj2m2K7O0akQ3Pro879/CuY8ZG9c4/5qE3HlTayuKsrtDXH2A7YK8uznbXtWVXlrYMFzsOxtW82ZPtJWN3abWP/39qayEvjiLpj/jP37OP+VulU5/TwDpt9mlyHuO9kmmZpKBhXldiDrxjm2PXDLfNt+4gyB1CE2YYz4Xb0+G00QqumorH7a9DX0ON1WI3mrfnXLAvjoBlsi6X8JnHy/52+oFeX2BpTzA2QvtI/56+0+cUJSH/vtdNsyKN5rtzuCoW1Pm9CqSxrG2C6/s/4PUofC5DcgPK7+v8/sv8Hch2DC/TDsxrqdawz89Jm9ie36GdJGQMbZsG8bFGTbaq+92bAv165L7i4s/kjyyF8Peauhx2lwyj+8V7LMXWIbsPM3wPCbYeyf7YJa62baev6NcyAoFPpeYL+lJ2V45329bc0n8OFv7ed9xpP2i05N9mbD57fD2k8gvjuc9qhtb6irkiLY8r1NFhvm2Mb4m5bU5zfQBKECSGmxval++7itvz/1Uds2kbPoSOkgd4ntbQX2Zpg6BFIH2UbU5MwjRXZjbNXV1qWwbaltHN261EPS6G8TxvYVsPhlW31y1nMQHNqw36WiAqZdAas/stVU1TUEV5W7BGb+n63KietqG8e7n+I5EZeX2cb+gmx78yrY4nrMsdvEYUsNPU5t2O/iSUkhzLjTtiMl9rE3ud0b7OzCg6+GAVc0LME2lj2b7biP3MUw6Bqb0Kv+25eXwvdP279NsL26hl7vvd5QpQePLvnVgSYIFXi2LoUPb4AdK45sE6f91p862CaD1EEQ07FuJZhjksZSmzgqk8bwW2xdssNLY1BLilw9m9bDVbNq7tm0dwt8eR+seMcmvrF3wIDLPfc4a0rWfgqf/clOPT/kOttrranHXFVZiR3n8v1Ttmry/FeOVC1u+hY+vRV2roXup8IpD0J0B7+G604ThApM5aV25HLJfpsQ2g+wPV28rTJplBTaBORt+7bCC2Or79lUXABfP2rbYERs19/ht9St55nyjrXT7ZiXinKY+Hc7UHDZm7Z9Z9I/bEmuifFbghCRicDjgBN4yRjzYJX9VwAPA7muTU8ZY15y7bsc+Itr+/3GmFeP936aIFSLlbsYXq7Ss6m8FBZNhTkP2h4u/SbDiX+xDc7Kf/Zusb2cchfZashhN8KoPzTZsTp+SRAi4gR+BsYDOcBC4CJjzGq3Y64AsowxN1Q5NxZYBGQBBlgMDDTG7KnpPTVBqBZt5bsw7dfQ/2L7TXTWXbbOvuMoGH/fkVl5lf+Vl9oBn2nDIKG7v6OpUU0JwpfjuQcD640xG11BvAWcCayu8SzrZGCWMWa369xZwETgTR/FqlTTl3GunbNp7kN2XElCD/jV/6DreJ+OtFX14Ay2Mwg0c75MEO2BbLfXOYCn0ULnisgobGnjd8aY7GrO9di/TkSuBa4F6NCh6TT8KOUTo2+33UEj2tquvI04Z48KPP6e7vtjIN0Y0xeYBRy3naEqY8wLxpgsY0xWQkKC1wNUqklxOGwXyYFXaHJQPufLBJELpLq9TuFIYzQAxph8Y8wh18uXgIG1PVcppZRv+TJBLAS6ikhHEQkBJgMfuR8gIu6T7pwBrHE9nwFMEJEYEYkBJri2KaWUaiQ+K6MaY8pE5Absjd0JTDXGrBKRe4FFxpiPgJtE5AygDNgNXOE6d7eI3IdNMgD3VjZYK6WUahw6UE4ppQJYTd1c/d1IrZRSqonSBKGUUsojTRBKKaU80gShlFLKoxbVSC0iO4HN9Tw9HtjlxXCaK/0cLP0cLP0crJb8OaQZYzyOMm5RCaIhRGRRdS35gUQ/B0s/B0s/BytQPwetYlJKKeWRJgillFIeaYI44gV/B9BE6Odg6edg6edgBeTnoG0QSimlPNIShFJKKY80QSillPIo4BOEiEwUkZ9EZL2I3O7vePxJRDaJyAoRWSoiATProYhMFZE8EVnpti1WRGaJyDrXY4w/Y2wM1XwOd4tIrutvYqmITPJnjI1BRFJFZLaIrBaRVSJys2t7wP1NBHSCEBEn8DRwCtALuEhEevk3Kr8ba4zpH2B9vl/Brnnu7nbgS2NMV+BL1+uW7hWO/RwA/uX6m+hvjJneyDH5Qxnwe2NML2AocL3rvhBwfxMBnSCAwcB6Y8xGY0wJ8BZwpp9jUo3MGDMPux6JuzM5sgTuq8BZjRmTP1TzOQQcY8w2Y8wS1/P92IXM2hOAfxOBniDaA9lur3Nc2wKVAWaKyGIRudbfwfhZojFmm+v5diDRn8H42Q0istxVBdXiq1XciUg6kAksIAD/JgI9QaijjTDGDMBWuV0vIqP8HVBTYGxf8EDtD/4s0BnoD2wD/unXaBqRiEQA7wK3GGP2ue8LlL+JQE8QuUCq2+sU17aAZIzJdT3mAe9jq+AC1Y7KNdNdj3l+jscvjDE7jDHlxpgK4EUC5G9CRIKxyeF1Y8x7rs0B9zcR6AliIdBVRDqKSAgwGfjIzzH5hYiEi0hk5XNgArCy5rNatI+Ay13PLwc+9GMsflN5Q3Q5mwD4mxARAf4NrDHGPOq2K+D+JgJ+JLWr295jgBOYaox5wL8R+YeIdMKWGgCCgDcC5bMQkTeBMdgpnXcAdwEfAO8AHbBTyF9gjGnRDbjVfA5jsNVLBtgE/MatHr5FEpERwNfACqDCtflObDtEYP1NBHqCUEop5VmgVzEppZSqhiYIpZRSHmmCUEop5ZEmCKWUUh5pglBKKeWRJgil6kBEyt1mNl3qzRmARSTdfSZVpfwtyN8BKNXMHDTG9Pd3EEo1Bi1BKOUFrrU0/uFaT+MHEeni2p4uIl+5Jrv7UkQ6uLYnisj7IrLM9TPMdSmniLzoWodgpoi09tsvpQKeJgil6qZ1lSqmC932FRhj+gBPYUfnAzwJvGqM6Qu8Djzh2v4EMNcY0w8YAKxybe8KPG2M6Q3sBc716W+jVA10JLVSdSAiB4wxER62bwJONMZsdE30tt0YEyciu4B2xphS1/Ztxph4EdkJpBhjDrldIx2Y5VqQBhH5ExBsjLm/EX41pY6hJQilvMdU87wuDrk9L0fbCZUfaYJQynsudHv83vX8O+wswQAXYyeBA7tk5RSwS9+KSFRjBalUbem3E6XqprWILHV7/bkxprKra4yILMeWAi5ybbsReFlE/gDsBK50bb8ZeEFErsKWFKZgF+RRqsnQNgilvMDVBpFljNnl71iU8hatYlJKKeWRliCUUkp5pCUIpZRSHmmCUEop5ZEmCKWUUh5pglBKKeWRJgillFIe/T89YckxDaQx+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## load model \n",
    "model = load_model(\"best_model.h5\")\n",
    "\n",
    "## get predictions on test set \n",
    "preds = model.predict(x_test)\n",
    "preds_class = np.argmax(preds, axis=1)\n",
    "obs = np.argmax(y_test, axis=1)\n",
    "\n",
    "## generate confusion matrix for best model \n",
    "cf = confusion_matrix(obs, preds_class)\n",
    "print(cf)\n",
    "\n",
    "## [[29  1  1]\n",
    " ## [ 0  6  3]\n",
    " ## [ 1  4  3]]\n",
    "\n",
    "## plot model accuracy on train and validation set \n",
    "with open('history_best_model.pkl', 'rb') as f:\n",
    "    history = pickle.load(f)\n",
    "plt.plot(history['accuracy'])\n",
    "plt.plot(history['val_accuracy'])\n",
    "plt.title('Model Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "## plot model loss on train and validation set \n",
    "with open('history_best_model.pkl', 'rb') as f:\n",
    "    history = pickle.load(f)\n",
    "plt.plot(history['loss'])\n",
    "plt.plot(history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "843f4a2f",
   "metadata": {},
   "source": [
    "## Evaluate first model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "38c081e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 1s 161ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Class 0       0.97      0.94      0.95        31\n",
      "     Class 1       0.55      0.67      0.60         9\n",
      "     Class 2       0.43      0.38      0.40         8\n",
      "\n",
      "    accuracy                           0.79        48\n",
      "   macro avg       0.65      0.66      0.65        48\n",
      "weighted avg       0.80      0.79      0.79        48\n",
      "\n"
     ]
    }
   ],
   "source": [
    "report = evaluate_model(model,x_test,y_test)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08681214",
   "metadata": {},
   "source": [
    "## Get model summary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eb78f57",
   "metadata": {},
   "source": [
    "###### model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e11e3d7",
   "metadata": {},
   "source": [
    "## Plot ROC curve of model on test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "f8d41804",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 1s 227ms/step\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfkAAAGDCAYAAAAoD2lDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAABRdElEQVR4nO3deZyN5f/H8dfHUmMPWUKFYmyzYBDKvoVSEUPWFC3aJS0iWvT7ftvzzVfZRSUVCcnWpmIwGGRJkiVbtrE1Zq7fH+eY72CGwZy5Z3k/H4/z6Nzn3Mt77mZ8znXf17kuc84hIiIiWU8OrwOIiIhIYKjIi4iIZFEq8iIiIlmUiryIiEgWpSIvIiKSRanIi4iIZFEq8iIiIlmUirxIJmJmW8zsmJnFmtlfZjbOzPKfsU49M1tgZofN7KCZfWlmVc5Yp6CZvWlmW/37+s2/fGUKxzUze9jMYszsiJltM7OpZhYSyJ9XRC6NirxI5nOLcy4/EA5UB54+9YaZ1QXmAtOBUkA5YCXwo5mV969zGTAfqAq0AgoCdYF9QO0UjvkW8AjwMFAEqAh8AbS50PBmlutCtxGRi2Ma8U4k8zCzLcA9zrl5/uX/A6o659r4l78HVjvnHjhju9nAHudcdzO7B3gJuM45F5uKY1YAfgXqOueWpLDOImCSc+4D/3JPf84b/csO6Ac8CuQC5gBHnHP9k+xjOvCtc+51MysFvAM0AGKBN5xzb5//DIlIUmrJi2RSZlYGuBnY5F/OC9QDpiaz+idAc//zZsCc1BR4v6bAtpQK/AW4DagDVAGmAJ3MzADMrDDQAvjIzHIAX+K7AlHaf/xHzazlJR5fJNtRkRfJfL4ws8PAn8BuYLD/9SL4/qZ3JrPNTuDU/faiKayTkgtdPyWvOOf+ds4dA74HHHCT/70OwE/OuR1ALaCYc26oc+4f59xm4H0gMg0yiGQrKvIimc9tzrkCQCOgEv8r3vuBBOCqZLa5Ctjrf74vhXVScqHrp+TPU0+c7z7hR0Bn/0tdgA/9z68FSpnZgVMP4BmgRBpkEMlWVORFMinn3LfAOODf/uUjwE/Ancms3hFfZzuAeUBLM8uXykPNB8qYWcQ51jkC5E2yXDK5yGcsTwE6mNm1+C7jT/O//ifwu3PuiiSPAs651qnMKyJ+KvIimdubQHMzC/MvDwR6+L/uVsDMCpvZi/h6z7/gX2civkI6zcwqmVkOMytqZs+Y2VmF1Dm3EfgPMMXMGpnZZWYWZGaRZjbQv1o0cIeZ5TWz64He5wvunFuB7+rCB8DXzrkD/reWAIfN7Ckzy2NmOc2smpnVuuCzI5LNqciLZGLOuT3ABOB5//IPQEvgDnz30f/A9zW7G/3FGufcCXyd734FvgEO4SusVwK/pHCoh4F3gRHAAeA34HZ8HeQA3gD+AXYB4/nfpffzmezPMjnJzxQPtMX3FcHf+d8HgUKp3KeI+OkrdCIiIlmUWvIiIiJZlIq8iIhIFqUiLyIikkWpyIuIiGRRKvIiIiJZVKabDerKK690ZcuW9TqGiIhIuli2bNle51yxi9k20xX5smXLEhUV5XUMERGRdGFmf1zstrpcLyIikkWpyIuIiGRRKvIiIiJZlIq8iIhIFqUiLyIikkWpyIuIiGRRKvIiIiJZlIq8iIhIFqUiLyIikkWpyIuIiGRRASvyZjbGzHabWUwK75uZvW1mm8xslZnVCFQWERGR7CiQLflxQKtzvH8zUMH/6AO8F8AsIiIi2U7AJqhxzn1nZmXPsUo7YIJzzgE/m9kVZnaVc25noDJlO1FjYfWnXqfIcHYdPs7e2BNex7gg8/L+w495//E6hoiks6N/X9rfvZf35EsDfyZZ3uZ/7Sxm1sfMoswsas+ePekSLktY/Sn8tdrrFBnO3tgTHP0n3usYF+THvP/wR+7MlVlELs2BP48xd9iGS9pHpphq1jk3ChgFEBER4TyOk7mUDIFeX3mdIkMZ+t+fAPi4b12Pk6Revjm9qAKMbTXW6ygikk5OnDjBAzseYMyYMRe9Dy9b8tuBq5Msl/G/JiIiki2tWbOGW265hQMHDnD55ZczevToS9qfl0V+BtDd38v+BuCg7seLiEh25Jxj1KhR1KpViyVLlrBp06Y02W/ALteb2RSgEXClmW0DBgO5AZxzI4FZQGtgE3AU6BWoLCIiIhnVgQMH6NOnD1OnTqVZs2ZMnDiRkiVLpsm+A9m7vvN53nfAg4E6voiISGbw8MMP8/nnnzN8+HCefPJJcuRIu4vsmaLjnYiISFaSkJDA4cOHKVSoEMOHD+fBBx+kTp06aX4cFXkREZF0tHPnTrp16wbA3LlzKVWqFKVKlQrIsTR2vYiISDqZPXs2YWFhLF68mM6dO2NmAT2eWvLpYPIvW5kenf7fDnx+30Hgf98LF5+1Ow9R5aqCXscQkWzkn3/+4emnn+b1118nNDSUjz76iMqVKwf8uGrJp4Pp0dtZu/OQ1zHEr8pVBWkXnuzgiiIiAXHkyBGmTZvGgw8+yC+//JIuBR7Ukk83Va4qmP4jrI0tBMDHvTLPyG4iIlnJjBkzaNWqFYULFyY6OporrrgiXY+vlryIiEgai42NpUePHrRr145Ro0YBpHuBB7XkRURE0tTy5cuJjIzkt99+Y/Dgwdx3332eZVGRFxERSSOTJ0+mV69eFCtWjAULFtCwYUNP8+hyvYiISBoJDQ2lXbt2rFy50vMCDyryIiIil2ThwoUMGDAAgGrVqvHJJ59QtGhRj1P5qMiLiIhchJMnT/Lcc8/RtGlTZsyYwYEDB7yOdBYVeRERkQv0xx9/0LBhQ1566SV69erFsmXLPOk9fz7qeCciInIB4uLiaNy4MXv37mXy5Ml07nzOSVc9pSIvIiKSCsePH+fyyy8nd+7c/Pe//6V8+fJcd911Xsc6J12uFxEROY81a9ZQq1Yt3nnnHQCaN2+e4Qs8qMiLiIikyDnHqFGjqFWrFrt37yY4ONjrSBdERV5ERCQZBw4coGPHjvTt25cbb7yRlStX0rJlS69jXRAVeRERkWRER0czY8YMXn31VebMmUPJkiW9jnTB1PFORETELz4+nh9//JEGDRrQqFEjfv/9d0qVKuV1rIumlryIiAiwY8cOWrRoQaNGjYiJiQHI1AUeVORFRESYNWsWYWFh/Pzzz4wePZqqVat6HSlNqMiLiEi29vTTT9OmTRtKlSrFsmXL6NWrF2bmdaw0oSIvIiLZWvHixenXrx+//PILlSpV8jpOmlLHOxERyXYmTpxIwYIFadeuHY899pjXcQJGLXkREck2Dh8+TPfu3enevTtjx471Ok7AqciLiEi2sGzZMmrUqMGHH37IkCFDmDZtmteRAk6X60VEJMv79ddfqVu3LiVKlGDhwoU0aNDA60jpQi15ERHJsuLj4wGoVKkSr732GtHR0dmmwIOKvIiIZFELFiwgODiYtWvXAvDQQw9RtGhRj1OlLxV5ERHJUk6ePMmzzz5Ls2bNyJUrFwkJCV5H8ozuyYuISJaxZcsWunTpwk8//cTdd9/N22+/Tb58+byO5RkVeREPTd0wlVmbZ513vfV/rye4SOaax1rECyNHjiQmJobJkyfTuXNnr+N4LtsX+cm/bGV69PaAHmPtzkNUuapgQI8hmdOszbNSVcCDiwTTunzrdEolkrkcPXqUP//8k+DgYF544QX69u1LuXLlvI6VIWT7Ij89envAi3CVqwrSLrx0wPYvmVtwkWDGtsr6g3KIBEJMTAyRkZEcPXqUX3/9lcsvv1wFPolsX+TBV4Q/7lvX6xgiIpJKzjlGjRrFo48+SsGCBZkwYQKXXXaZ17EyHBV5ERHJVI4cOUKPHj2YNm0azZs3Z8KECZQsWdLrWBmSvkInIiKZSlBQEIcPH+b//u//mDNnjgr8OaglLyIiGV58fDxvvPEGd911F1dddRWzZ88mRw61U89HRV5ERDK0HTt20LVrVxYuXEhCQgIDBgxQgU8lFXkREcmwvvrqK3r27MnRo0cZM2YMPXv29DpSpqKPQiIikiFNmDCBtm3bUrp0aZYtW0avXr0wM69jZSoq8iIikqE45wC45ZZbGDRoED///DOVKlXyOFXmpCIvIiIZxoQJE2jatCn//PMPhQsXZujQoQQFBXkdK9NSkRcREc8dPnyYbt260aNHD+Lj4zl06JDXkbIEFXkREfHUsmXLqFGjBpMnT2bIkCEsWLCAK6+80utYWYJ614uIiGecc9xzzz0cP36chQsX0qBBA68jZSkq8iIiku52795N3rx5yZ8/P5988glFihShaNGiXsfKcnS5XkRE0tX8+fMJCwvjscceA6BChQoq8AGiIi8iIukiLi6OZ599lubNm3PFFVfQr18/ryNlebpcLyIiAbd161YiIyP56aef6N27N2+99Rb58uXzOlaWpyIvIiIBl5CQwPbt25kyZQqRkZFex8k2dLleREQC4ujRo4wYMQLnHGXLlmXjxo0q8OksoEXezFqZ2Xoz22RmA5N5/xozW2hmK8xslZm1DmQeERFJHzExMdSuXZt+/fqxePFiAC677DKPU2U/ASvyZpYTGAHcDFQBOptZlTNWew74xDlXHYgE/hOoPCIiEnjOOUaOHEmtWrXYu3cvc+fOpX79+l7HyrYC2ZKvDWxyzm12zv0DfAS0O2MdBxT0Py8E7AhgHhERCbB+/fpx//3307BhQ1auXEnz5s29jpStBbLjXWngzyTL24A6Z6wzBJhrZg8B+YBmye3IzPoAfQCuueaaNA8qIiJp44477qBcuXI8/vjj5Mihbl9e8/r/QGdgnHOuDNAamGhmZ2Vyzo1yzkU45yKKFSuW7iFFRCR58fHxvPTSSwwdOhSApk2b0r9/fxX4DCKQ/xe2A1cnWS7jfy2p3sAnAM65n4AgQLMSiIhkAtu3b6d58+Y899xzbNy4MXEeeMk4AlnklwIVzKycmV2Gr2PdjDPW2Qo0BTCzyviK/J4AZhIRkTQwc+ZMwsLC+OWXXxg7diwTJkzAzLyOJWcI2D1559xJM+sHfA3kBMY459aY2VAgyjk3A3gCeN/MHsPXCa+n00dBEZEMbdu2bbRv357KlSvz0UcfUalSJa8jSQoCOuKdc24WMOuM155P8nwtoO9WiIhkAnv27KFYsWKUKVOG2bNnU69ePYKCgryOJeegnhEiInJOzjnGjx9PuXLl+OKLLwBo0qSJCnwmoCIvIiIpOnz4MN26daNnz57UrFmTiIgIryPJBVCRFxGRZC1btowaNWowZcoUXnjhBRYsWECZMmW8jiUXQLPQiYhIsmJiYjh+/DiLFi3ipptu8jqOXAS15EVEJNHu3buZM2cOAN27d2fdunUq8JmYWvIiIgLA/Pnz6dq1K//88w9//PEH+fPnJ3/+/F7HkkuglryISDYXFxfHM888Q/PmzSlcuDALFy5Ucc8i1JIXEcnGTpw4QePGjfnpp5+49957eeONN8iXL5/XsSSNqMiLiGRjl19+OU2bNuXRRx+lY8eOXseRNKYiL2lu6oapzNo86/wrCuv/Xk9wkWCvY0g2c/ToUZ544gl69OjBDTfcwLBhw7yOJAGie/KS5mZtnsX6v9d7HSNTCC4STOvyrb2OIdnI6tWrqVWrFiNHjmTx4sVex5EAU0teAiK4SDBjW431OoaI+DnnGDlyJI8//jiFChVi7ty5NG/e3OtYEmBqyYuIZANTp07lgQceoGHDhqxcuVIFPptQkRcRycJiY2MBaN++PR9++CGzZs2iRIkSHqeS9KIiLyKSBcXHxzNs2DAqVqzIX3/9Rc6cOenSpQs5cuif/exE9+RFRLKY7du307VrVxYtWkTnzp3Jmzev15HEIyryIiJZyMyZM+nZsyfHjh1j7Nix9OjRAzPzOpZ4REVeRCQLGTt2LFdffTUfffQRwcEagyG7U5EXEcnkNmzYQK5cuShfvjxjx47lsssuIygoyOtYkgGoB4aISCblnGPcuHHUqFGDBx54AICCBQuqwEsiFXkRkUzo0KFDdO3alV69ehEREcHo0aO9jiQZkC7Xi4hkMps2baJVq1b8/vvvDB06lGeeeYacOXN6HUsyIBV5EZFMplSpUlSoUIFx48Zx4403eh1HMjBdrhcRyQR2797N/fffT2xsLHnz5mX27Nkq8HJeKvIiIhncvHnzCAsLY+zYsSxZssTrOJKJqMiLiGRQcXFxPP3007Ro0YLChQuzdOlSmjRp4nUsyURU5EVEMqhHH32U4cOHc8899xAVFUVISIjXkSSTUcc7EZEMJi4ujty5c9O/f38aNWrEnXfe6XUkyaRU5EVEMoijR4/yyCOPsHv3br744gvKlStHuXLlvI4lmZgu14uIZACrV69OHNSmSpUqJCQkeB1JsgAVeRERDznneO+996hVqxb79+9n7ty5vPLKKxrcRtKEiryIiIcOHDjA0KFDady4MStXrqRZs2ZeR5IsRPfkRUQ8sGzZMsLCwihcuDA//fQT11xzDTlyqN0laUu/USIi6Sg+Pp6hQ4dSu3Zt3nrrLQDKli2rAi8BoZa8iEg62bZtG127duXbb7/lrrvu4t577/U6kmRxKvIiIungm2++ITIykhMnTjBu3Di6d++OmXkdS7I4FXkRkXRQuHBhKlSowIQJE6hYsaLXcSSbSPVNIDPLG8ggIiJZzfr163nttdcAiIiI4KefflKBl3R13iJvZvXMbC3wq385zMz+E/BkIiKZlHOOcePGUbNmTYYPH86ePXsAdHle0l1qWvJvAC2BfQDOuZVAg0CGEhHJrA4dOkTXrl3p1asXtWrVIjo6mmLFinkdS7KpVN2Td879ecYn0PjAxBERybzi4+Np0KABMTExDBs2jKeffloj14mnUlPk/zSzeoAzs9zAI8C6wMYSEck8EhISMDNy5szJs88+S6lSpahfv77XsURSdbn+PuBBoDSwHQgHHghgJhGRTGPXrl20bt2aMWPGAHDnnXeqwEuGkZoiH+ycu8s5V8I5V9w51xWoHOhgIiIZ3TfffENYWBiLFi3COed1HJGzpKbIv5PK10REsoW4uDgGDhxIy5YtKVKkCEuXLuWee+7xOpbIWVK8J29mdYF6QDEzezzJWwUB9SQRkWzrhx9+4NVXX+Xee+/lzTffJG9eDSMiGdO5Ot5dBuT3r1MgyeuHgA6BDCUikhFt2LCBihUr0rhxY1asWEF4eLjXkUTOKcUi75z7FvjWzMY55/5Ix0wiIhnKkSNHeOSRRxg/fjxLly4lPDxcBV4yhdR8he6omf0LqAoEnXrROdckYKlERDKIVatW0alTJ9avX8/AgQOpWrWq15FEUi01He8+xDekbTngBWALsDSAmUREMoSRI0dSu3ZtDhw4wDfffMPLL79M7ty5vY4lkmqpKfJFnXOjgTjn3LfOubsBteJFJMvbs2cPTZo0YeXKlTRt2tTrOCIXLDWX6+P8/91pZm2AHUCRwEUSEfHO999/T1xcHE2aNOGZZ57BzMiRI9UTdopkKKn5zX3RzAoBTwD9gQ+ARwMZSkQkvcXHxzN06FAaNWrE888/j3OOnDlzqsBLpnbe317n3Ezn3EHnXIxzrrFzribwd2p2bmatzGy9mW0ys4EprNPRzNaa2Rozm3yB+UVELtm2bdto2rQpgwcPpkuXLsyePVvTwkqWcK7BcHICHfGNWT/HORdjZm2BZ4A8QPVz7di//QigObANWGpmM5xza5OsUwF4GqjvnNtvZsUv9QcSEbkQW7ZsoWbNmpw4cYLx48fTvXt3ryOJpJlz3ZMfDVwNLAHeNrMdQAQw0Dn3RSr2XRvY5JzbDGBmHwHtgLVJ1rkXGOGc2w/gnNt9wT+BiMgluPbaa+nTpw+9evWiYsWKXscRSVPnulwfATR3zj0NtAba4mtxf5HKfZcG/kyyvM3/WlIVgYpm9qOZ/WxmrZLbkZn1MbMoM4vas2dPKg8vIpK89evX07BhQ3777TfMjFdeeUUFXrKkcxX5f5xzCQDOuePAZufcvjQ+fi6gAtAI6Ay8b2ZXnLmSc26Ucy7CORdRrFixNI4gItmFc46xY8dSo0YN1qxZw7Zt27yOJBJQ57pcX8nMVvmfG3Cdf9kA55wLPc++t+O73H9KGf9rSW0DfnHOxQG/m9kGfEVfg+2ISJo6dOgQ9913H1OmTKFRo0ZMmjSJ0qXPvLgokrWcq8hf6pzxS4EKZlYOX3GPBLqcsc4X+FrwY83sSnyX7zdf4nFFRM7yyiuv8Mknn/Diiy8ycOBAcubUZJqS9Z1rgppLmpTGOXfSzPoBX+ObmnaMc26NmQ0FopxzM/zvtTCztUA88GQAbgmISDaVkJDA7t27KVmyJM899xzt2rXjhhtu8DqWSLpJzYh3F805NwuYdcZrzyd57oDH/Q8RkTSza9cuevTowZYtW1ixYgX58uVTgZdsJ6BFXkTEC3PnzqV79+4cPHiQN998k6CgoPNvJJIFpWq8RjPLY2bBgQ4jInIp4uLieOqpp2jZsiVFixZl6dKl9O3bV6PXSbZ13iJvZrcA0cAc/3K4mc0IcC4RkYvy3Xff0bdvX5YuXUq1atW8jiPiqdRcrh+Cb/S6RQDOuWh/j3kRkQxh2rRpNGrUiKJFi7JgwQLy5MnjdSSRDCE1l+vjnHMHz3jNBSKMiMiFOHLkCPfccw8dOnTgtddeA1CBF0kiNS35NWbWBcjpn1DmYWBxYGOJiJzbqlWr6NSpE+vXr+eZZ55hyJAhXkcSyXBS05J/CKgKnAAmAwfRfPIi4qGZM2dSu3ZtDh48yDfffMNLL71E7ty5vY4lkuGkpiVfyTn3LPBsoMNIKkSNhdWfpm7dv1ZDyZDA5hHxQK1atejUqRP//ve/0XwWIilLTUv+NTNbZ2bDzExdVb22+lNf8U6NkiEQ0iGweUTSyXfffUeXLl04efIkJUqUYPz48SrwIudx3pa8c66xmZUEOgL/NbOCwMfOuRcDnk6SVzIEen3ldQqRdBEfH8+LL77I0KFDKV++PDt37uTqq68+/4YikrrBcJxzfznn3gbuw/ed+efPvYWIyKXbtm0bTZs2ZciQIdx1110sX75cBV7kApy3JW9mlYFOQHtgH/Ax8ESAc4lINueco0OHDsTExDB+/Hi6d+/udSSRTCc1He/G4CvsLZ1zOwKcR0SyuePHjwMQFBTEyJEjyZs3LxUrVvQ4lUjmdN7L9c65us65N1XgRSTQ1q1bR506dejfvz8A4eHhKvAilyDFIm9mn/j/u9rMViV5rDazVekXUUSyOuccY8aMISIigh07dtC6dWuvI4lkCee6XP+I/79t0yOIiGRPBw8e5L777uOjjz6iSZMmTJw4kVKlSnkdSyRLSLEl75zb6X/6gHPuj6QP4IH0iSciWd1ff/3F7Nmzeemll5g7d64KvEgaSs1X6Jon89rNaR1ERLKPhIQEpk+fjnOO4OBgfv/9d5555hly5szpdTSRLOVc9+TvN7PVQPAZ9+R/B3RPXkQuyq5du7j55pu57bbbmDdvHgCFCxf2OJVI1nSue/KTgdnAK8DAJK8fds79HdBUIpIlzZ07l27dunHo0CFGjhxJs2bNvI4kkqWd63K9c85tAR4EDid5YGZFAh9NRLKSl156iZYtW1KsWDGWLl1K3759MTOvY4lkaedrybcFlgEOSPrX6IDyAcwlIllMSEgIffv25fXXXydv3rxexxHJFlIs8s65tv7/lku/OCKSlUyZMoW9e/fy0EMPceutt3Lrrbd6HUkkWzlv73ozq29m+fzPu5rZ62Z2TeCjiUhmdeTIEXr37k2XLl2YNm0a8fHxXkcSyZZSM3b9e0CYmYXhm5jmA2Ai0DCQwSR9TN0wlVmbZ6XpPtf/vZ7gIsFpuk/JPFauXElkZCTr16/n2WefZciQIfpqnIhHUlPkTzrnnJm1A951zo02s96BDibpY9bmWWlelIOLBNO6vIYlzY727NlDvXr1KFSoEPPmzaNJkyZeRxLJ1lJT5A+b2dNAN+AmM8sB5A5sLElPwUWCGdtqrNcxJBM7fvw4QUFBFCtWjDFjxtCkSROKFSvmdSyRbC81I951Ak4Adzvn/gLKAP8KaCoRyTS+++47KlasyNdffw1Ap06dVOBFMojUTDX7F/AhUMjM2gLHnXMTAp5MRDK0kydPMmTIEBo3bpzYiheRjCU1ves7AkuAO4GOwC9m1iHQwUQk4/rzzz9p0qQJL7zwAl27dmXZsmXUqFHD61gicobU3JN/FqjlnNsNYGbFgHnAp4EMJiIZ16xZs1ixYgUTJ06ka9euXscRkRSk5p58jlMF3m9fKrcTkSzk+PHjLF26FIA+ffqwbt06FXiRDC41Lfk5ZvY1MMW/3AlI2y9Wi0iGtm7dOiIjI9myZQtbtmyhcOHClClTxutYInIeqel49yTwXyDU/xjlnHsq0MFExHvOOUaPHk1ERAQ7duxgypQpmhZWJBNJsSVvZhWAfwPXAauB/s657ekVTES8dfLkSbp168ZHH31EkyZNmDhxIqVKlfI6lohcgHO15McAM4H2+GaieyddEolIhpArVy4KFSrESy+9xNy5c1XgRTKhc92TL+Cce9//fL2ZLU+PQCLinYSEBF577TVatGhBWFgY7733nuZ8F8nEzlXkg8ysOv+bRz5P0mXnnIq+SBby119/0aNHD+bOncuePXsICwtTgRfJ5M5V5HcCrydZ/ivJsgM084RIFjF37ly6devGoUOH+O9//8u9997rdSQRSQMpFnnnXOP0DCIi3pg9ezatW7ematWqLFiwgKpVq3odSUTSiAa1Ecmm4uPjAWjWrBnDhw9n6dKlKvAiWYyKvEg2NGXKFEJDQ9m3bx+5c+fmqaeeIk+ePF7HEpE0piIvko3ExsZy991306VLFwoXLszx48e9jiQiAZSaWejMzLqa2fP+5WvMrHbgo4lIWoqOjiYiIoJx48bx3HPPsWjRIkqXLu11LBEJoNSMXf8fIAFfb/qhwGFgGlArgLlEJI0NHjyYw4cPM3/+fBo3Vr9akewgNUW+jnOuhpmtAHDO7TezywKcS0TSwL59+/jnn3+46qqr+OCDDzAzrrzySq9jiUg6Sc09+Tgzy4nvu/Gn5pNPCGgqEblk3377LWFhYXTv3h2AYsWKqcCLZDOpKfJvA58Dxc3sJeAH4OWAphKRi3by5EkGDx5MkyZNyJs3L6+++qrXkUTEI+e9XO+c+9DMlgFN8Q1pe5tzbl3Ak4nIBdu5cyedOnXi+++/p3v37rz77rsUKFDA61gi4pHzFnkzuwY4CnyZ9DXn3NZABhORC5cvXz4OHTrExIkT6dq1q9dxRMRjqel49xW++/EGBAHlgPWAhsYSyQCOHz/OG2+8wWOPPUbBggVZvnw5OXJoCAwRSd3l+pCky2ZWA3ggYIlEJNXWrVtHZGQkq1atolKlStx+++0q8CKS6IL/NfBPMVsnAFlEJJWcc4wePZqIiAh27tzJV199xe233+51LBHJYFJzT/7xJIs5gBrAjtTs3MxaAW8BOYEPnHPDU1ivPfApUMs5F5WafYtkZ4MHD2bYsGE0bdqUiRMnctVVV3kdSUQyoNTck0/aNfckvnv00863kf+79SOA5sA2YKmZzXDOrT1jvQLAI8AvqQ0tkl055zAzunTpQr58+ejfvz85c+b0OpaIZFDnLPL+Ql3AOdf/IvZdG9jknNvs39dHQDtg7RnrDQNeBZ68iGOIZAsJCQn861//Ys2aNYwfP55KlSpRqVIlr2OJSAaX4j15M8vlnIsH6l/kvksDfyZZ3uZ/LekxagBXO+e+ushjiGR5f/31Fy1btmTgwIEcP36cf/75x+tIIpJJnKslvwTf/fdoM5sBTAWOnHrTOffZpRzYzHIArwM9U7FuH6APwDXXXHMphxXJVObMmUP37t2JjY1l1KhR3HPPPZiZ17FEJJNIzT35IGAfvlnoTn1f3gHnK/LbgauTLJfxv3ZKAaAasMj/j1ZJYIaZ3Xpm5zvn3ChgFEBERIRLRWaRTO/w4cN069aNkiVLsmjRIqpUqeJ1JBHJZM5V5Iv7e9bH8L/ifkpqCu1SoIKZlcNX3COBLok7cO4gkDhbhpktAvqrd71kd3/++SelS5emQIECzJ07l0qVKpEnTx6vY4lIJnSu78nnBPL7HwWSPD/1OCfn3EmgH/A1sA74xDm3xsyGmtmtlxpcJCuaPHkyVatW5a233gKgevXqKvAictHO1ZLf6Zwbeik7d87NAmad8drzKazb6FKOJZKZxcbG8tBDDzFu3Djq16/PHXfc4XUkEckCztWSV+8ekXSwcuVKatasyfjx4xk0aBCLFi3i2muv9TqWiGQB52rJN023FJnE1A1TmbV51vlXDCTb5fvvnF5psrv1f68nuEhwmuxLLs7Bgwc5duwY8+fPp3Hjxl7HEZEsJMWWvHPu7/QMkhnM2jyL9X+v9zpGmgouEkzr8q29jpHt7Nu3j0mTJgHQoEEDNm7cqAIvImkuNV+hkySCiwQzttVY7wKMbeP7r5cZ5JJ8++233HXXXezdu5fGjRtTunRpLr/8cq9jiUgWpDkpRdLJyZMnGTx4ME2aNCFv3rwsXryY0qVLn39DEZGLpJa8SDpwztG6dWu++eYbevTowbvvvkv+/Of9JqqIyCVRkRdJB2bGnXfeSY8ePbjrrru8jiMi2YSKvEiAHDt2jP79+3PTTTcRGRnJvffe63UkEclmVORFAmDt2rVERkayevVqihcv7nUcEcmmVORF0pBzjtGjR/Pwww+TP39+Zs+eTatWrbyOJSLZlHrXi6ShH3/8kXvvvZf69euzcuVKFXgR8ZRa8iJpYN++fRQtWpQbb7yRmTNncvPNN5Mjhz5Di4i39K+QyCVISEhg+PDhlC1bllWrVgHQpk0bFXgRyRDUkhe5SDt37qR79+7MmzePjh07cs0113gdSUTkNCryIhdhzpw5dO/endjYWN5//3169+6NmSZuFJGMRUVe5CIsXLiQEiVKsGjRIqpUqeJ1HBGRZOnGoUgq/fbbbyxduhSAF198kSVLlqjAi0iGpiIvkgoffvgh1atX55577sE5R+7cucmTJ4/XsUREzklFXuQcYmNj6dmzJ127diU0NJQvv/xS995FJNPQPXmRFOzcuZNGjRqxceNGBg0axPPPP0+uXPqTEZHMQ/9iiaSgRIkS1KtXj5EjR9K4cWOv44iIXDBdrhdJYu/evXTr1o1t27aRI0cOxo4dqwIvIpmWiryI36JFiwgLC+OTTz5J7EUvIpKZqchLtnfy5EkGDRpEkyZNKFCgAL/88gu3336717FERC6Zirxkey+//DIvvvgiPXv2JCoqivDwcK8jiYikCXW8k2zr6NGj5M2bl0ceeYSqVavSvn17ryOJiKQpteQl2zl27BgPPPAA9erV4/jx4xQqVEgFXkSyJBV5yVbWrl1LnTp1eO+992jevLmmhBWRLE2X6yVbcM7x/vvv8+ijj5I/f35mz55Nq1atvI4lIhJQKvKSLcTFxfGf//yH+vXrM3HiREqWLOl1JBGRgFORlyxtyZIlVKpUiYIFCzJ37lyuvPJKXaIXkWxD/9pJlpSQkMArr7xCvXr1GDx4MADFixdXgReRbEUteclydu7cSbdu3Zg/fz4dO3ZMLPIiItmNirxkKT/++CO33347sbGxvP/++/Tu3VtTw4pItqUiL1nKNddcQ9WqVRkxYgRVqlTxOo6IiKd0g1IyvU2bNvHEE0+QkJDA1VdfzcKFC1XgRURQkZdMbtKkSVSvXp2xY8eyadMmr+OIiGQoKvKSKcXGxtKjRw+6detGeHg40dHRVKxY0etYIiIZioq8ZEq33XYbkyZNYvDgwSxcuJBrrrnG60giIhmOOt5JpuGcIz4+nly5cvHCCy8waNAgGjZs6HUsEZEMK9sX+f05v+NgziX0mlPwvOuu/3s9wUWC0yGVnGnv3r306tWLqlWrMnz4cOrXr+91JBGRDC/bX64/mHMJx+3PVK0bXCSY1uVbBziRnGnhwoWEhYUxd+5cypQp43UcEZFMI9u35AGC3NWMbTXW6xhyhpMnT/LCCy/w0ksvUbFiRb766ivCw8O9jiUikmlk+5a8ZFwbN27kX//6Fz179mTZsmUq8CIiF0gteclwli9fTo0aNahcuTIxMTFcf/31XkcSEcmU1JKXDOPYsWPcf//91KxZk1mzZgGowIuIXAK15CVDWLNmDZGRkcTExPDkk0/SrFkzryOJiGR6KvLiufHjx3P//fdToEAB5syZQ8uWLb2OJCKSJajIi+cuu+wy6tevz8SJEylZsqTXcUREsgwV+UCJGgurP037/f61GkqGpP1+09nixYvZvHkzXbt2pXPnzkRGRmredxGRNKaOd4Gy+lNfQU5rJUMgpEPa7zedxMfH8/LLL9OgQQNefvll4uLiAFTgRUQCQC35QCoZAr2+8jpFhrFjxw66devGggULiIyMZOTIkeTOndvrWCIiWZaKvKSLAwcOUL16dWJjYxk9ejS9evVS611EJMACernezFqZ2Xoz22RmA5N5/3EzW2tmq8xsvpldG8g8kv6ccwBcccUVDBo0iKioKO6++24VeBGRdBCwIm9mOYERwM1AFaCzmVU5Y7UVQIRzLhT4FPi/QOWR9Ldx40bq1q3L999/D0C/fv2oXLmyx6lERLKPQLbkawObnHObnXP/AB8B7ZKu4Jxb6Jw76l/8GdAUY1nEpEmTqFGjBhs2bCA2NtbrOCIi2VIgi3xpIOkcrtv8r6WkNzA7gHkkHcTGxtKjRw+6detG9erVWblyJTfffLPXsUREsqUM8RU6M+sKRAD/SuH9PmYWZWZRe/bsSd9wckEmTZrEpEmTGDJkCAsWLODqq6/2OpKISLYVyN7124Gk/8KX8b92GjNrBjwLNHTOnUhuR865UcAogIiICJf2UeVSOOfYvHkz1113HX369KFOnTpUr17d61giItleIFvyS4EKZlbOzC4DIoEZSVcws+rAf4FbnXO7A5hFAmTPnj3ccsst1K5dm927d5MjRw4VeBGRDCJgLXnn3Ekz6wd8DeQExjjn1pjZUCDKOTcD3+X5/MBU/1eqtjrnbg1UJklbCxcu5K677mLfvn289tprFCtWzOtIIiKSREAHw3HOzQJmnfHa80meaz7RTCghIYHnn3+el19+mYoVKzJr1izCw8O9jiUiImfIEB3vJHMxMzZt2kSvXr1YtmyZCryISAalYW0l1aZNm0a1atUIDg5m4sSJGndeRCSDU0tezuvYsWPcd999dOjQgVdffRVABV5EJBNQS17OKSYmhsjISNasWcOAAQMYNmyY15FERCSVVOQlRT/88APNmzenYMGCfP3117Ro0cLrSJLBxMXFsW3bNo4fP+51FJFMLygoiDJlyqTplVIVeUlRREQE99xzD88++ywlS5b0Oo5kQNu2baNAgQKULVtWMwuKXALnHPv27WPbtm2UK1cuzfare/JymsWLF9OsWTMOHTpEUFAQ77zzjgq8pOj48eMULVpUBV7kEpkZRYsWTfOrYiryAkB8fDwvv/wyDRo0YPPmzWzfftYIxCLJUoEXSRuB+FvS5Xphx44ddOvWjQULFhAZGcnIkSMpVKiQ17FEROQSqSUv9OvXj59//pnRo0czefJkFXjJVHLmzEl4eDjVqlXjlltu4cCBA4nvrVmzhiZNmhAcHEyFChUYNmwYzv1vjqvZs2cTERFBlSpVqF69Ok888YQHP8G5rVixgt69e3sdI0UnTpygU6dOXH/99dSpU4ctW7Yku95bb71FtWrVqFq1Km+++Wbi69HR0dxwww2Eh4cTERHBkiVLAJg5cybPP/98svs6ceIEzZo1Izw8nI8//jhNf578+fOn6f5Occ7x8MMPc/311xMaGsry5csDcpxkD5yZHjVr1nRpqfaYO1ztMXek6T6dc86Nae17ZFDHjx93f//9t3POua1bt7p169Z5nEgyo7Vr13odweXLly/xeffu3d2LL77onHPu6NGjrnz58u7rr792zjl35MgR16pVK/fuu+8655xbvXq1K1++fOLv/smTJ91//vOfNM0WFxd3yfvo0KGDi46OTtdjXogRI0a4vn37OuecmzJliuvYseNZ66xevdpVrVrVHTlyxMXFxbmmTZu6jRs3Oueca968uZs1a5ZzzrmvvvrKNWzY0DnnXEJCggsPD3dHjhw5a38//fSTa9q06QXlPHnyZKrWS/r7lJa++uor16pVK5eQkOB++uknV7t27WTXS+5vCt98LxdVM3W5PhvauHEjkZGRFC9enFmzZmnOd0kTL3y5hrU7DqXpPquUKsjgW6qmev26deuyatUqACZPnkz9+vUTv/qZN29e3n33XRo1asSDDz7I//3f//Hss89SqVIlwHdF4P777z9rn7GxsTz00ENERUVhZgwePJj27duTP39+YmNjAfj000+ZOXMm48aNo2fPngQFBbFixQrq16/PZ599RnR0NFdccQUAFSpU4IcffiBHjhzcd999bN26FYA333yT+vXrn3bsw4cPs2rVKsLCwgBYsmQJjzzyCMePHydPnjyMHTuW4OBgxo0bx2effUZsbCzx8fHMmjWLhx56iJiYGOLi4hgyZAjt2rVjy5YtdOvWjSNHjgDw7rvvUq9evVSf3+RMnz6dIUOGANChQwf69euHc+60+8vr1q2jTp065M2bF4CGDRvy2WefMWDAAMyMQ4d8vzcHDx6kVKlSgO/+dKNGjZg5cyYdO3ZM3Nfu3bvp2rUre/bsITw8nGnTprFlyxb69+/PyZMnqVWrFu+99x6XX345ZcuWpVOnTnzzzTcMGDCAyMjIxP3s2rWL++67j82bNwPw3nvvnXYuYmNjadeuHfv37ycuLo4XX3yRdu3aceTIETp27Mi2bduIj49n0KBBdOrUiYEDBzJjxgxy5cpFixYt+Pe//33WeerevTtmxg033MCBAwfYuXMnV1111SWd//NRkc9mJk6cyAMPPMBll13G888/r05TkmXEx8czf/78xEvba9asoWbNmqetc9111xEbG8uhQ4eIiYlJ1eX5YcOGUahQIVavXg3A/v37z7vNtm3bWLx4MTlz5iQ+Pp7PP/+cXr168csvv3DttddSokQJunTpwmOPPcaNN97I1q1badmyJevWrTttP1FRUVSrVi1xuVKlSnz//ffkypWLefPm8cwzzzBt2jQAli9fzqpVqyhSpAjPPPMMTZo0YcyYMRw4cIDatWvTrFkzihcvzjfffENQUBAbN26kc+fOREVFnZX/pptu4vDhw2e9/u9//5tmzU6fV2z79u2JDYVcuXJRqFAh9u3bx5VXXpm4TrVq1Xj22WfZt28fefLkYdasWURERAC+DzctW7akf//+JCQksHjx4sTtIiIi+P77708r8sWLF+eDDz7g3//+NzNnzuT48eM0atSI+fPnU7FiRbp37857773Ho48+CkDRokWTvTT+8MMP07BhQz7//HPi4+MTP7CdEhQUxOeff07BggXZu3cvN9xwA7feeitz5syhVKlSfPXVV4Dvg8m+ffv4/PPP+fXXXzGz024ZJXeeAMqUKcP27dtV5CVtHD58mAcffJCJEyfSoEEDJk2apBa8pKkLaXGnpWPHjhEeHs727dupXLkyzZs3T9P9z5s3j48++ihxuXDhwufd5s477yRnzpwAdOrUiaFDh9KrVy8++ugjOnXqlLjftWvXJm5z6NAhYmNjT7snvHPnztOmcD548CA9evRg48aNmBlxcXGJ7zVv3pwiRYoAMHfuXGbMmJHYmjx+/Dhbt26lVKlS9OvXj+joaHLmzMmGDRuSzf/999+f92e8EJUrV+app56iRYsW5MuXj/Dw8MTz89577/HGG2/Qvn17PvnkE3r37s28efMAX0HfsWPHOfe9fv16ypUrR8WKFQHo0aMHI0aMSCzyp873mRYsWMCECRMA31WcM/siOed45pln+O6778iRIwfbt29n165dhISE8MQTT/DUU0/Rtm1bbrrpJk6ePElQUBC9e/embdu2tG3b9qLPVVpTx7ts4sSJE3z77bcMGTKEBQsWqMBLlpEnTx6io6P5448/cM4xYsQIAKpUqcKyZctOW3fz5s3kz5+fggULUrVq1bPevxBJr4Kd+d3mfPnyJT6vW7cumzZtYs+ePXzxxRfccccdgG/K5p9//pno6Giio6PZvn37WZ2+8uTJc9q+Bw0aROPGjYmJieHLL7887b2kx3TOMW3atMR9b926lcqVK/PGG29QokQJVq5cSVRUFP/880+yP9tNN91EeHj4WY9TxTep0qVL8+effwJw8uRJDh48SNGiRc9ar3fv3ixbtozvvvuOwoULJxbl8ePHJ56TO++8M7Hj3anzmidPnmQzplbS83IhPvzwQ/bs2cOyZcuIjo6mRIkSHD9+nIoVK7J8+XJCQkJ47rnnGDp0KLly5WLJkiV06NCBmTNn0qpVq7P2l/Q8ge9qT+nSpS/650otFfkszDnHpEmTiIuL48orr2Tt2rUMHjw48RO0SFaSN29e3n77bV577TVOnjzJXXfdxQ8//JBYmI4dO8bDDz/MgAEDAHjyySd5+eWXE1uzCQkJjBw58qz9Nm/ePPGDA/zvcn2JEiVYt24dCQkJfP755ynmMjNuv/12Hn/8cSpXrpxYAFu0aME777yTuF50dPRZ21auXJlNmzYlLh88eDCxMIwbNy7FY7Zs2ZJ33nkn8ZsEK1asSNz+qquuIkeOHEycOJH4+Phkt//+++8TPyAkfZx5qR7g1ltvZfz48YCvb0KTJk2SvQ24e/duALZu3cpnn31Gly5dAChVqhTffvst4GtdV6hQIXGbDRs2nHa7IjnBwcFs2bIl8TxNnDiRhg0bnnMbgKZNm/Lee+8Bvls9Bw8ePO39gwcPUrx4cXLnzs3ChQv5448/AN9XjvPmzUvXrl158sknWb58ObGxsRw8eJDWrVvzxhtvsHLlymTP04QJE3DO8fPPP1OoUKGAX6oHFfksa/fu3bRt25Zu3boxefJk4OI/0YpkFtWrVyc0NJQpU6aQJ08epk+fzosvvkhwcDAhISHUqlWLfv36ARAaGsqbb75J586dqVy5MtWqVUvshJXUc889x/79+6lWrRphYWEsXLgQgOHDh9O2bVvq1at33n+sO3XqxKRJk067dPz2228TFRVFaGgoVapUSfYDRqVKlTh48GDi/fEBAwbw9NNPU716dU6ePJni8QYNGkRcXByhoaFUrVqVQYMGAfDAAw8wfvx4wsLC+PXXX9Pk34TevXuzb98+rr/+el5//XWGDx8O+Iph69atE9dr3749VapU4ZZbbmHEiBGJHRHff/99nnjiCcLCwnjmmWcYNWpU4jYLFy6kTZs25zx+UFAQY8eO5c477yQkJCSxQ+P5vPXWWyxcuJCQkBBq1qx52q0TgLvuuouoqChCQkKYMGFCYgfN1atXU7t2bcLDw3nhhRd47rnnOHz4MG3btiU0NJQbb7yR119//azjtW7dmvLly3P99ddz77338p///Oe8GdOCnfqkl1lERES45DqKXKw6Y9sD8EuvaWm2TwDG+n8xe32VtvtNhfnz59O1a1f279/Pa6+9xgMPPKAOdhIQ69ato3Llyl7HyNLeeOMNChQowD333ON1lHS1a9cuunTpwvz5872Okq6S+5sys2XOuYiL2Z9a8lnMu+++S/Pmzbniiiv45ZdfePDBB1XgRTKx+++/n8svv9zrGOlu69atvPbaa17HyPTUuz6LufHGG+nTpw+vvfaaLs+LZAFBQUF069bN6xjprlatWl5HyBLUks8CPv30U/r37w9AeHg4I0eOVIEXEREV+czs6NGj9O3blzvvvJPvv/+eo0ePeh1JREQyEBX5TComJobatWszatQoBgwYwA8//JA4ZKSIiAjonnymdOzYscTvq3799deJY3OLiIgkpZZ8JnL48GGcc+TJk4cPP/yQlStXqsBLtqepZr11qVPN/v333zRv3pwKFSrQvHnzxMGGstpUs7/++it169bl8ssvP2vymkBSkc8kFi9eTLVq1RIHzGjatCklSpTwOJWI904NaxsTE0ORIkUSR6c7duwYt956KwMHDmT9+vWsXLmSxYsXJw5CEhMTQ79+/Zg0aRJr164lKiqK66+/Pk2znWvAmtR6+eWXefjhh9P1mBdi9OjRFC5cmE2bNvHYY4/x1FNPnbVOTEwM77//PkuWLGHlypXMnDkzcYS64cOH07RpUzZu3EjTpk0TB9Np06YNX375ZbJ9jU6N4BcdHZ3i2PRnSml0v/RSpEgR3n777cRO0ulFRT6Di4+P56WXXqJBgwbkypXrrFm1RDKM2QN9g0Cl5WP2wAuKULduXbZv3w6kPNXsqSJyIVPN9urVi5CQEEJDQxNnfUva4vv000/p2bMnAD179uS+++6jTp06DBgwgLJly552daFChQrs2rWLPXv20L59e2rVqkWtWrX48ccfzzp2clPN1q1bl+rVq1OvXj3Wr18P+Ia4vfXWW2nSpAlNmzblyJEj3H333dSuXZvq1aszffp0ALZs2cJNN91EjRo1qFGjxmkzvl2s6dOn06NHD8A31ez8+fM5c5C1pFPN5sqVK3Gq2TO379GjB1988QVw+lSzSZ2aanbp0qWEh4fz22+/MX/+fKpXr05ISAh33303J06cAKBs2bI89dRT1KhRg6lTp562n127dnH77bcTFhZGWFjYWeciNjaWpk2bUqNGDUJCQhLP4ZEjR2jTpg1hYWFUq1Yt8UrCwIEDqVKlCqGhockW8uLFi1OrVi1y5859wef4UuiefAa2Y8cOunbtysKFC+ncuTMjR46kYMGCXscSyZA01WzmnGp2165dicMClyxZkl27diVul5WmmvWKinwGFhMTw9KlSxkzZgw9e/bUyHWSsd083JPDaqpZn8w61WxSZnbav3OaavbS6XJ9BnPixAnmzp0L+Gap2rJlC7169VKBF0mBppo9+5iZaarZEiVKsHPnTsD3oaZ48eKnndesMtWsV1TkM5ANGzZQt25dWrdunTgbVnJ/LCJyNk01+z+ZaarZpNuPHz+edu3aJW6Tlaaa9YxzLlM9atas6dJS7TF3uNpj7kjTfTrnnBvT2vdIpfHjx7t8+fK5IkWKuOnTp6d9HpEAWLt2rdcRXL58+U5bbtu2rZswYYJzzrlVq1a5hg0buooVK7rrrrvODRkyxCUkJCSu++WXX7oaNWq4SpUqucqVK7snn3zyrP0fPnzYde/e3VWtWtWFhoa6adOmOeecmzp1qitfvryrU6eOe/DBB12PHj2cc8716NHDTZ069bR9LF261AFu3Lhxia/t2bPHdezY0YWEhLjKlSu7vn37JvvzVatWzR06dMg559zixYtdhQoVXHh4uHv22Wfdtdde65xzbuzYse7BBx9M3Obo0aOuT58+rlq1aq5KlSquTZs2zjnnNmzY4EJCQlxoaKgbMGDAWefuYhw7dsx16NDBXXfdda5WrVrut99+c845t337dnfzzTcnrnfjjTe6ypUru9DQUDdv3rzE1/fu3euaNGnirr/+ete0aVO3b9++xPfatGnjVq1addYxFy5cmPgzOefcvHnzXHh4uKtWrZrr1auXO378uHPOuWuvvdbt2bMn2dx//fWXu/XWW121atVcWFiYW7x4sXPuf79Pe/bscTfccIOrVq2a69mzp6tUqZL7/fff3Zw5c1xISIgLCwtzERERbunSpW7Hjh2uVq1aLiQkxFWrVu20/8+n7Ny505UuXdoVKFDAFSpUyJUuXdodPHjwrPWS+5sCotxF1kxNNevxVLPOOXr37s3YsWNp2LAhkyZNokyZMmmbRSRANNVs4GmqWU01q6lmMzEzo0qVKrzwwgvMnz9fBV5ETqOpZuVSqHe9BxISEnjzzTepWLEibdu2TffBEUQk89BUs3Ip1JJPZ7t376Zt27Y88cQT5+ysIyIicqlU5NPR/PnzCQsLY8GCBYwYMYIPPvjA60giIpKF6XJ9Olm2bBnNmzenUqVKfP3114SGhnodSUREsji15APs1GATNWrUYOTIkSxdulQFXkRE0oWKfABNXbqT8uXLs2HDBsyMPn36XPToSyKSPE01663UTjX7xhtvULVqVapVq0bnzp0TR+vr3bs3YWFhhIaG0qFDh8Qx5N99913GjBmT7L727NlDnTp1qF69epoOwbtly5bzDr5zsV5//fXECWyaNm2aOLhOoKnIB8DRo0fpO341Hd9bQZkyZdJ91iGR7ERTzab9MS9Eaqaa3b59O2+//TZRUVHExMQQHx+fOB/AqRHiVq1axTXXXMO7774LwN13333aiIBJzZ8/n5CQEFasWMFNN92UqpxeTzVbvXp1oqKiWLVqFR06dEgceTHQdE8+jcXExBAZGcmaNX/y1M3lGTb9exV5yRZeXfIqv/79a5rus1KRSjxV++yikZK6deuyatUqIOWpZhs1asSDDz54QVPNPvTQQ0RFRWFmDB48mPbt25M/f/7EVuenn37KzJkzGTduHD179iQoKIgVK1ZQv359PvvsM6Kjo7niiisA31SzP/zwAzly5OC+++5j69atALz55pvUr1//tGMnN9XsI488kjim+9ixYwkODmbcuHF89tlnxMbGEh8fz6xZs3jooYeIiYkhLi6OIUOG0K5dO7Zs2UK3bt04cuQI4Gst16tXL9XnNznTp09nyJAhgG+q2X79+vlGWjtjaNuTJ09y7NgxcufOzdGjRylVqhRA4syazjmOHTuWuF3evHkpW7YsS5YsoXbt2on7iY6OZsCAARw7doyoqCh++uknvvjiC15++WWcc7Rp04ZXX30V8E0H3LdvX+bNm8eIESO48cYbE/ezadMm7rvvPvbs2UPOnDmZOnXqaZPmpHSudu7cSadOnTh06BAnT57kvffeo169evTu3Tvxd+Tuu+/mscceO+3nb9y4ceLzG264gUmTJl3SeU8tFfk09sEHH7B3717mPlGL5lWLgQq8SLrQVLMZd6rZ0qVL079/f6655hry5MlDixYtEj98AfTq1YtZs2ZRpUqV0wbAOTXVbNIiHx4eztChQ4mKiuLdd99lx44dPPXUUyxbtozChQvTokULvvjiC2677TaOHDlCnTp1kh1U56677mLgwIHcfvvtHD9+nISEhMTx9YEUz9XkyZNp2bIlzz77LPHx8Rw9ejRxgqGYmBiA8041O3r0aG6++eZzrpNWVOTTwP79+/nrr7+oXLkyr7zyCk8//TQlZt3tdSyRdHUhLe60pKlmfTLyVLP79+9n+vTp/P7771xxxRXceeedTJo0ia5duwIwduxY4uPjeeihh/j444/p1asX4Cu0v/567qtDS5cupVGjRonn6a677uK7777jtttuI2fOnLRv3/6sbQ4fPsz27du5/fbbAd+AQ2eKi4tL9lzVqlWLu+++m7i4OG677TbCw8MpX748mzdv5qGHHqJNmzanfYA506RJk4iKiuLbb79NxZm7dLonf4l++OEHwsLCuOOOO4iPjydPnjyUKFHC61gi2Yammj37mC6DTTU7b948ypUrR7FixcidOzd33HEHixcvPm2dnDlzEhkZmXhl4tR5vZSpZoOCgpKdtz41UjpXDRo04LvvvqN06dL07NmTCRMmULhwYVauXEmjRo0YOXJkivMMzJs3j5deeokZM2ak21DFKvIXKT4+nmHDhtGwYUNy587NhAkTLvqXSUQunaaa/Z+MNtXsNddcw88//8zRo0dxzjF//nwqV66Mcy7x53POMWPGjMQ+EpC6qWZr167Nt99+y969e4mPj2fKlCnnnWq2QIEClClThi+++ALwfUPg6NGjp62T0rn6448/KFGiBPfeey/33HMPy5cvZ+/evSQkJNC+fXtefPFFli9fftYxV6xYQd++fZkxYwbFixc/Z760pCJ/Efbv30+zZs14/vnniYyMZMWKFRpnWSQDqF69OqGhoUyZMoU8efIwffp0XnzxRYKDgwkJCaFWrVr069cPgNDQUN588006d+5M5cqVqVatGps3bz5rn8899xz79++nWrVqhIWFsXDhQgCGDx9O27ZtqVevHlddddU5c3Xq1IlJkyYlXqoHEnubh4aGUqVKlWQ/YFSqVImDBw8m3h8fMGAATz/9NNWrVz9nL/pBgwYRFxdHaGgoVatWZdCgQQA88MADjB8/nrCwMH799dc0+Upv79692bdvH9dffz2vv/46w4cPB3zzrrdu3RqAOnXq0KFDB2rUqEFISAgJCQn06dMH5xw9evQgJCSEkJAQdu7cyfPPP5+47x9//PG8t1+uuuoqhg8fTuPGjQkLC6NmzZqnzUmfkokTJ/L2228TGhpKvXr1+Ouvv057P6VztWjRIsLCwqhevToff/wxjzzyCNu3b6dRo0aEh4fTtWtXXnnllbOO9+STTxIbG8udd95JeHg4t95663kzpgVNNXsRU82ePHmSW265hU6dOtGjR4+zPrUCqZ5qViQz01SzgZddp5pdsWIFr7/+OhMnTvQ6SrrSVLMeOXHiBM899xy7d+8mV65czJo1i549eyZf4EVE0kh2nWp27969DBs2zOsYmZ5616fChg0bEi/LX3vttdx7770q7iKSLrLrVLNp/S2J7Eot+XNwzjF+/Hhq1KjB1q1bmTFjBvfee6/XsUQylMx2y08kowrE35KK/Dm888479OzZk4iICFauXMktt9zidSSRDCUoKIh9+/ap0ItcIucc+/btS/Y7+5dCl+uTkZCQQI4cOejSpQv//PMPjz32mL4eJ5KMMmXKsG3bNvbs2eN1FJFMLygoiDJlyqTpPgNa5M2sFfAWkBP4wDk3/Iz3LwcmADWBfUAn59yWQGY6l4SEBN58802mT5/OvHnzuPLKK+nfv79XcUQyvNy5c1OuXDmvY4hICgJ2ud7McgIjgJuBKkBnM6tyxmq9gf3OueuBN4BXA5XnfHbv3k3btm154oknKFKkCMeOHfMqioiISJoI5D352sAm59xm59w/wEfAmSMUtAPG+59/CjQ1D7qtH1y7m7CwMBYsWMCIESP47LPPEmdGEhERyawCebm+NPBnkuVtQJ2U1nHOnTSzg0BRYG8Ac52m+IntrPswmjKWm6+fqUVonq9gXBoMYPPXaigZcun7ERERuUiZouOdmfUB+vgXY81sfRof4sp1nNgbNviHNN7tn3B3tv0+/ZWk44e1bELnNO3pnKY9ndO0F3yxGwayyG8Hrk6yXMb/WnLrbDOzXEAhfB3wTuOcGwWMClBOzCzqYocMlOTpnKY9ndO0p3Oa9nRO056ZXfRY7oG8J78UqGBm5czsMiASmHHGOjOAHv7nHYAFTl+4FRERSRMBa8n777H3A77G9xW6Mc65NWY2FIhyzs0ARgMTzWwT8De+DwIiIiKSBgJ6T945NwuYdcZrzyd5fhy4M5AZUilgtwKyMZ3TtKdzmvZ0TtOezmnau+hzmummmhUREZHU0dj1IiIiWVS2KvJm1srM1pvZJjMbmMz7l5vZx/73fzGzsh7EzFRScU4fN7O1ZrbKzOab2bVe5MxMzndOk6zX3sycmakn83mk5pyaWUf/7+oaM5uc3hkzm1T87V9jZgvNbIX/77+1FzkzEzMbY2a7zSwmhffNzN72n/NVZlbjvDt1zmWLB77Of78B5YHLgJVAlTPWeQAY6X8eCXzsde6M/EjlOW0M5PU/v1/n9NLPqX+9AsB3wM9AhNe5M/Ijlb+nFYAVQGH/cnGvc2fkRyrP6Sjgfv/zKsAWr3Nn9AfQAKgBxKTwfmtgNmDADcAv59tndmrJZ5phdjOR855T59xC59xR/+LP+MZLkJSl5vcUYBi+uR6Op2e4TCo15/ReYIRzbj+Ac253OmfMbFJzTh1wanzwQsCOdMyXKTnnvsP3TbOUtAMmOJ+fgSvM7Kpz7TM7FfnkhtktndI6zrmTwKlhdiV5qTmnSfXG9ylUUnbec+q/RHe1cy4Nxl/OFlLze1oRqGhmP5rZz/4ZNCVlqTmnQ4CuZrYN37esHkqfaFnahf6bmzmGtZXMz8y6AhFAQ6+zZGZmlgN4HejpcZSsJhe+S/aN8F1t+s7MQpxzB7wMlcl1BsY5514zs7r4xkSp5pxL8DpYdpKdWvIXMswu5xpmVxKl5pxiZs2AZ4FbnXMn0ilbZnW+c1oAqAYsMrMt+O7LzVDnu3NKze/pNmCGcy7OOfc7sAFf0Zfkpeac9gY+AXDO/QQE4RvXXi5eqv7NTSo7FXkNs5v2zntOzaw68F98BV73Oc/vnOfUOXfQOXelc66sc64svn4OtzrnLnps62wgNX/7X+BrxWNmV+K7fL85HTNmNqk5p1uBpgBmVhlfkd+TrimznhlAd38v+xuAg865nefaINtcrncaZjfNpfKc/gvID0z192Hc6py71bPQGVwqz6lcgFSe06+BFma2FogHnnTO6SpeClJ5Tp8A3jezx/B1wuupRtO5mdkUfB82r/T3ZRgM5AZwzo3E17ehNbAJOAr0Ou8+dc5FRESypux0uV5ERCRbUZEXERHJolTkRUREsigVeRERkSxKRV5ERCSLUpEX8YCZxZtZdJJH2XOsG5sGxxtnZr/7j7XcPwLZhe7jAzOr4n/+zBnvLb7UjP79nDovMWb2pZldcZ71wzW7mUjK9BU6EQ+YWaxzLn9ar3uOfYwDZjrnPjWzFsC/nXOhl7C/S850vv2a2Xhgg3PupXOs3xPfLHz90jqLSFaglrxIBmBm+c1svr+VvdrMzpp5zsyuMrPvkrR0b/K/3sLMfvJvO9XMzld8vwOu92/7uH9fMWb2qP+1fGb2lZmt9L/eyf/6IjOLMLPhQB5/jg/978X6//uRmbVJknmcmXUws5xm9i8zW+qfB7tvKk7LT/gn3zCz2v6fcYWZLTazYP9Ia0OBTv4snfzZx5jZEv+6yc3gJ5JtZJsR70QymDxmFu1//jtwJ3C7c+6Qf1jVn81sxhkjhHUBvnbOvWRmOYG8/nWfA5o5546Y2VPA4/iKX0puAVabWU18I2bVwTc/9S9m9i2+OcJ3OOfaAJhZoaQbO+cGmlk/51x4Mvv+GOgIfOUvwk2B+/GNY37QOVfLzC4HfjSzuf5x4s/i//ma4huFEuBX4Cb/SGvNgJedc+3N7HmStOTN7GV8w1Hf7b/Uv8TM5jnnjpzjfIhkWSryIt44lrRImllu4GUzawAk4GvBlgD+SrLNUmCMf90vnHPRZtYQqIKvaAJchq8FnJx/mdlz+MYP742viH5+qgCa2WfATcAc4DUzexXfJf7vL+Dnmg285S/krYDvnHPH/LcIQs2sg3+9QvgmgDmzyJ/68FMaWAd8k2T98WZWAd8QqblTOH4L4FYz6+9fDgKu8e9LJNtRkRfJGO4CigE1nXNx5pthLijpCs657/wfAtoA48zsdWA/8I1zrnMqjvGkc+7TUwtm1jS5lZxzG8w3Z31r4EUzm++cO9eVgaTbHjezRUBLoBPw0anDAQ85574+zy6OOefCzSwvvnHRHwTeBoYBC51zt/s7KS5KYXsD2jvn1qcmr0hWp3vyIhlDIWC3v8A3Bq49cwUzuxbY5Zx7H/gAqIFvFrr6ZnbqHns+M6uYymN+D9xmZnnNLB9wO/C9mZUCjjrnJuGbYKhGMtvG+a8oJOdjfLcBTl0VAF/Bvv/UNmZW0X/MZDnnjgIPA0/Y/6Z9PjWlZs8kqx7GN/3uKV8DD5n/sob5ZkEUybZU5EUyhg+BCDNbDXTHdw/6TI2AlWa2Al8r+S3n3B58RW+Kma3Cd6m+UmoO6JxbDowDlgC/AB8451YAIfjuZUfjmwXrxWQ2HwWsOtXx7gxzgYbAPOfcP/7XPgDWAsvNLAbf9MPnvJLoz7IK6Az8H/CK/2dPut1CoMqpjnf4Wvy5/dnW+JdFsi19hU5ERCSLUkteREQki1KRFxERyaJU5EVERLIoFXkREZEsSkVeREQki1KRFxERyaJU5EVERLIoFXkREZEs6v8BwwZZyp0sCjYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_roc_curve(model, x_test, y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}