ood_cifar10_vs_cifar100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import time\n",
    "from pprint import pprint\n",
    "from pathlib import Path\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar10.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequently used variables\n",
    "model_name = config.model.name\n",
    "patch_size = config.model.patch_size\n",
    "summary = config.summary\n",
    "\n",
    "# log directories\n",
    "log_root = Path('./logs') / model_name / summary\n",
    "checkpoint_path = log_root / 'checkpoints'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ViT model and load pretrained weights\n",
    "from ood_with_vit.models.vit import ViT\n",
    "\n",
    "def initialize_vit_model(config, verbose=0):\n",
    "    n_class = config.dataset.n_class\n",
    "    if config.model.pretrained:\n",
    "        model = torch.hub.load(\n",
    "            repo_or_dir=config.model.repo,\n",
    "            model=config.model.pretrained_model,\n",
    "            pretrained=False,\n",
    "        )\n",
    "        model.head = nn.Linear(model.head.in_features, n_class)\n",
    "    else:\n",
    "        model = ViT(\n",
    "            image_size=config.model.img_size,\n",
    "            patch_size=config.model.patch_size,\n",
    "            num_classes=n_class,\n",
    "            dim=config.model.dim_head,\n",
    "            depth=config.model.depth,\n",
    "            heads=config.model.n_heads,\n",
    "            mlp_dim=config.model.dim_mlp,\n",
    "            dropout=config.model.dropout,\n",
    "            emb_dropout=config.model.emb_dropout,\n",
    "            visualize=True,\n",
    "        )\n",
    "\n",
    "    model = model.to(device=device)\n",
    "    if verbose:\n",
    "        print(model)\n",
    "\n",
    "    checkpoint = torch.load(checkpoint_path / f'{summary}_best.pt')\n",
    "\n",
    "    state_dict = checkpoint['model_state_dict']\n",
    "    trimmed_keys = []\n",
    "    for key in state_dict.keys():\n",
    "        # remove prefix 'module.' for each key (in case of DataParallel)\n",
    "        trimmed_keys.append(key[7:])\n",
    "    trimmed_state_dict = OrderedDict(list(zip(trimmed_keys, state_dict.values())))\n",
    "\n",
    "    model.load_state_dict(trimmed_state_dict)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n",
      "{'apple': 0, 'aquarium_fish': 1, 'baby': 2, 'bear': 3, 'beaver': 4, 'bed': 5, 'bee': 6, 'beetle': 7, 'bicycle': 8, 'bottle': 9, 'bowl': 10, 'boy': 11, 'bridge': 12, 'bus': 13, 'butterfly': 14, 'camel': 15, 'can': 16, 'castle': 17, 'caterpillar': 18, 'cattle': 19, 'chair': 20, 'chimpanzee': 21, 'clock': 22, 'cloud': 23, 'cockroach': 24, 'couch': 25, 'crab': 26, 'crocodile': 27, 'cup': 28, 'dinosaur': 29, 'dolphin': 30, 'elephant': 31, 'flatfish': 32, 'forest': 33, 'fox': 34, 'girl': 35, 'hamster': 36, 'house': 37, 'kangaroo': 38, 'keyboard': 39, 'lamp': 40, 'lawn_mower': 41, 'leopard': 42, 'lion': 43, 'lizard': 44, 'lobster': 45, 'man': 46, 'maple_tree': 47, 'motorcycle': 48, 'mountain': 49, 'mouse': 50, 'mushroom': 51, 'oak_tree': 52, 'orange': 53, 'orchid': 54, 'otter': 55, 'palm_tree': 56, 'pear': 57, 'pickup_truck': 58, 'pine_tree': 59, 'plain': 60, 'plate': 61, 'poppy': 62, 'porcupine': 63, 'possum': 64, 'rabbit': 65, 'raccoon': 66, 'ray': 67, 'road': 68, 'rocket': 69, 'rose': 70, 'sea': 71, 'seal': 72, 'shark': 73, 'shrew': 74, 'skunk': 75, 'skyscraper': 76, 'snail': 77, 'snake': 78, 'spider': 79, 'squirrel': 80, 'streetcar': 81, 'sunflower': 82, 'sweet_pepper': 83, 'table': 84, 'tank': 85, 'telephone': 86, 'television': 87, 'tiger': 88, 'tractor': 89, 'train': 90, 'trout': 91, 'tulip': 92, 'turtle': 93, 'wardrobe': 94, 'whale': 95, 'willow_tree': 96, 'wolf': 97, 'woman': 98, 'worm': 99}\n"
     ]
    }
   ],
   "source": [
    "from torchvision.datasets import CIFAR10, CIFAR100\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "dataset_mean, dataset_std = config.dataset.mean, config.dataset.std\n",
    "dataset_root = config.dataset.root\n",
    "img_size = config.model.img_size\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "\n",
    "id_dataset = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_test_dataloader = DataLoader(\n",
    "    dataset=id_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "id_dataset_name = 'cifar10'\n",
    "\n",
    "ood_dataset = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "ood_test_dataloader = DataLoader(\n",
    "    dataset=ood_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "ood_dataset_name = 'cifar100'\n",
    "\n",
    "print(id_dataset.class_to_idx)\n",
    "print(ood_dataset.class_to_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seeds\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "seed = 1234"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "# add hooks for feature extraction\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 76.22it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 97.89it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.19it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.69it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 107.62it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.33it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.31it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 105.17it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.45it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 99.59it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 100.19it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.13it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 81.30it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 107.11it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 106.36it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.28it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.54it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.24it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.47it/s]\n"
     ]
    }
   ],
   "source": [
    "# gather penultimate features\n",
    "from ood_with_vit.utils import compute_penultimate_features\n",
    "\n",
    "set_seed(seed)\n",
    "num_samples = 20\n",
    "num_class = 10\n",
    "\n",
    "print('processing in-distribution samples...')\n",
    "X_in, y_in = [], []\n",
    "in_img_paths = []\n",
    "cls_indices = random.sample(range(len(id_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(id_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = id_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_in.append(penultimate_features.squeeze().numpy())\n",
    "        y_in.append((cls_idx, id_dataset.classes[cls_idx]))\n",
    "        img_path = f'./assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        in_img_paths.append(img_path)\n",
    "        \n",
    "print('processing out-of-distribution samples...')\n",
    "X_out, y_out = [], []\n",
    "out_img_paths = []\n",
    "cls_indices = random.sample(range(len(ood_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(ood_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = ood_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_out.append(penultimate_features.squeeze().numpy())\n",
    "        y_out.append((cls_idx, ood_dataset.classes[cls_idx]))\n",
    "        img_path = f'./assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        out_img_paths.append(img_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import HTML, VBox\n",
    "\n",
    "def interactive_plot(partitioned_info, fig, event=\"click\") :\n",
    "    \"\"\"\n",
    "    Make a plot react on hover or click of a data point and update a HTML preview below it.    \n",
    "    \"\"\"\n",
    "    html = HTML(\"\")\n",
    "    fig = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    if event == \"hover\" :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\">\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_hover(update)\n",
    "    else :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\">\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_click(update)\n",
    "\n",
    "    return VBox([fig, html])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb4e7c7b5a5545e8979f2341c3a3aab8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(7, 'horse')<br>x=%{x}<br>y=%{y}<extra></e…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e7886d76c5a49509da8a0b103c356ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(58, 'pickup_truck')<br>x=%{x}<br>y=%{y}<e…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ab0ad047b64485e9f55138eebab8a47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(7, 'horse')<br>x=%{x}<br>y=%{y}<extra></e…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR10 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c78e69c7ef948fcaa0b05982927aa3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(58, 'pickup_truck')<br>x=%{x}<br>y=%{y}<e…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR100 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ood_with_vit.utils import compute_ood_scores\n",
    "from ood_with_vit.utils.ood_metrics import auroc, aupr, fpr_at_95_tpr\n",
    "from ood_with_vit.utils.visualization import plot_roc_curve, plot_precision_recall_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OOD detection by MSP (Maximum over Softmax Probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.msp import MSP\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "msp = MSP(config, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:05<00:00, 17.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: -0.9825615\n",
      "ood mean score: -0.7670236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=msp,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/kElEQVR4nO3dd5gUVdbA4d9hyDmqSI7CEEQYQUSSKEkQXRNmdlEWAcE17KdiZM2CAQUEBUFURFkDCoIYcU1kkCASJIMMSBqGMOF8f9waaYcJDUx3TXef93n6qe6q6qpTHepU3Vt1r6gqxhhjYlcBvwMwxhjjL0sExhgT4ywRGGNMjLNEYIwxMc4SgTHGxDhLBMYYE+MsEUQ5EVkhIh38jiO/EJH7ReQ1n9Y9UUQe82PdeU1ErheRz07yvfabzGcsEYSRiGwQkUMikiQiO7wdQ8lQrlNVG6nq16FcRwYRKSIiT4rIJm8714jIPSIi4Vh/FvF0EJEtgeNU9QlVvSVE6xMRGSwiy0XkoIhsEZH3RKRJKNZ3skTkERF581SWoapvqWrnINZ1XPI72d+kiBT2Yl/jfb4bRGSCiNQ80WWZv7JEEH49VbUk0Aw4B7jP33BOnIgUzGbSe0AnoDtQCrgR6Ae8GIIYRETy2+/3RWAIMBgoD9QHPgQuyesV5fAdhJyP654GXApcB5QBzgYW4n5zJ8TPzy9fUlV7hOkBbAAuCnj9DDAj4PV5wPfAXmAp0CFgWnngdWAbsAf4MGBaD2CJ977vgaaZ1wmcCRwCygdMOwfYBRTyXv8DWOUtfzZQI2BeBQYCa4Dfsti2TsBhoFqm8a2ANKCu9/pr4ElgHrAf+ChTTDl9Bl8DjwPfedtSF/i7F/MBYD3wT2/eEt486UCS9zgTeAR405unprddNwObvM9iaMD6igGTvM9jFfBvYEs23209bztb5vD9TwRGATO8eH8C6gRMfxHY7H0uC4G2AdMewe0I3/Sm3wK0BH7wPqvtwMtA4YD3NALmAH8AvwP3A12Bo0CK95ks9eYtA4z3lrMVeAyI86b18T7z54Hd3rQ+wP+86eJN2+nF9jPQGHcQkOKtLwn4OPP/AIjz4lrnfSYLyfQb8ua7yPs+j5uWw/8rq++6r/ddzwU+BQZlWsZS4G/e8wYBn99q4Gq/9yEh2zf5HUAsPTL9Aap6f5gXvddVvD9Zd9yZ2sXe60re9BnAVKAcUAho740/x/sDtvL+VDd76ymSxTq/BG4NiOdZ4BXveS9gLdAQKAg8AHwfMK96f4ryQLEstu0p4Jtstnsjx3bQX3s7msa4nfV/A/6suX0GX3t/4kZejIVwR9t1cDuj9kAy0NybvwOZdtzZ7Bxexe30zwaOAA0Dt8n7zKsCyzIvL2C5/YGNuXz/E73taenF/xbwTsD0G4AK3rS7gB1A0YC4U4DLvM+mGNAClzgLetuyCrjDm78Ubqd+F1DUe90q82cQsO4PgLHed3IaLlFnfGd9gFTgdm9dxfhrIuiC24GX9b6HhkDlgG1+LIf/wT24/8FZ3nvPBiqcyO8rq+Xm8F2/4W1jMeAm4LuA+eNxSbWIN89m3IFGQY4dNMX7vR8JxSO/nVrHgg9F5ADuR7YTeNgbfwMwU1Vnqmq6qs4BFgDdRaQy0A3or6p7VDVFVb/x3tcPGKuqP6lqmqpOwu3Mzsti3W8D14IrWgF6e+PA7cieVNVVqpoKPAE0E5EaAe9/UlX/UNVDWSy7Im7Hk5Xt3vQMk1V1uaoeBB4ErhaRuJw+g4D3TlTVFaqa6n0OM1R1nTrfAJ8BbbOJIzuPquohVV2KOyI82xt/NfCE95lvAUbmsIwKOWx/oA9UdZ73Gb+FKyIEQFXfVNXd3raNwO2Qzgp47w+q+qH32RxS1YWq+qM3/wbcjry9N28PYIeqjlDVw6p6QFV/yiogETkd9xnfoaoHVXUn7gi/d8Bs21T1JW9dmb//FFyiaQCI9xsK5rMAd2bzgKqu9r7Dpaq6O4v5gv18c/OIt42HcMkv8Dd+PfC+qh7BfX4bVPV1b5sX4w5arsqDGPIdSwThd5mqlsIdrTbg2A6yBnCViOzNeAAXAJWBasAfqroni+XVAO7K9L5quGKQzP4LtPYSSztcscm3Act5MWAZf+CO0KoEvH9zDtu1y4s1K5W96VktZyPuyL4iOX8GWcYgIt1E5EcR+cObvzt/TTrB2BHwPBnIqMA/M9P6ctr+3WS//cGsCxG5W0RWicg+b1vK8Ndtybzt9UXkE+/Cg/245J0xfzVccUswauC+g+0Bn/tY3JlBlusOpKpf4oqlRgE7RWSciJQOct3Bxhns55ubP7dDVQ/gzrQzEt61uOQM7jNplem3eD1wRh7EkO9YIvCJd/Q6ERjujdqMO1IuG/AooapPedPKi0jZLBa1GXg80/uKq+qULNa5B3fEfA2uwu0dVdWA5fwz03KKqer3gYvIYZM+x/1xqgWOFJFWuD/7lwGjA+epjjui3JXLZ3BcDCJSBJfchgOnq2pZYCYugeUWbzC244qEsoo7sy+AqiKScDIrEpG2uDqIq4Fy3rbs49i2wPHbMwb4BainqqVxZe0Z828GamezuszL2Yw7i6wY8LmXVtVGObznrwtUHamqLXDFK/VxRT65vs9bd51c5gH3+2opIlVzmOcgUDzgdVY77czxTAGuFZHWuCK0rwLi+ibTb7Gkqt4WRKwRxxKBv14ALhaRs3GVgD1FpIuIxIlIUe/yx6reafanwGgRKScihUSknbeMV4H+ItLKu5KmhIhcIiKlslnn27iy0Ss5ViwE8Apwn4g0AhCRMiIS9Gmwqn6O2xn+V0QaedtwnrddY1R1TcDsN4hIvIgUB4YB01Q1LafPIJvVFsYVnyQCqSLSDQi8pPF3oIKIlAl2OzJ5F/eZlBORKsCg7Gb0tm80MMWLubAXf28RuTeIdZXClcMnAgVF5CEgt6PqUrjK2SQRaQAE7qQ+ASqLyB3iLust5SVlcJ9LzYyrrrzf12fACBEpLSIFRKSOiLQnCCJyrvf7K4TbGR/GnW1mrCu7hATwGvAfEann/X6bikiFzDN5v685wAci0kJECnrb1F9E/uHNtgTo7f0/EnC/8dzMxB39DwOmqmpG3J8A9UXkRm95hbztbBjEMiOOJQIfqWoirvLqIVXdjKuwvR+3M9iMO6rK+I5uxB05/4KrW7jDW8YC4FbcqfkeXIVvnxxWOx13hcsOr0w8I5YPgKeBd7xihuW4eokTcQXuiGoW7iqRN3FXotyeab7JuLOhHbijsMFeDLl9Bn/hndoPxu2w9+DOcqYHTP8Fd8S33ju9z6q4LCfDgC3Ab7gj0mm4I+fsDOZYEcleXJHH5cDHQaxrNu5z+xVXXHaYnIuiAO7GbfMB3AHB1IwJ3mdzMdAT9zmvATp6k9/zhrtFZJH3/CZcYl2J+yynEXxRTGlv/Xu82HfjLkQA9/3He5//h1m89znc9/cZLqmNx1XkZuVK3I57Ku5saTmQgPtuwNU31fHieJS/HuhkyasPeB93VdLbAeMP4A4qeuOu1NuB+38UyW2ZkUiOlQwYE3oi8jXuSg5f7u49FSJyG9BbVYM6UjYmUtgZgTHZEJHKItLGKyo5C3cp5gd+x2VMXrO764zJXmHc1TO1cEU97+DqAYyJKlY0ZIwxMc6KhowxJsZFXNFQxYoVtWbNmn6HYYwxEWXhwoW7VLVSVtMiLhHUrFmTBQsW+B2GMcZEFBHZmN00KxoyxpgYZ4nAGGNinCUCY4yJcRFXR5CVlJQUtmzZwuHDh/0OJWSKFi1K1apVKVSokN+hGGOiTFQkgi1btlCqVClq1qyJ+NM9bkipKrt372bLli3UqlXL73CMMVEmZEVD4jqV3ikiy7OZLiIyUkTWisgyEWl+sus6fPgwFSpUiMokACAiVKhQIarPeIwx/gllHcFEXP+o2emGawWzHq6XrTGnsrJoTQIZon37jDH+CVnRkKrOFZGaOczSC3jD6xjlRxEpKyKVT6CLO2NMNFOF1FQ4fPjY48gRN0xPP7FHWhocOuSGgeMChxnP09IgJeVYDBnN8AQzDNE8B/UIiSRTs+eNcO65efDh/pWfdQRV+Gt761u8ccclAhHphztroHr16mEJ7kTFxcXRpEkTUlNTqVWrFpMnT6Zs2bIArFixgttvv52tW7eSnp7OTTfdxAMPPPDnUf6nn37Kgw8+SHJyMkWKFOHCCy9kxIgRPm6NiRhpaXDgABw96h5HjrgdXsbrwEfGTi419djzjEdSkhsfOC3jeUqKe396utsxZd7JZjcuLQ0OHnTvz1h2xvPDh90ys5p25Mixh7WFxpe14NaeUOYILFjQgAJRlgiCpqrjgHEACQkJ+fKXUaxYMZYsWQLAzTffzKhRoxg6dCiHDh3i0ksvZcyYMXTu3Jnk5GSuuOIKRo8ezcCBA1m+fDmDBg1ixowZNGjQgLS0NMaNG+fvxphTl5YGO3fCrl2wfbvbWW/a5HZs+/a510eOHNt5HzkCycnHP5KS3A4y84474xGOHWVcHBQpAgUK/PUhkvu4EiWgUCEoWPDYsFgxKF/eLbNgweMfhQtD0aJuepEi7nmxYn99HRd3/LpzexQt6pZfoMCx9wcuJ3BcoUJuW8ANA5/nNsyjefYe2cc9n93Da4tfo275ujzf8zUK1AxNVxh+JoKt/LUP2KreuIjXunVrli1bBsDbb79NmzZt6NzZ9aBYvHhxXn75ZTp06MDAgQN55plnGDp0KA0aNADcmcVtt0Vlt6j5R0oK7NjhjkoTE90wJeXY48gRWLHC7WQzdtRHj7odc2DxRMbrdevcDi/jSDc52c2TkxIl3I6pcOFjO7jixd2jZEk444xjrwsXdjun7B6lSx9bVuHCx96T8ShU6K/LKFjw+GWUKHFsnsDpGTt3E1Zp6WmcP/58Vu9ezb/P/zePdHiEYoWy67jt1PmZCKYDg0TkHaAVsC9P6gfuuAO8I/M806wZvPBCULOmpaXxxRdf0LdvX8AVC7Vo0eIv89SpU4ekpCT279/P8uXLueuuu/I23lhz+DBs2wa//w4bNrgj8L173SMx0f0eiheHVauOFT8EK+PItHBhd1QaeKRaogRUqABVqrij/0aN3PSiRd20kiWhenWoWBHKlXPvP/10t+MuYPdymuPtTt5N+WLliSsQx+MXPk61MtVIODMh5OsNWSIQkSlAB6CiiGwBHgYKAajqK7i+R7vj+thNBv4eqljC4dChQzRr1oytW7fSsGFDLr74Yr9DinxJSa445eBBV5Sybh0sWwZbt8L+/fDbb7B+ffbvL1HC7YArVXLFKBdd5I52zzwTypaFhg3dDl0EKld2R84ZjwoV3E7bjoZNGKgqb/38FkNmDeGpTk9xa4tbubzh5WFbfyivGro2l+kKDMzzFQd55J7XMuoIkpOT6dKlC6NGjWLw4MHEx8czd+7cv8y7fv16SpYsSenSpWnUqBELFy7k7LPP9iXusFN1R+yrV7vimfnz3U56xQooVcodwa9b56blpFQp6NYNevVyR/lnn+128KefDrVquQRgO3ETATbv20z/Gf2ZuWYm51U9jzbV24Q/CFWNqEeLFi00s5UrVx43LtxKlCjx5/NFixZp9erVNSUlRZOTk7VWrVo6Z84cVVVNTk7WSy65REeOHKmqqkuXLtU6dero6tWrVVU1LS1Nx4wZk+U68sN2nrBNm1SnTFEdNEi1du2MC+OOf1SqpFqunGrLlqodO6refLPqkCGqw4erfvyx6pdfqi5frpqY6PcWGZNn3l72tpZ6opQWf7y4vvDDC5qalhqydQELNJv9akRcNRRpzjnnHJo2bcqUKVO48cYb+eijj7j99tsZOHAgaWlp3HjjjQwaNAiApk2b8sILL3DttdeSnJyMiNCjRw+ft+AEHT3qinB++cVdEbNjB3z/Pcyc6crvM1SrBu3aQUICdOjgil9q17YiGBOzyhUrR6uqrRjXYxy1yvnXfEzE9VmckJCgmTumWbVqFQ0bNvQpovDJF9upCj//DB9/DD/+CLNmZV35WrEinHee29lfeaUrny9aNPzxGpOPpKan8vwPz3M07ShD2w0FXKlMOFoOEJGFqpplzbOdEZicHT4MP/0E334LX3wBy5e7a+MznHEG1KgBN98M9eq5HX+1ai4RGGP+tHTHUvpO78vC7Qu5utHVfyaA/NB8jCUC81eq7sqcuXPd0f6337ordgCaN4eePaFFC3eEX6+eXQZpTC6OpB7hsbmP8dR3T1G+WHneu+o9rmh4Rb5IABmiJhGE6/TKLyEtwtuzBz76CD7/HObNgzVr3Phy5aBpUxgyxJXpV8qy32tjTA7W/LGGp797muuaXMdznZ+jQvEKfod0nKhIBEWLFmX37t1R2xS1ev0RFM2rMnZVWLAAxo93w4UL3fhy5Vy5/j/+AVdf7S7DjMLP05hQSzqaxEe/fMT1Ta+n8WmN+WXQL9QuV9vvsLIVFYmgatWqbNmyhcTERL9DCZmMHspOWmqqK+559VV4551j4+vUgfvuc8U9l19uRT3GnKI56+bQ75N+bNy7keaVm9OwUsN8nQQgShJBoUKFrOeu7KxbB5MmwXPPuTt0wR319+gBffu6yl5jzCnbc2gPd392NxOWTKB+hfp80+cbGlaKjKsZoyIRmEw2b4YXX4T333fNMADUrQt33w0XX+yu3TfG5Jm09DTaTGjDr7t/5b4L7uOh9g9RtGDkXC5tiSBaHD0K//mP2/mvXOnGVawIjz/ummJo1szK+43JY7uSd/3ZSNwTnZ6gepnqNK980r3u+sYSQaRLSoIJE+DZZ2HLFlfh+9BD0LGju9LHGJPnVJXJyyZzx6w7eOqip+jXoh+XNbjM77BOmiWCSPX77/DAA/Daa+515crubOCyy+zI35gQ2rh3I//85J/MXjeb86udT7sa7fwO6ZRZIog0yckwYAC8/bZrdbNzZ/f60kstARgTYm8ue5PbZtyGqvJSt5cYcO4ACkjkX2lniSBS7NoFTz4JI0e6S0FbtnRFQo0a+R2ZMTGjUvFKtKnWhrE9xlKjbA2/w8kzlggiwfLl0L49/PGHSwAPPQSXXOJ3VMZEvZS0FEb8MIKUtBQebP8gXep2oXOdzlF346olgvxuyhR3p2+BAvDmm3D99X5HZExMWLx9MX2n92XxjsX0btw7XzUSl9civ3Armj3zDFx3HTRo4C4JtSRgTMgdTj3M/V/cz7mvnsu2A9v479X/ZcoVU6IyAWSwM4L8aPt26NcPPvnEtffz1Veuj11jTMit/WMtw78fzk1n38SIziMoV6yc3yGFnJ0R5Dfz5rm+dz/5BAYPhkWLLAkYE2JJR5OYvHQyAI1Pa8zqQauZ0GtCTCQBsESQv3z0EbRq5Z4/+6xrJsKSgDEhNXvtbBqNbsTNH97MqsRVAL52G+kHSwT5QVqauzT0ssvc63nzXLtAxpiQ2Z28m5s/vJmub3WleKHifPv3byOmkbi8ZnUEfktLgwsvdE1EFy7sOoC3llSNCamMRuLW/rGWoW2H8kC7ByKqkbi8ZonAT+npMHCgSwLXXOM6iilRwu+ojIlaiQcTqVC8AnEF4nj6oqepUbYGzc5o5ndYvrOiIT899hiMHQu33OLuF7AkYExIqCqvL36d+i/X59WFrwLQq0EvSwIeOyPwy+bN8PDD0Lo1jBtn7QQZEyIb9m6g38f9mLN+Dm2rt6VjrY5+h5TvWCLwQ1qa6xYSXCWxJQFjQmLy0sncNuM2RITR3Ufzz4R/RkUjcXnNEoEfrrzSdRj/f//n2hAyxoTE6SVPp12NdrzS4xWql6nudzj5liWCcLvnHvjwQ9dpzJNP+h2NMVElJS2FZ757hjRN46H2D9G5Tmc61+nsd1j5np0jhdOsWTB8ODRvDp9/bkVCxuShRdsXce6r5/LAVw+wevdqVNXvkCKGJYJwSUqCG290z6dPh7g4f+MxJkocSjnEvZ/fS8tXW/L7wd/54JoPeOtvb0V1I3F5LaSJQES6ishqEVkrIvdmMb26iHwlIotFZJmIdA9lPL66/HLXucy4cVClit/RGBM11u9Zz3M/PEefZn1YOWBlRPcd7JeQJQIRiQNGAd2AeOBaEYnPNNsDwLuqeg7QGxgdqnh89euvrijo2mvh1lv9jsaYiLf/yH4mLpkIQKPTGrHm9jW8dulrMdNIXF4L5RlBS2Ctqq5X1aPAO0CvTPMoUNp7XgbYFsJ4/JGWBn/7m3v+yCO+hmJMNJi5ZiaNRzem7/S+fzYSF03dRvohlImgCrA54PUWb1ygR4AbRGQLMBO4PasFiUg/EVkgIgsSExNDEWvovPsurFjhupesX9/vaIyJWLuSd3HjBzdyyduXUKpIKb77x3cx20hcXvO7svhaYKKqVgW6A5NFjr/bQ1XHqWqCqiZUqlQp7EGetJQU14po2bJw331+R2NMxMpoJO6d5e/wULuHWNRvEedVPc/vsKJGKO8j2ApUC3hd1RsXqC/QFUBVfxCRokBFYGcI4wqfe++FbdtgwgQoGrstGxpzsn5P+p1KJSoRVyCO4RcPp0bZGjQ9vanfYUWdUJ4RzAfqiUgtESmMqwyenmmeTUAnABFpCBQFIqzsJxu//grPPeeamO7Tx+9ojIkoqsr4ReM56+WzGLdwHAA9z+ppSSBEQnZGoKqpIjIImA3EARNUdYWIDAMWqOp04C7gVRH5F67iuI9Gy10g//63Gz71lN04ZswJWL9nPbd+fCtf/vYl7Wu056LaF/kdUtSTSNvvJiQk6IIFC/wOI2f79rl6ga5d4dNP/Y7GmIgxackkBswcQJzE8ezFz3Jri1utkbg8IiILVTUhq2nW1lAoDBrkhvfc428cxkSYM0udyYW1LmTMJWOoWrqq3+HEDDsjyGuLFsG550KjRrBsmd/RGJOvHU07ylP/e4p0TeeRDo/4HU5Uy+mMwM658trIka4LSisSMiZH87fOp8W4Fjz89cOs37PeGonzkSWCvLRzJ0yaBJddZu0JGZON5JRk7v7sbs4bfx57Du1heu/pvHH5G9ZInI+sjiAvZfQvcNNN/sZhTD72257feGneS9za/FaevuhpyhQt43dIMc/qCPLS6ae7s4L0dLtk1JgA+w7v4/1V7/P3c/4OwOZ9m6lWplou7zJ5yeoIwmHGDJcELrnEkoAxAWb8OoNGoxtxy8e38MuuXwAsCeQzlgjyQkqK63TmjDNg6lS/ozEmX0g8mMj1719Pjyk9KFesHD/0/YEGFRv4HZbJgtUR5IV774U9e+CVV6BECb+jMcZ3aelpXPD6Bfy25zce7fAo915wL4XjCvsdlsmGJYK88NtvbmidzpgYtyNpB6eVOI24AnGM6DyCmmVr0vi0xn6HZXIRdNGQiBQPZSARa98+mDkTrroKClhJm4lN6ZrO2AVjqf9SfcYuGAtAj/o9LAlEiFz3XCJyvoisBH7xXp8tItHZpeTJGD4cjhyB27PsU8eYqLf2j7V0eqMT/Wf059wq59Klbhe/QzInKJiioeeBLnhNSKvqUhFpF9KoIkVqKjz2mGtOom1bv6MxJuxeX/w6A2YOoHBcYV7t+Sp9z+lrN4ZFoKDqCFR1c6YvNy004USYCRPcsF8/f+MwxifVy1SnS50ujOo+iiql7W76SBVMItgsIucDKiKFgCHAqtCGFSHmzHHDAQP8jcOYMDmSeoQn//ck6ZrOsI7D6FS7E51qd/I7LHOKgqnd7A8MxHU8vxVoBtieb8kSmDbNXSlU0C6+MtHvpy0/0WJcCx795lE27dtkjcRFkWD2YGep6vWBI0SkDfBdaEKKEC++6IbDhvkbhzEhdvDoQR786kFe+PEFqpSuwifXfsIl9S/xOyyTh4I5I3gpyHGxQxXeew+aN3d3ExsTxTbu28jo+aPpn9CfFQNWWBKIQtmeEYhIa+B8oJKI3BkwqTSuD+LY9d13cPAg9OjhdyTGhMTew3uZtnIatzS/hfhK8awdvNZ6DItiORUNFQZKevOUChi/H7gylEHleyNGuIbl7rjD70iMyXMf/fIRt824jZ0Hd3JB9QtoULGBJYEol20iUNVvgG9EZKKqbgxjTPmbKnz4IbRqBeXK+R2NMXlm58GdDP50MFNXTKXp6U2Zfu10ayQuRgRTWZwsIs8CjYCiGSNV9cKQRZWfLV3qhq1b+xuHMXkoLT2NNhPasGnfJh7r+Bj/bvNvCsUV8jssEybBJIK3gKlAD9ylpDcDiaEMKl8b69pRoU8fX8MwJi9sO7CNM0qeQVyBOF7s+iI1y9YkvlK832GZMAvmqqEKqjoeSFHVb1T1H0Bsng2kp7umpmvWhLPP9jsaY05auqYzZv4YGrzcgFcWvAJA93rdLQnEqGDOCFK84XYRuQTYBpQPXUj52KxZbti3r79xGHMKft39K7d+fCtzN87lotoX0a1uN79DMj4LJhE8JiJlgLtw9w+UBu4IZVD51jffuKG1LWQi1PhF4xn06SCKFizKhEsn0KdZH2skzuSeCFT1E+/pPqAj/HlnceyZMgUaNoTTTvM7EmNOSs2yNelWtxujuo+icqnKfodj8omcbiiLA67GtTE0S1WXi0gP4H6gGHBOeELMJ7Zuhc2b4frrc5/XmHziSOoR/jP3PwA8duFj1kicyVJOZwTjgWrAPGCkiGwDEoB7VfXDMMSWv0ye7Ia33eZvHMYE6fvN39N3el9+2fUL/2j2D1TVioFMlnJKBAlAU1VNF5GiwA6gjqruDk9o+cxnn7nheef5G4cxuUg6msTQL4by0ryXqFamGrOun2W9hpkc5XT56FFVTQdQ1cPA+hNNAiLSVURWi8haEbk3m3muFpGVIrJCRN4+keWH1e+/Q716EBfbzSyZ/G/Tvk2MXTiWgecOZPltyy0JmFzldEbQQESWec8FqOO9FkBVtWlOC/bqGEYBFwNbgPkiMl1VVwbMUw+4D2ijqntEJH/Wwh49CqtX29VCJt/ac2gP7618j34t+hFfKZ71Q9ZzZqkz/Q7LRIicEkHDU1x2S2Ctqq4HEJF3gF7AyoB5bgVGqeoeAFXdeYrrDI0ff4S0NDj3XL8jMeY4H6z6gAEzB5B4MJH2NdpzVsWzLAmYE5JTo3On2tBcFWBzwOstQKtM89QHEJHvcE1bP6KqszIvSET6Af0AqlevfophnYSMLil79Qr/uo3Jxo6kHdz+6e1MWzmNZmc0Y8Z1Mzir4ll+h2UikN99LBYE6gEdgKrAXBFpoqp7A2dS1XHAOICEhITw9483Zw5UqwblY/OGapP/pKWn0fb1tmzet5knLnyCu8+/2xqJMyctlIlgK+7y0wxVvXGBtgA/qWoK8JuI/IpLDPNDGNeJSU+HFSugUSO/IzGGLfu3cGapM4krEMfIriOpVa6WNRVtTlkwjc4hIsVE5ETPOecD9USklogUBnoD0zPN8yHubAARqYgrKlp/gusJrSVLICkJrr7a70hMDEvXdF766SUavNyAMfPHANCtXjdLAiZP5JoIRKQnsASY5b1uJiKZd+jHUdVUYBAwG1gFvKuqK0RkmIhc6s02G9gtIiuBr4B78t19ChntC1lFsfHJL7t+od3r7Rg8azAXVL+AHvWti1STt4IpGnoEdwXQ1wCqukREagWzcFWdCczMNO6hgOcK3Ok98qdNm9ywVeZ6bmNC77VFrzFo5iCKFyrOpMsmcWPTG+3uYJPngmqGWlX3Zfrxhb/C1i/jx7uK4sKF/Y7ExKA65erQ86yevNztZU4vebrf4ZgoFUwiWCEi1wFx3g1gg4HvQxtWPpKaaknAhM3h1MMM+2YYAE90eoKOtTrSsVZHn6My0S6YyuLbcf0VHwHexjVHfUcIY8o/jh6FQ4fgkkv8jsTEgO82fUezV5rx5P+eJPFgIq7k1JjQC+aMoIGqDgWGhjqYfOd778SnSRN/4zBR7cCRA9z/xf2Mmj+KGmVrMPuG2XSu09nvsEwMCSYRjBCRM4BpwFRVXR7imPKPBQvcsE1s9sNjwmPL/i28tvg1bm95O493epyShUv6HZKJMbkWDalqR1zPZInAWBH5WUQeCHlk+cFmr4WMGjX8jcNEnd3Ju/+8H6BhpYasH7yeF7u9aEnA+CKoG8pUdYeqjgT64+4peCjnd0SJ336DcuWgeHG/IzFRQlWZtnIa8aPjGTxrMKt3rQawbiONr4K5oayhiDwiIj/jOq//HtdcRHRLS3N1BF2sLXeTN7Yf2M4V717BVe9dRbXS1Vhw6wJrJM7kC8HUEUwApgJdVHVbiOPJP1atgt27oV07vyMxUSCjkbitB7byzEXP8K/W/6JgAb/bfDTGyfWXqKqtwxFIvrNokRvaHcXmFGzet5kqpasQVyCOUd1HUatcLepXqO93WMb8RbZFQyLyrjf8WUSWBTx+Dui5LHpNnQolSkDDU+2fx8SitPQ0Rv40kgajjjUS16VuF0sCJl/K6YxgiDeMzRau5s93Dc0VK+Z3JCbCrEpcRd/pfflhyw90q9uNnmf19DskY3KU7RmBqm73ng5Q1Y2BD2BAeMLzyebNkJgI3bv7HYmJMOMWjqPZ2Gb8uvtXJl8+mRnXzaB6GR961TPmBARz+ejFWYzrlteB5CujRrlhp07+xmEiTr3y9bi8weWsHLiSG5reYC2FmoiQbdGQiNyGO/KvnalOoBTwXagD89W0aXDaadC8ud+RmHzuUMohHvn6EUSEpy56yhqJMxEppzqCt4FPgSeBewPGH1DVP0IalZ9SUmDdOmtozuRq7sa53DL9Ftb8sYb+LfqjqnYGYCJSTkVDqqobgIHAgYAHIhK9vbgv805+zj/f3zhMvrX/yH4GzBhA+4ntSdM0vrjpC8b0GGNJwESs3M4IegALcR3RBP7KFagdwrj88/HHbmgNzZlsbDuwjYlLJnLneXcyrOMwShQu4XdIxpySbBOBqvbwhkF1Sxk1MtqAb9vW3zhMvrIreRfvrniXAecOoEHFBvw25DfrMcxEjWDaGmojIiW85zeIyHMiEr3Xw33+ORQtCgWCao/PRDlVZeryqcSPiueOWXfw6+5fASwJmKgSzN5uDJAsImcDdwHrgMkhjcpPiYlQqJDfUZh8YNuBbVw29TJ6/7c3NcrWYGG/hXZnsIlKwbR6laqqKiK9gJdVdbyI9A11YL5ZswYuu8zvKIzP0tLTaPd6O7Ye2Mrwi4cz5Lwh1kiciVrB/LIPiMh9wI1AWxEpAETnIfOWLW7YqJG/cRjfbNy7kaqlqxJXII7Rl4ymdrna1C1f1++wjAmpYIqGrsF1XP8PVd2B64vg2ZBG5ZdVq9zQKopjTlp6Gs/98BwNRzVkzALXSFznOp0tCZiYEExXlTuAt4AyItIDOKyqb4Q8Mj/Mm+eG9a0cOJYs37mc8yecz12f3UWn2p24rMFlfodkTFgFc9XQ1cA84CrgauAnEbky1IH5Yu1aN6xSxd84TNi8suAVmo9tzvo963n7b28zvfd0qpaO/g74jAkUTB3BUOBcVd0JICKVgM+BaaEMzBeffALx8VC4sN+RmBDLaA6iYcWGXNXoKl7o8gKVSlTyOyxjfBFMIiiQkQQ8uwmy0/uIcuiQdU0ZA5JTknnoq4eIkzievvhp2tdsT/ua7f0OyxhfBbNDnyUis0Wkj4j0AWYAM0Mblg8WLnR3Ff/tb35HYkLk6w1f03RMU0b8MIKko0loxl3kxsS4YPosvkdE/gZc4I0ap6ofhDYsHyxe7IZWURx19h3ex7/n/Jtxi8ZRp1wdvrzpS2sq2pgAOfVHUA8YDtQBfgbuVtWt4Qos7A4edMN69fyNw+S57UnbefPnN7m79d082vFRihcq7ndIxuQrORUNTQA+Aa7AtUD60okuXES6ishqEVkrIvfmMN8VIqIiknCi68gz69dDmTJQtqxvIZi8k3gwkZd+cj/ZBhUbsGHIBp7t/KwlAWOykFPRUClVfdV7vlpEFp3IgkUkDhiF6+pyCzBfRKar6spM85UChgA/ncjy89zcuVCzpq8hmFOnqkxZPoXBnw5m/5H9dKnbhfoV6tsVQcbkIKczgqIico6INBeR5kCxTK9z0xJYq6rrVfUo8A7QK4v5/gM8DRw+4ejz0urV7ozARKzN+zbTc0pPrn//euqWr8vify62RuKMCUJOZwTbgecCXu8IeK3AhbksuwqwOeD1FqBV4AxeQqmmqjNE5J7sFiQi/YB+ANWrh6AF7NRUN7T6gYiVmp5Kh0kd2JG0g+e7PM/tLW8nrkCc32EZExFy6pgmpJdVeI3XPQf0yW1eVR0HjANISEjI+2v+fnVtzHPeeXm+aBNaG/ZuoFrpahQsUJCxPcZSu1xtapeLzs7zjAmVUN4YthWoFvC6qjcuQymgMfC1iGwAzgOm+1Jh/OOPbtigQdhXbU5Oanoqw78fTsNRDRk9fzQAF9W+yJKAMSchlA2szwfqiUgtXALoDVyXMVFV9wEVM16LyNe4S1QXhDCmrGV0WN+4cdhXbU7cst+X0Xd6XxZsW0Cvs3pxRfwVfodkTEQLWSJQ1VQRGQTMBuKACaq6QkSGAQtUdXqo1n3CMoqG7NLRfG/0/NEMmTWEckXLMfXKqVwVfxUi4ndYxkS0XBOBuH/Z9UBtVR3m9Vd8hqrOy+29qjqTTM1RqOpD2czbIaiIQ2HFCrtiKJ/LaCSu8WmN6d24N893eZ6KxSvm/kZjTK6COSMYDaTjrhIaBhwA/gucG8K4wmvTJqsozqcOHj3IA18+QMECBXm287O0q9GOdjWsYUBj8lIwlcWtVHUg3nX+qroHiJ52mpOS3LBWLX/jMMf5Yv0XNBnThBd+eoEjaUeskThjQiSYM4IU7y5hhT/7I0gPaVThtG6dG3bo4GsY5pi9h/dy92d3M37xeOqVr8fcPnNpW8O6DzUmVII5IxgJfACcJiKPA/8DnghpVOG0erUbVq7sbxzmT78n/c47y9/h/9r8H0v7L7UkYEyIBdMM9VsishDoBAhwmaquCnlk4fK//7mhdVjvq4yd/5DzhnBWxbPYcMcGqww2JkyCuWqoOpAMfBw4TlU3hTKwsLFLR32lqrz181sMmTWEpKNJdK/XnXoV6lkSMCaMgqkjmIGrHxCgKFALWA00CmFc4XPwoN1R7JNN+zbR/5P+fLr2U1pXbc34S8dTr4K192RMuAVTNNQk8LXXUNyAkEUUbqtXQ8uWfkcRc1LTU+kwsQM7D+5kZNeRDDh3gDUSZ4xPTvjOYlVdJCKtcp8zQhQqBAcO+B1FzFi/Zz01ytSgYIGCvNrzVeqUr0PNsjX9DsuYmJbrVUMicmfA424ReRvYFobYQi89HbZtgyZNcp/XnJLU9FSe/t/TxI+KZ9T8UQB0qt3JkoAx+UAwZwSlAp6n4uoM/huacMJs7143rFDB1zCi3ZIdS+g7vS+Lti/i8gaXc1X8VX6HZIwJkGMi8G4kK6Wqd4cpnvD65Rc3PP10f+OIYi/Pe5l/zf4XFYpVYNpV06ylUGPyoWwTgYgU9FoQbRPOgMIq49LRhg39jSMKZTQS1/T0plzf5Hqe6/Ic5YuV9zssY0wWcjojmAc0B5aIyHTgPeBgxkRVfT/EsYXekSNuaJ3W55mko0kM/WIoheIKMbzzcGskzpgIEEwTE0WB3bjWR3sAPb1h5Nu40Q3POMPfOKLEZ+s+o/Hoxrw07yVS0lKskThjIkROZwSnicidwHKO3VCWITr+4Rs2uDaGihXzO5KItufQHu787E4mLpnIWRXOYu7f53JB9Qv8DssYE6ScEkEcUJK/JoAM0ZEIfvjBmpbIAzsP7mTaymncd8F9PNT+IYoWLOp3SMaYE5BTItiuqsPCFokfkpKgalW/o4hIO5J2MOXnKfyr9b9cI3FDNlChuF2Ga0wkyqmOILo7gk1Ph1274Oyz/Y4koqgqk5ZMIn5UPPd9cR9rdq8BsCRgTATLKRF0ClsUfsi4h6BkSX/jiCAb9m6g61td6fNRH+IrxbOk/xJrJM6YKJBt0ZCq/hHOQMIuIxG0id7bJPJSanoqHSd1ZFfyLkZ1H0X/hP4UkGAuOjPG5Hcn3Ohc1MjomaxuXX/jyOfW/rGWWmVrUbBAQSZcOoHa5WpTo2wNv8MyxuSh2D2kyzgjqF3b3zjyqZS0FJ749gkajW70ZyNxHWt1tCRgTBSK3TOCDz90FcVFivgdSb6zaPsi+k7vy5IdS7gq/iquaXSN3yEZY0IodhPB/v12I1kWRv40kjtn30mlEpV4/+r3ubzh5X6HZIwJsdhMBGvcJY9cYHe/ZshoJO6cM87hprNvYkTnEZQrVs7vsIwxYRCbiWDtWjc8/3x/48gHDhw5wH1f3EeRuCKM6DKCtjXa0rZGW7/DMsaEUWxWFm/f7oaNGvkbh89mrZ1F4zGNGT1/NIpaI3HGxKjYPCP47js3jNHmp3cn7+bOz+7kjaVv0LBiQ777x3e0rtba77CMMT6JzUSwaROULw+FC/sdiS92H9rNB6s+4MF2DzK07VCKFLQrp4yJZSEtGhKRriKyWkTWisi9WUy/U0RWisgyEflCREJ/kXpammt19IrY6jJx+4HtDP9+OKpK/Qr12XjHRoZ1HGZJwBgTukTg9Xc8CugGxAPXikh8ptkWAwmq2hSYBjwTqnj+9PvvcPAgxGcOJTqpKhMWT6DhqIY8+NWDrP3DVZTbFUHGmAyhPCNoCaxV1fWqehR4B+gVOIOqfqWqyd7LH4HQtwn96aduGAP9EPy25zc6v9mZvtP7cvYZZ7O0/1JrJM4Yc5xQ1hFUATYHvN4CtMph/r7Ap1lNEJF+QD+A6tWrn1pUixa5Yc+ep7acfC41PZUL37iQ3cm7GXPJGPq16GeNxBljspQvKotF5AYgAWif1XRVHQeMA0hISMibaxwrRGf7+Wt2r6F2udoULFCQ13u9Tp1ydahWpprfYRlj8rFQHiJuBQL3QFW9cX8hIhcBQ4FLVfVICONxvvsuKusHUtJSeGzuYzQe05iX570MQIeaHSwJGGNyFcozgvlAPRGphUsAvYHrAmcQkXOAsUBXVd0ZwliO2bYNqkXXznHBtgX0nd6XZb8vo3fj3lzb5Fq/QzLGRJCQnRGoaiowCJgNrALeVdUVIjJMRC71ZnsWKAm8JyJLRGR6qOLxgoLERDjrrJCuJpxe/PFFWr3Wil3Ju/io90dMuWIKp5U4ze+wjDERJKR1BKo6E5iZadxDAc8vCuX6j7PVK5lq2TKsqw2FjEbiEs5MoO85fXnm4mcoW7Ss32EZYyJQvqgsDpv1693wjDP8jeMU7D+yn/+b838ULViU57s+T5vqbWhT3brbNMacvNi6nnDFCjeM0DqCmWtm0mh0I8YtGkfBAgWtkThjTJ6IrTOCAwfcMMLqCHYl7+KOWXfw1s9v0ahSI6ZdNY1WVXO6JcMYY4IXW4kgMdF1TVmxot+RnJA9h/bw8a8f83D7h7m/7f0UjovNxvKMMaERW4lg5UooUcLvKIKydf9W3vr5Le45/x7qVajHxjs2WmWwMSYkYquOoGTJY8VD+ZSq8urCV4kfHc8jXz/Cuj3rACwJGGNCJrYSwd690LSp31Fka90f6+j0Rif6fdKP5pWbs+y2ZdQtX9fvsIwxUS62ioZ27863bQylpqfS6Y1O/HHoD8b2GMstzW+xRuKMMWERW4ng6NF8V0ewetdq6pSvQ8ECBZl02STqlK9D1dKhb43bGGMyxNYh59GjUKiQ31EAcDTtKI9+/ShNxjRh1LxRALSv2d6SgDEm7GLvjCAf9FM8b+s8+k7vy/Kdy7muyXVc3/R6v0MyxsSw2EoEu3ZBqVK+hvDCjy9w12d3UblkZT6+9mN61O/hazzGGBM7iSA11V06evrpvqw+o5G4llVacmvzW3n6oqcpU7SML7EYY0yg2EkE+/a54cGD4V3t4X38e86/KVaoGC90fYHzq53P+dXOD2sMxhiTk9ipLE5JccNatcK2yo9Xf0z86HheW/waReKKWCNxxph8KXbOCDISQRiuGko8mMiQWUOYsnwKTU5rwofXfMi5Vc4N+XqNMeZkxE4iSE11wzAkgn1H9jFzzUwe7fAo915wrzUSZ4zJ12InEfz+uxsWDM0mb963mTeXvcm9F9xL3fJ12XjHRqsMNsZEhNipI0hKcsOSJfN0semazisLXqHR6EY89u1jfzYSZ0nAGBMpYicRHD7shlWq5Nki1+xew4WTLuS2GbfRskpLfr7tZ2skzhgTcWKnaGjTJjfMo6Kh1PRULp58MXsP72X8peP5e7O/IyJ5smxjjAmn2EkExYu7YenSp7SYVYmrqFehHgULFGTy5ZOpU74OZ5Y6Mw8CNMYYf8RO0VB6uhvGxZ3U24+kHuHhrx6m6StNeXneywC0rdHWkoAxJuLFzhlBxs1cBU489/245Uf6Tu/LysSV3Nj0Rm5semMeB2eMMf6JnUSQcUZwgolgxPcjuGfOPVQtXZWZ182kW71uIQjOGGP8Y4kgu9k1nQJSgNbVWtM/oT9PXfQUpYucWv2CMcbkR7GXCHK5smfv4b3cNfsuihcqzkvdX7JG4owxUS92KouDqCP48JcPiR8Vz6SlkyhVpJQ1EmeMiQmxd0aQRSLYeXAng2YO4r2V79HsjGZ8ct0nNK/cPMwBGmOMP2IvEWRRNLT/yH7mrJ/D4xc+zj3n30OhuPzRr7ExxoRD7CSCTEVDm/ZtYvLSydzf9n7qlq/Lpjs2UaqIv91YGmOMH0JaRyAiXUVktYisFZF7s5heRESmetN/EpGaIQvGOyNIFxg9fzSNRjfiif898WcjcZYEjDGxKmSJQETigFFANyAeuFZE4jPN1hfYo6p1geeBp0MVD+nprK4AHab1YODMgbSu2poVA1ZYI3HGmJgXyjOClsBaVV2vqkeBd4BemebpBUzynk8DOkmIWm5LTUuhy43w866VvN7rdWbfMJuaZWuGYlXGGBNRQllHUAXYHPB6C9Aqu3lUNVVE9gEVgF2BM4lIP6AfQPXq1U8qmIIN4nlzZUfqDB1P5Yrh67fYGGPyu4i4j0BVx6lqgqomVKpU6eQWcumlXDDxS0sCxhiTSSgTwVagWsDrqt64LOcRkYJAGWB3CGMyxhiTSSgTwXygnojUEpHCQG9geqZ5pgM3e8+vBL5Uu53XGGPCKmR1BF6Z/yBgNhAHTFDVFSIyDFigqtOB8cBkEVkL/IFLFsYYY8IopDeUqepMYGamcQ8FPD8MXBXKGIwxxuQsIiqLjTHGhI4lAmOMiXGWCIwxJsZZIjDGmBgnkXa1pogkAhtP8u0VyXTXcgywbY4Nts2x4VS2uYaqZnlHbsQlglMhIgtUNcHvOMLJtjk22DbHhlBtsxUNGWNMjLNEYIwxMS7WEsE4vwPwgW1zbLBtjg0h2eaYqiMwxhhzvFg7IzDGGJOJJQJjjIlxUZkIRKSriKwWkbUicm8W04uIyFRv+k8iUtOHMPNUENt8p4isFJFlIvKFiNTwI868lNs2B8x3hYioiET8pYbBbLOIXO191ytE5O1wx5jXgvhtVxeRr0Rksff77u5HnHlFRCaIyE4RWZ7NdBGRkd7nsUxEmp/ySlU1qh64Jq/XAbWBwsBSID7TPAOAV7znvYGpfscdhm3uCBT3nt8WC9vszVcKmAv8CCT4HXcYvud6wGKgnPf6NL/jDsM2jwNu857HAxv8jvsUt7kd0BxYns307sCngADnAT+d6jqj8YygJbBWVder6lHgHaBXpnl6AZO859OATiIiYYwxr+W6zar6laomey9/xPUYF8mC+Z4B/gM8DRwOZ3AhEsw23wqMUtU9AKq6M8wx5rVgtlmB0t7zMsC2MMaX51R1Lq5/luz0At5Q50egrIhUPpV1RmMiqAJsDni9xRuX5TyqmgrsAyqEJbrQCGabA/XFHVFEsly32TtlrqaqM8IZWAgF8z3XB+qLyHci8qOIdA1bdKERzDY/AtwgIltw/Z/cHp7QfHOi//dchbRjGpP/iMgNQALQ3u9YQklECgDPAX18DiXcCuKKhzrgzvrmikgTVd3rZ1Ahdi0wUVVHiEhrXK+HjVU13e/AIkU0nhFsBaoFvK7qjctyHhEpiDud3B2W6EIjmG1GRC4ChgKXquqRMMUWKrltcymgMfC1iGzAlaVOj/AK42C+5y3AdFVNUdXfgF9xiSFSBbPNfYF3AVT1B6AornG2aBXU//1ERGMimA/UE5FaIlIYVxk8PdM804GbvedXAl+qVwsToXLdZhE5BxiLSwKRXm4MuWyzqu5T1YqqWlNVa+LqRS5V1QX+hJsngvltf4g7G0BEKuKKitaHMca8Fsw2bwI6AYhIQ1wiSAxrlOE1HbjJu3roPGCfqm4/lQVGXdGQqqaKyCBgNu6KgwmqukJEhgELVHU6MB53+rgWVynT27+IT12Q2/wsUBJ4z6sX36Sql/oW9CkKcpujSpDbPBvoLCIrgTTgHlWN2LPdILf5LuBVEfkXruK4TyQf2InIFFwyr+jVezwMFAJQ1Vdw9SDdgbVAMvD3U15nBH9exhhj8kA0Fg0ZY4w5AZYIjDEmxlkiMMaYGGeJwBhjYpwlAmOMiXGWCEy+JCJpIrIk4FEzh3mT8mB9E0XkN29di7w7VE90Ga+JSLz3/P5M074/1Ri95WR8LstF5GMRKZvL/M0ivTVOE3p2+ajJl0QkSVVL5vW8OSxjIvCJqk4Tkc7AcFVtegrLO+WYcluuiEwCflXVx3OYvw+u1dVBeR2LiR52RmAigoiU9PpRWCQiP4vIcS2NikhlEZkbcMTc1hvfWUR+8N77nojktoOeC9T13nunt6zlInKHN66EiMwQkaXe+Gu88V+LSIKIPAUU8+J4y5uW5A3fEZFLAmKeKCJXikiciDwrIvO9Nub/GcTH8gNeY2Mi0tLbxsUi8r2InOXdiTsMuMaL5Rov9gkiMs+bN6sWW02s8bvtbXvYI6sH7q7YJd7jA9xd8KW9aRVxd1VmnNEmecO7gKHe8zhce0MVcTv2Et74/wMeymJ9E4ErvedXAT8BLYCfgRK4u7JXAOcAVwCvBry3jDf8Gq/Pg4yYAubJiPFyYJL3vDCuFcliQD/gAW98EWABUCuLOJMCtu89oKv3ujRQ0Ht+EfBf73kf4OWA9z8B3OA9L4tri6iE39+3Pfx9RF0TEyZqHFLVZhkvRKQQ8ISItAPScUfCpwM7At4zH5jgzfuhqi4Rkfa4zkq+85rWKIw7ks7KsyLyAK6dmr649ms+UNWDXgzvA22BWcAIEXkaV5z07Qls16fAiyJSBOgKzFXVQ15xVFMRudKbrwyusbjfMr2/mIgs8bZ/FTAnYP5JIlIP18xCoWzW3xm4VETu9l4XBap7yzIxyhKBiRTXA5WAFqqaIq5F0aKBM6jqXC9RXAJMFJHngD3AHFW9Noh13KOq0zJeiEinrGZS1V/F9XXQHXhMRL5Q1WHBbISqHhaRr4EuwDW4jlbA9TZ1u6rOzmURh1S1mYgUx7W/MxAYieuA5ytVvdyrWP86m/cLcIWqrg4mXhMbrI7ARIoywE4vCXQEjutzWVw/zL+r6qvAa7ju/n4E2ohIRpl/CRGpH+Q6vwUuE5HiIlICV6zzrYicCSSr6pu4xvyy6jM2xTszycpUXENhGWcX4Hbqt2W8R0Tqe+vMkrre5gYDd8mxptQzmiLuEzDrAVwRWYbZwO3inR6Ja5XWxDhLBCZSvAUkiMjPwE3AL1nM0wFYKiKLcUfbL6pqIm7HOEVEluGKhRoEs0JVXYSrO5iHqzN4TVUXA02AeV4RzcPAY1m8fRywLKOyOJPPcB0Dfa6u+0VwiWslsEhcp+VjyeWM3YtlGa5jlmeAJ71tD3zfV0B8RmUx7syhkBfbCu+1iXF2+agxxsQ4OyMwxpgYZ4nAGGNinCUCY4yJcZYIjDEmxlkiMMaYGGeJwBhjYpwlAmOMiXH/Dyb/hwMmXQq1AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUROC score: 0.93029612\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('MSP AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAm1klEQVR4nO3deZhU1Z3/8feHZt8VUcMmqBgwggvghkbct4hjVoyOo/EXTRSdbM6YqHGJGaN5NI4GTUhidDCKW2KImhjXaIwa2wUVCIq4gRoIsoiAbN/fH+e2tG1DV0PfWro+r+ep51bdOnXv97DUt845956jiMDMzKpXm1IHYGZmpeVEYGZW5ZwIzMyqnBOBmVmVcyIwM6tyTgRmZlXOicAqhqTjJP25gHI/k3ReMWIqBkmvSTooe36BpBtLHZO1Lk4E1iKyL6vlkpZK+qek6yV1bclzRMRvIuKQAsp9LSJ+0JLnriMpJL2f1XOupCsk1eRxro0hqbukKyW9kcX4SvZ6i1LHZuXLicBa0lER0RXYDRgJnNuwgKS2RY+q5e2c1XM/4EvAV0ocDwCS2gMPAJ8CDgO6A3sBC4DdN+J4reHvygrgRGAtLiLmAn8EdoIPf0WfLull4OVs32ckPSdpkaS/SRpe93lJ/SX9VtJ8SQsk/TTbf6Kkv2bPJeknkuZJWiLpBUl157te0sX1jvdVSbMkvStpiqQ+9d4LSV+T9HIWywRJKrCes4DHgF3qHW9j6rWdpAezff+S9BtJPZv5xw5wAjAAOCYipkfE2oiYFxE/iIh76tV3+3oxffhnJWmMpDmS/lvSO8CvJc2Q9Jl65dtm8e+Wvd4zq+ciSVMljdmIuK3EnAisxUnqDxwBPFtv978BewA7StoVuA44FegF/ByYIqlD1s1yF/A6MBDoC0xu5DSHAJ8GdgB6AF8k/fJtGMsBwCXZ+5/IjtvweJ8BRgHDs3KHFljPIcC+wKzs9cbWS1mMfYChQH/ggkJiaOAg4E8RsXQjPltna2BzYBvgFOBm4Nh67x8K/CsinpHUF7gbuDj7zHeAOyT13oTzWwk4EVhLulPSIuCvwF+A/6n33iUR8W5ELCd9wfw8Ip6MiDURcQPwAbAnqQujD3BWRLwfESsi4q+NnGsV0A0YAigiZkTE242UOw64LiKeiYgPgO8Ce0kaWK/MjyJiUUS8ATxEvV/46/GMpPeBGcDDwDXZ/o2qV0TMioj7IuKDiJgPXEHqdmquXkBjfwbNsRY4P4tlOXATMFZS5+z9L5OSA8DxwD0RcU/W+rgPqCX9CLAK4kRgLenfIqJnRGwTEadlXyR13qz3fBvg21l3wqIsefQnfVH2B16PiNUbOlFEPAj8FJgAzJM0UVL3Ror2If0Kr/vcUlLLoW+9Mu/Ue74M6AogaVo24LpU0r71yuyWlfkSqZXTZVPqJWkrSZOzweclwI3AxgzuLiC1ejbF/IhYUfci6/6aARyVJYOxpOQAqb5faFDffVogBisyJwIrlvrT3L4J/DBLGnWPzhFxc/begEIGKiPiqogYAexI6iI6q5Fib5G+sACQ1IX0y3luAcf/VER0zR6PNngvIuJW4HHg+5tYr/8h/fkMi4jupF/aBY1TNHA/cGhWx/VZBnSu93rrBu83Nh1xXffQ0cD0LDlAqtOkBvXtEhE/2ojYrYScCKwUfgF8TdIe2aBvF0lHSuoG/J3UvfGjbH9HSaMbHkDSqOzz7YD3gRWkbo2GbgZOkrSLpA6kL90nI+K1FqrLj4CvStp6E+rVDVgKLM763RtLaIWYRPpyvkPSEEltJPWS9D1Jdd01zwFfllQj6TAK64KaTBqT+TrrWgOQWi5HSTo0O17HbMC530bGbyXiRGBFFxG1wFdJXTsLSYOtJ2bvrQGOArYH3gDmkLpgGupO+uJdSOr6WQD8uJFz3Q+cB9xB+iLeDhjXgnV5AXiE1Pe/sfW6kNTdtJg0+PrbjYzlA9KA8T+A+4AlpAS0BfBkVuw/szgWkcZP7izguG+TWj57A7fU2/8mqZXwPWA+KQmdhb9XKo68MI2ZWXVz5jYzq3JOBGZmVc6JwMysyjkRmJlVuYqbVGqLLbaIgQMHljoMM7OK8vTTT/8rIhqd/qPiEsHAgQOpra0tdRhmZhVF0uvre89dQ2ZmVc6JwMysyjkRmJlVuYobIzAz21irVq1izpw5rFixounCFapjx47069ePdu3aFfwZJwIzqxpz5syhW7duDBw4kAIXoqsoEcGCBQuYM2cOgwYNKvhzuXUNSbpOaRnBF9fzviRdpbSE4PN1S9+ZmeVlxYoV9OrVq1UmAQBJ9OrVq9ktnjzHCK4nLaC9PocDg7PHKcC1OcZiZgbQapNAnY2pX25dQxHxSIPlABs6Gvi/SNOfPiGpp6RPrGe5wU3317/Cn//cdLkxY+CAA3IJwcysHJVyjKAvH12+cE6272OJQNIppFYDAwYM2LizPf44XHzxhstEwKWXwqJF0KnTxp3HzGwDampqGDZsGKtXr2bQoEFMmjSJnj17AjBt2jTOOOMM5s6dy9q1aznhhBM499xzP/yV/8c//pHzzjuPZcuW0aFDBw444AAuv/zyTY6pIi4fjYiJETEyIkb27t3oHdJNO+ssWLt2w48vfAFWroTx4+G22+Dcc+G442DvvWHzzaF7d7jhhpRQTj0VDj8cZs9u2cqaWavWqVMnnnvuOV588UU233xzJkyYAMDy5csZO3YsZ599NjNnzmTq1Kn87W9/45prrgHgxRdfZPz48dx4441Mnz6d2tpatt9++xaJqZQtgrmkBb3r9KOAdWRzddllKQFcd1161NTAgAEwcCAsXJjKnHjiRz+z3Xaw335p/+zZKZkMG1bkwM2sEu211148//zzANx0002MHj2aQw45BIDOnTvz05/+lDFjxnD66adz2WWXcc455zBkyBAgtSy+/vWvt0gcpUwEU4DxkiYDewCLcxsfKNTAgXDnnTB3Luy+e/pC79Ahvffee/Dcc/CJT0C/fmn//vvDX/6y7gHwm9/AK6/AkiUwc2Z6zJ2bWiRtKqIBZlYdvvGN9H+6Je2yC1x5ZUFF16xZwwMPPMDJJ58MpG6hESNGfKTMdtttx9KlS1myZAkvvvgi3/72t1s23kxuiUDSzcAYYAtJc4DzgXYAEfEz4B7gCNK6rsuAk/KKpVmOPrrx/d26wb77fnTfww/D0qVQW5uSw847p1ZB377w1lsfLXv22XDIIXDJJbCbr5Q1q1bLly9nl112Ye7cuQwdOpSDDz641CHletXQsU28H8DpeZ2/aLp2TVcaAZxzDtxxB+y0EwwZAkOHwqBB6VcCpKuW6q5cuvRSOPBAmDYNnn4aOnaEPfaAtm1h7NhS1MSsuhT4y72l1Y0RLFu2jEMPPZQJEyZw5plnsuOOO/LII498pOzs2bPp2rUr3bt351Of+hRPP/00O++8c8sHFREV9RgxYkRUpNWrI047LSJdm9T044UXImbMiJgzJ+K990odvVmrMH369FKHEF26dPnw+TPPPBMDBgyIVatWxbJly2LQoEFx3333RUTEsmXL4sgjj4yrrroqIiKmTp0a2223XcycOTMiItasWRPXXntto+dorJ5Abazne9Wd1sVSUwMTJqSv+YceSlck3XYbTJ8Os2bBpEnpdZ1hw1KLol+/1C0lQY8e8P77pauDmbWoXXfdleHDh3PzzTfTqVMnfv/733PxxRfzyU9+kmHDhjFq1CjGjx8PwPDhw7nyyis59thjGTp0KDvttBOzW+iqRaVEUTlGjhwZrXphmmXL4Ior0uD03Lnwj3+krqOGpkyBo44qfnxmFWzGjBkMHTq01GHkrrF6Sno6IkY2Vt6TzpWbzp1Ta6Gh5cvh+OPht79Nr+uPIzz+OOy5Z3HiM7NWx11DlaJTpzQQHQH33//R9/baK3Ud3XprakmYmTWDE0ElOvDAdcPK3//+uv1f+lK6+3mzzWDFCli9unQxmpWpSusOb66NqZ8TQaW78EL44AOYPDl1K8G6uZLatUsthezORbNq17FjRxYsWNBqk0Fk6xF07NixWZ/zYHFrs2RJuo9h8GB48MF1+884A374w3S/QjNWLjJrTap5hbINDRY7EbR2Y8fCH/7w0X2nnZYuZTWzqrGhROCuodZuyhS49940R1Kda65JXUaLFpUsLDMrH04E1eCQQ9LcRxHw97+v27/ZZikhSHDLLfDrX6cyZlZVnAiqzahRaRxh+PCP7h83Dr7ylTRDqpS6jpYvL02MZlZUTgTVqFs3mDo1/fpfsya1BH75y3VTbkNanKdz55QUJk50S8GsFXMiqHZt2qRFdU4+Od17sHIlXHvtR8uceuq6lkLd44QT4LXXUkL54APfs2BWwXzVkG3Ygw/Cl78M//xn4Z+pqUn3MYwenSbKu/VWOOmktOzn6tVp6c/3308J5O234d130zoOxx8P7duvux/CzFqMLx+1lrNmDTz2WOo62nZb+P3v07jDU0+17Hl23RX69EnJZOVK+NSnUgLZcsu0ZoOZNYsTgRVPRPq1/9570KVLejz3XGpZ9O6dns+blxb02X77NC4xcmRa0vPRR9OkekuWFHauPn1SMho8OC3q06MH9OqVtiNGpJaJmQFOBFapZs5MX+YvvQQPPABvvpmWB50/H7bZBl5/vfBjbbtt6n4aMyYtJXrkkdCzJxx0kO+0tqrgRGCt1/Ll8OSTqRXy+utpPYe334bbb0/rSb/7blpLeurUdN/EwoUfP8bgwbD//inpbL55ShoRcMwxKVm08TUVVvmcCMzqRMA996QrpG67Ld1I1xw775wWC4pI03+PGgX9+6d7MLp2zSdmsxbgRGC2IUuXrruKafr09IU+cWJqTbz6ahp3ePTRtGzoggXpyqdVqxo/Vs+ecN99aYxCKmo1zDbEK5SZbUjXrut+zQ8YkLb77LPhz6xdm5LEqlVw6aXwu9/B4sVp/qZRo9aV69IlrRnx2c+mRNLM6YHNisEtArOWsno1/OY38K1vpfso5s5tvFyHDqk7aa+9YODANDbRo0e643v0aNhqK3jnnTRe0aOHk4e1CHcNmZXKvHnws59BbW2612Lx4o2bw6lt2zQu0bkz7Ltvarl87nOplSGlbZcuLR+/tRpOBGblZs0aeOWVlBgWL06/+h96CJ55Jk0I+OqrqdykSWnbq1can2jK7rvDbrulY/bokVodO+2UbtDzmEVVcyIwa01WrUpdUMuXp8tin38eHn+86c+1bQtHHZXuo/j0p9P05D165B+vlQUnArNqsXJl+sJ//3149ll4+eV0iex9963/M717p6uk9tjjowsYWaviRGBm68ybB1dfDXfcATNmfPS9jh3T1B3vvgvf+U4ahxgypDRxWotyIjCz9XvrrdRiuOWWNCD9pz99vEz37rDffmlqjrFj3XKoQE4EZtY8770Hd90FN9yQ1rxuqE+fNG34vvum5OCB6LLnxevNrHm6dYNjj02tg4j0WLECfvzj9P5bb8Fll6XB5zZtYNAgOO+8lECs4uSaCCQdJmmmpFmSzm7k/W0kPSDpeUkPS+qXZzxmtgk6dEjjBnVJ4Z//THdM77lnWq3u4otTF1JNTbqM9Zhj4Ior0l3YVtZy6xqSVAO8BBwMzAGeAo6NiOn1ytwG3BURN0g6ADgpIv59Q8d115BZGVq5EiZMgIsuSl/8dcue1hk2LA1ODx5cuhirXKm6hnYHZkXE7IhYCUwGjm5QZkfgwez5Q428b2aVoH17+OY30zTfixenCfwWLYLDD0/vv/AC7LBDGksYNizNx3T11TBrVknDtiTPRNAXeLPe6znZvvqmAp/Nnh8DdJPUq+GBJJ0iqVZS7fz583MJ1sxaWI8eacrvCJgyJQ0wt20LL76Yptw488zUQpDS4/77U1krulIPFn8H2E/Ss8B+wFxgTcNCETExIkZGxMjevXsXO0Yz21RHHZUm4Vu1Kn3Z19bCJZeku5vrHHxwGnj+zGfgtNPSAkNWFHkmgrlA/3qv+2X7PhQRb0XEZyNiV+CcbN+iHGMys3IwYgScfXa6NDUC/vIX2Hvv9N7dd8O116YWhJQSw8svlzbeVi7PRPAUMFjSIEntgXHAlPoFJG0hqS6G7wLX5RiPmZWrT38aHnssJYXly1OC6Jv1JN99dxpfGDIELrhg42ZvtQ3KLRFExGpgPHAvMAO4NSKmSbpI0tis2BhgpqSXgK2AH+YVj5lViI4dU5fRnDkpMdx5J2y5JcycCRdemKbi7to1tRSWL/e4QgvwncVmVhnWrk1J4Zpr4IEH1u2vqYHDDoPJk71u9Ab4zmIzq3xt2qQb2O6/P7UELrgAvvjFtLbD3Xenu6F33BF+8hN3HzWTE4GZVZ6OHeH889NEeWvWwDnnpP0zZqSlQjt3TvMkWUGcCMyssrVpk6a3iEirvn3+82n/iSemAea6QWhbLycCM2s9tt0Wbrtt3Z3MM2fCPvukZPGNb6QZVe1jnAjMrPXZaaeUBGbPTjOjduoE//u/6cY2CR59tNQRlhUnAjNrvQYNSslg2bJ0OeqBB6b9n/40bLNN42stVCEnAjOrDn37piuOamuhXz9444102WnD5TqrkBOBmVWXESPgzTfXtQ523DENLK9eXdKwSsmJwMyq0/33w8MPp+c33ADt2qUFdqqQE4GZVa/99kvrJ4wYkV4PGgSnnlramErAicDMqlv37mnc4Kqr0uuJE+G440obU5E5EZiZAZxxxrrlNW+6CXbZJd25XAWcCMzM6rRrl5bYHDECpk6FceNgjz3S0putmBOBmVl9PXqkrqK52Tpaf/97mtvod78rbVw5ciIwM2tMnz5pjqLx49Prz342raTWCjkRmJltyNVXp6mtAQ49NM122so4EZiZNeUb34CLLkpjBdtsky45bUWcCMzMCnHuubD33mnsYIcdWlXLwInAzKwQUlrbYP/9Yd482GqrUkfUYpwIzMya48EHYeedYcGCVnOfgROBmVlzPfBA2o4b1yquJHIiMDNrrl694M9/Ts/HjIFnnilpOJvKicDMbGMcfDDceGN6XjdpXYVyIjAz21jHHZdmMAV45JHSxrIJnAjMzDZF3ayl++2XFrypQE4EZmabYvhw+K//Ss932qm0sWwkJwIzs0116aVpu2QJvP12aWPZCE4EZmYt4frr03bYMFixoqShNJcTgZlZS/iP/4Af/jDdaPbv/17qaJrFicDMrKV897tpe/vt8MQTpY2lGZwIzMxaipQWsgE47bTSxtIMuSYCSYdJmilplqSzG3l/gKSHJD0r6XlJR+QZj5lZ7kaNgtNPh2efhUmTSh1NQXJLBJJqgAnA4cCOwLGSdmxQ7Fzg1ojYFRgHXJNXPGZmRVPXRXTCCbB2bWljKUCeLYLdgVkRMTsiVgKTgaMblAmge/a8B/BWjvGYmRVH377rBowvuaS0sRSgoEQgabSk+yS9JGm2pFclzW7iY32B+rfZzcn21XcBcLykOcA9wBnrOf8pkmol1c6fP7+QkM3MSmvChLQ999y0slkZK7RF8CvgCmAfYBQwMttuqmOB6yOiH3AEMEnSx2KKiIkRMTIiRvbu3bsFTmtmlrNu3dbdW/Cf/1nSUJpSaCJYHBF/jIh5EbGg7tHEZ+YC/eu97pftq+9k4FaAiHgc6AhsUWBMZmbl7fjj0/bnP4d//au0sWxAoYngIUk/lrSXpN3qHk185ilgsKRBktqTBoOnNCjzBnAggKShpETgvh8zax1qalISALj66tLGsgGKiKYLSQ81sjsi4oAmPncEcCVQA1wXET+UdBFQGxFTsquIfgF0JQ0c/1dE/HlDxxw5cmTU1tY2GbOZWVmISOsbf/ABvPMOdOpUkjAkPR0RIxt9r5BEUE6cCMys4txxB3z+83DAAeuWuSyyDSWCQq8a6iHpirordyRdLqlHy4ZpZtZKfe5zsMsuaeH7554rdTQfU+gYwXXAe8AXs8cS4Nd5BWVm1urceWfaluF9BYUmgu0i4vzs5rDZEXEhsG2egZmZtSrbbANdusCtt8L775c6mo8oNBEsl7RP3QtJo4Hl+YRkZtZKXX552l55ZUnDaKjQq4Z2AW4gTQMh4F3gxIiYmmt0jfBgsZlVrNWroWvXdAVRkS/U2eTB4oh4LiJ2BoYDwyJi11IkATOzita2LRx8cHp+772ljaWetht6U9LxEXGjpG812A9ARFyRY2xmZq3PlVfCXXfBLbfAoYeWOhqg6RZBl2zbbT0PMzNrju22S4PGvy6fCy832CKIiJ9n2wuLE46ZWRUYMgSefrrUUXyo0BvKLpPUXVI7SQ9Imi/p+LyDMzNrlQ46KG3PP7+0cWQKvXz0kIhYAnwGeA3YHjgrr6DMzFq1H/wgbSdNKvrVQ40pNBHUdSEdCdwWEYtzisfMrPVr1w4uvBBefXXdspYlVGgiuEvSP4ARwAOSegMr8gvLzKyVq1us5vbbS94qKPQ+grOBvYGREbEKeJ+Prz9sZmaF6tEDvvlNeOWVdDlpCW0wEUg6INt+FhgDHJ09P4yUGMzMbGOdfXba3nRTScPY4OWjwH7Ag8BRjbwXwG9bPCIzs2qx5ZYweDBMngw331yyMJq6j+D8bHtSccIxM6sye+4JL78MS5emeYhKoND7CP5HUs96rzeTdHFuUZmZVYu6uYcefrhkIRR61dDhEbGo7kVELASOyCUiM7Nqsk82w//zz5cshEITQY2kDnUvJHUCOmygvJmZFaJv37R9552ShdDUYHGd35DuH6ibJekk0voEZma2Kdq3T8lg+vSShVBQIoiISyVNBbIJMvhBRJTPZNpmZpXsoINgypS0cE3bQn+ft5xCu4YAZgB/iojvAI9K8jTUZmYtYfRoWLgQppZmva9Crxr6KnA78PNsV1/gzpxiMjOrLgMGpG2JxgkKbRGcDowGlgBExMvAlnkFZWZWVYYPT9sSLVZTaCL4ICJW1r2Q1JZ0Z7GZmW2qT3wibe+4oySnLzQR/EXS94BOkg4GbgP+kF9YZmZV5stfTtuXXir6qQtNBP8NzAdeAE4F7gHOzSsoM7Oq873vpe399xf91E1epySpBpgWEUOAX+QfkplZFdp++7SdNAlOO62op26yRRARa4CZkgYUIR4zs+rUoQP07w9r1hT91IV2DW0GTMsWrp9S98gzMDOzqnPssfDUU0W/jLTQW9jO25iDSzoM+F+gBvhlRPyowfs/AfbPXnYGtoyInhtzLjOzirfvvnDZZfDkk3B08RaB3GAikNQR+BqwPWmg+FcRsbqQA2djCxOAg4E5wFOSpkTEhxNqRMQ365U/A9i12TUwM2stPvnJtL3rrqImgqa6hm4ARpKSwOHA5c049u7ArIiYnd2DMJkNr3N8LFC6JXrMzEqtbsB4dUG/t1tMU11DO0bEMABJvwL+3oxj9wXerPd6DrBHYwUlbQMMIi2LaWZWnaS0PsErrxT1tE21CFbVPSm0S2gjjQNuz65Q+hhJp0iqlVQ7f/78HMMwMyux/v3h0Udh7dqinbKpRLCzpCXZ4z1geN1zSUua+OxcoH+91/2yfY0Zxwa6hSJiYkSMjIiRvXv3buK0ZmYVbNiwtC3iimUbTAQRURMR3bNHt4hoW+959yaO/RQwWNIgSe1JX/Yfu+RU0hDS5amPb2wlzMxajREj0vaNN4p2yuasR9AsWVfSeOBe0loGt0bENEkXSRpbr+g4YHJEeBI7M7OttkrbIrYIcl0KJyLuIc1LVH/f9xu8viDPGMzMKsrAgWlbxN/GubUIzMxsI3TLFn9curRop3QiMDMrJ23aQNeucOedRTtl8VdJNjOzDevfHzp1Ktrp3CIwMys3o0ZBEe+ZciIwMys33bvDm282Xa6FOBGYmZWbZcvSduXKDZdrIU4EZmblpm7yuddfL8rpnAjMzMrN4MFp+9prRTmdE4GZWbnZdtu0XbiwKKdzIjAzKzcDsiXi3367KKdzIjAzKzebbZa2jz1WlNM5EZiZlZuamnRTWW1tUU7nRGBmVo722Qfmzi3K5HNOBGZm5WivvdJ9BHPXt55Xy3EiMDMrR5/8ZNpOn577qZwIzMzK0R57pO2zz+Z+KicCM7Ny1L17mpL6vfdyP5UTgZlZOZLSZaQLFuR+KicCM7NytfXWRbmpzInAzKxcbbUV3Htv7qdxIjAzK1c1NamLKGdOBGZm5Wr33eGDD3K/qcyJwMysXHXtCmvXwvLluZ7GicDMrFz16JG2S5bkehonAjOzctWlS9q+/36up3EiMDMrV716pe28ebmexonAzKxc9e+ftm++metpnAjMzMrV1lunrVsEZmZVqmvXtPUYgZlZlerUKd1QlvPEc04EZmblSkqXkPryUTOzKtaxI6xYkespck0Ekg6TNFPSLElnr6fMFyVNlzRN0k15xmNmVnFWroTHHsv1FG3zOrCkGmACcDAwB3hK0pSImF6vzGDgu8DoiFgoacu84jEzq1hb5vvVmGeLYHdgVkTMjoiVwGTg6AZlvgpMiIiFABGR7zVSZmaVZvRoWLQo11PkmQj6AvXvgpiT7atvB2AHSY9JekLSYY0dSNIpkmol1c6fPz+ncM3MylCnTpU9RlCAtsBgYAxwLPALST0bFoqIiRExMiJG9u7du7gRmpmVUseOsGxZrqfIMxHMBfrXe90v21ffHGBKRKyKiFeBl0iJwczMIF0+WsH3ETwFDJY0SFJ7YBwwpUGZO0mtASRtQeoqmp1jTGZmlaWSLx+NiNXAeOBeYAZwa0RMk3SRpLFZsXuBBZKmAw8BZ0XEgrxiMjOrOB06pIVp1q7N7RS5XT4KEBH3APc02Pf9es8D+Fb2MDOzhjbfPG0XLlw3LXULK/VgsZmZbUinTmm7alVup3AiMDMrZ+3bp+3KlbmdwonAzKyctWuXtm4RmJlVqc6d0zbHS0idCMzMytkWW6RtjrMqOBGYmZWzPn3S9p//zO0UTgRmZuWsS5e0zXG5SicCM7Ny1rNn2r77bm6ncCIwMytnnTunaSacCMzMqtjWW8Mrr+R2eCcCM7NyN3QozM5vPk4nAjOzctenj68aMjOrap06wQcf5HZ4JwIzs3LXvr0TgZlZVevcOS1Ok9OaBE4EZmblrnv3lARyWrvYicDMrNzVTTy3dGkuh3ciMDMrdz16pO2SJbkc3onAzKzc1a1SltOAsROBmVm569gxbZcvz+XwTgRmZuWubozAg8VmZlWqbdu0XbMml8M7EZiZlbs22Ve17yMwM6tSdYnALQIzsypVU5O2bhGYmVUpdw2ZmVU5dw2ZmVU5dw2ZmVU5dw2ZmVU5dw2ZmVW5uhvKFi3K5fC5JgJJh0maKWmWpLMbef9ESfMlPZc9/l+e8ZiZVaRtt4Ujj4TBg3M5fNtcjgpIqgEmAAcDc4CnJE2JiOkNit4SEePzisPMrOK1awd33ZXb4fNsEewOzIqI2RGxEpgMHJ3j+czMbCPkmQj6Am/Wez0n29fQ5yQ9L+l2Sf0bO5CkUyTVSqqdP39+HrGamVWtUg8W/wEYGBHDgfuAGxorFBETI2JkRIzs3bt3UQM0M2vt8kwEc4H6v/D7Zfs+FBELIqJuyZ1fAiNyjMfMzBqRZyJ4ChgsaZCk9sA4YEr9ApI+Ue/lWGBGjvGYmVkjcrtqKCJWSxoP3AvUANdFxDRJFwG1ETEFOFPSWGA18C5wYl7xmJlZ4xQRpY6hWUaOHBm1tbWlDsPMrKJIejoiRjb2XqkHi83MrMQqrkUgaT7w+kZ+fAvgXy0YTiVwnauD61wdNqXO20REo5ddVlwi2BSSatfXNGqtXOfq4DpXh7zq7K4hM7Mq50RgZlblqi0RTCx1ACXgOlcH17k65FLnqhojMDOzj6u2FoGZmTXgRGBmVuVaZSIoYGW0DpJuyd5/UtLAEoTZogqo87ckTc+m/H5A0jaliLMlNVXneuU+JykkVfylhoXUWdIXs7/raZJuKnaMLa2Af9sDJD0k6dns3/cRpYizpUi6TtI8SS+u531Juir783he0m6bfNKIaFUP0rxGrwDbAu2BqcCODcqcBvwsez6OtEpayWPPuc77A52z51+vhjpn5boBjwBPACNLHXcR/p4HA88Cm2Wvtyx13EWo80Tg69nzHYHXSh33Jtb508BuwIvref8I4I+AgD2BJzf1nK2xRVDIymhHs27tg9uBAyWpiDG2tCbrHBEPRcSy7OUTpGnBK1mhK+D9ALgUWFHM4HJSSJ2/CkyIiIUAETGvyDG2tELqHED37HkP4K0ixtfiIuIR0iSc63M08H+RPAH0bDCTc7O1xkRQyMpoH5aJiNXAYqBXUaLLR6GrwdU5mfSLopI1Weesydw/Iu4uZmA5KuTveQdgB0mPSXpC0mFFiy4fhdT5AuB4SXOAe4AzihNayTT3/3uTcpuG2sqTpOOBkcB+pY4lT5LaAFdQfVObtyV1D40htfoekTQsIhaVMqicHQtcHxGXS9oLmCRpp4hYW+rAKkVrbBE0uTJa/TKS2pKakwuKEl0+Cqkzkg4CzgHGxrqV4SpVU3XuBuwEPCzpNVJf6pQKHzAu5O95DjAlIlZFxKvAS6TEUKkKqfPJwK0AEfE40JE0OVtrVdD/9+ZojYmgyZXRstf/kT3/PPBgZKMwFaqQ1eB2BX5OSgKV3m8MTdQ5IhZHxBYRMTAiBpLGRcZGRCUvZlHIv+07Sa0BJG1B6iqaXcQYW1ohdX4DOBBA0lBSIphf1CiLawpwQnb10J7A4oh4e1MO2Oq6hqKwldF+RWo+ziINyowrXcSbrsA6/xjoCtyWjYu/ERFjSxb0Jiqwzq1KgXW+FzhE0nRgDXBWRFRsa7fAOn8b+IWkb5IGjk+s5B92km4mJfMtsnGP84F2ABHxM9I4yBHALGAZcNImn7OC/7zMzKwFtMauITMzawYnAjOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwa4SkNZKek/SipD9I6tnCx38tu84fSUtb8thmzeVEYNa45RGxS0TsRLrX5PRSB2SWFycCs6Y9Tjapl6TtJP1J0tOSHpU0JNu/laTfSZqaPfbO9t+ZlZ0m6ZQS1sFsvVrdncVmLUlSDWn6gl9luyYCX4uIlyXtAVwDHABcBfwlIo7JPtM1K/+ViHhXUifgKUl3VPKdvtY6ORGYNa6TpOdILYEZwH2SugJ7s26aDoAO2fYA4ASAiFhDmtoc4ExJx2TP+5MmgHMisLLiRGDWuOURsYukzqR5bk4HrgcWRcQuhRxA0hjgIGCviFgm6WHShGhmZcVjBGYbkK3qdiZpYrNlwKuSvgAfrh27c1b0AdISoEiqkdSDNL35wiwJDCFNhW1WdpwIzJoQEc8Cz5MWQDkOOFnSVGAa65ZN/E9gf0kvAE+T1s79E9BW0gzgR6SpsM3KjmcfNTOrcm4RmJlVOScCM7Mq50RgZlblnAjMzKqcE4GZWZVzIjAzq3JOBGZmVe7/A0gksCe0xICrAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUPR score: 0.9186822983633804\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('MSP AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP fpr95 score: 0.2615\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('MSP fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by Mahalanobis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covairance norms: [58.213383, 111.35267, 49.149143, 44.527348, 99.387886, 59.178444, 104.78499, 80.21675, 87.104904, 63.132908]\n",
      "covariance norm: 29.670227021092874\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar10 = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_train_dataloader = DataLoader(\n",
    "    dataset=cifar10, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, id_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 215.34715 id variance: 21136.686\n",
      "ood mean score: 714.6479 ood variance: 82778.88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+1UlEQVR4nO3dd3gU1frA8e9LQu9NRSmhCqGIGEEFBQRBEESvFy8WFC+KCAgq6k/EylWs2AFFRRS72FBQrMi9NnpHBJESBCkCAqEleX9/nFldYxIWyOxkd9/P8+wzW2Zn3tky75xzZs4RVcUYY0ziKhJ0AMYYY4JlicAYYxKcJQJjjElwlgiMMSbBWSIwxpgEZ4nAGGMSnCWCOCciS0SkXdBxFBYicquIPBfQuieIyD1BrLugicglIvLJYb7XfpOFjCWCKBKR1SKyR0R2ichGb8dQxs91qmpjVZ3u5zpCRKS4iNwnImu97VwhIjeJiERj/bnE005E0sOfU9WRqnqlT+sTERksIotFZLeIpIvIWyLS1I/1HS4RuUtEXj6SZajqK6raKYJ1/S35He5vUkSKebGv8D7f1SIyXkRSDnVZ5q8sEURfd1UtAzQHTgSGBRvOoROR5DxeegvoAHQFygK9gX7A4z7EICJS2H6/jwNDgMFAJaAB8B5wTkGvKJ/vwHcBrnsScC5wMVAeOAGYg/vNHZIgP79CSVXtFqUbsBroGPb4QWBK2ONTgG+A7cACoF3Ya5WAF4BfgG3Ae2GvdQPme+/7BmiWc53AscAeoFLYaycCW4Ci3uN/A8u85U8DaoXNq8BAYAXwcy7b1gHYC9TI8XwrIAuo5z2eDtwHzAR+B97PEVN+n8F04F7ga29b6gFXeDHvBFYBV3vzlvbmyQZ2ebdjgbuAl715UrztuhxY630Ww8PWVxJ40fs8lgE3A+l5fLf1ve1smc/3PwEYDUzx4v0eqBv2+uPAOu9zmQOcHvbaXbgd4cve61cCLYFvvc9qA/AUUCzsPY2BT4HfgF+BW4Gzgf3AAe8zWeDNWx543lvOeuAeIMl7rY/3mT8KbPVe6wP8z3tdvNc2ebEtAprgDgIOeOvbBXyQ838AJHlx/eR9JnPI8Rvy5uvofZ9/ey2f/1du33Vf77ueAXwEDMqxjAXAP7z7DcM+v+XAhUHvQ3zbNwUdQCLdcvwBqnt/mMe9x8d5f7KuuJLaWd7jqt7rU4A3gIpAUaCt9/yJ3h+wlfenutxbT/Fc1vkFcFVYPA8BT3v3ewArgUZAMnAb8E3YvOr9KSoBJXPZtvuBr/LY7jX8uYOe7u1omuB21m+H/VkP9hlM9/7Ejb0Yi+KOtuvidkZtgQyghTd/O3LsuPPYOTyL2+mfAOwDGoVvk/eZVwcW5lxe2HL7A2sO8v1P8LanpRf/K8DrYa9fClT2XhsKbARKhMV9ADjP+2xKAifhEmeyty3LgOu8+cvidupDgRLe41Y5P4Owdb8LPON9J0fhEnXoO+sDZALXeusqyV8TQWfcDryC9z00AqqFbfM9+fwPbsL9D4733nsCUPlQfl+5LTef7/olbxtLApcBX4fNn4pLqsW9edbhDjSS+fOgKTXo/Ygft8JWtE4E74nITtyPbBNwp/f8pcBUVZ2qqtmq+ikwG+gqItWALkB/Vd2mqgdU9Svvff2AZ1T1e1XNUtUXcTuzU3JZ96vAReCqVoBe3nPgdmT3qeoyVc0ERgLNRaRW2PvvU9XfVHVPLsuugtvx5GaD93rIRFVdrKq7gduBC0UkKb/PIOy9E1R1iapmep/DFFX9SZ2vgE+A0/OIIy93q+oeVV2AOyI8wXv+QmCk95mnA0/ks4zK+Wx/uHdVdab3Gb+CqyIEQFVfVtWt3raNwu2Qjg9777eq+p732exR1Tmq+p03/2rcjrytN283YKOqjlLVvaq6U1W/zy0gETka9xlfp6q7VXUT7gi/V9hsv6jqk966cn7/B3CJpiEg3m8oks8CXMnmNlVd7n2HC1R1ay7zRfr5Hsxd3jbuwSW/8N/4JcA7qroP9/mtVtUXvG2ehzto6VkAMRQ6lgii7zxVLYs7Wm3InzvIWkBPEdkeugFtgGpADeA3Vd2Wy/JqAUNzvK8Grhokp7eBU73Ecgau2uS/Yct5PGwZv+GO0I4Le/+6fLZrixdrbqp5r+e2nDW4I/sq5P8Z5BqDiHQRke9E5Ddv/q78NelEYmPY/Qwg1IB/bI715bf9W8l7+yNZFyJyo4gsE5Ed3raU56/bknPbG4jIh96JB7/jkndo/hq46pZI1MJ9BxvCPvdncCWDXNcdTlW/wFVLjQY2icg4ESkX4bojjTPSz/dg/tgOVd2JK2mHEt5FuOQM7jNpleO3eAlwTAHEUOhYIgiId/Q6AXjYe2od7ki5QtittKre771WSUQq5LKodcC9Od5XSlVfy2Wd23BHzP/CNbi9rqoatpyrcyynpKp+E76IfDbpM9wfp0b4kyLSCvdn/yLs6fB5auKOKLcc5DP4WwwiUhyX3B4GjlbVCsBUXAI7WLyR2ICrEsot7pw+B6qLSNrhrEhETse1QVwIVPS2ZQd/bgv8fXvGAj8A9VW1HK6uPTT/OqBOHqvLuZx1uFJklbDPvZyqNs7nPX9doOoTqnoSrnqlAa7K56Dv89Zd9yDzgPt9tRSR6vnMsxsoFfY4t512znheAy4SkVNxVWhfhsX1VY7fYhlVvSaCWGOOJYJgPQacJSIn4BoBu4tIZxFJEpES3umP1b1i9kfAGBGpKCJFReQMbxnPAv1FpJV3Jk1pETlHRMrmsc5XcXWj/+TPaiGAp4FhItIYQETKi0jExWBV/Qy3M3xbRBp723CKt11jVXVF2OyXikiqiJQCRgCTVDUrv88gj9UWw1WfbAYyRaQLEH5K469AZREpH+l25PAm7jOpKCLHAYPymtHbvjHAa17Mxbz4e4nILRGsqyyuHn4zkCwidwAHO6oui2uc3SUiDYHwndSHQDURuU7cab1lvaQM7nNJCZ115f2+PgFGiUg5ESkiInVFpC0REJGTvd9fUdzOeC+utBlaV14JCeA54D8iUt/7/TYTkco5Z/J+X58C74rISSKS7G1TfxH5tzfbfKCX9/9Iw/3GD2Yq7uh/BPCGqobi/hBoICK9veUV9bazUQTLjDmWCAKkqptxjVd3qOo6XIPtrbidwTrcUVXoO+qNO3L+Ade2cJ23jNnAVbii+TZcg2+ffFY7GXeGy0avTjwUy7vAA8DrXjXDYly7xKG4AHdE9THuLJGXcWeiXJtjvom40tBG3FHYYC+Gg30Gf+EV7QfjdtjbcKWcyWGv/4A74lvlFe9zqy7LzwggHfgZd0Q6CXfknJfB/FlFsh1X5XE+8EEE65qG+9x+xFWX7SX/qiiAG3HbvBN3QPBG6AXvszkL6I77nFcA7b2X3/KmW0Vkrnf/MlxiXYr7LCcReVVMOW/927zYt+JORAD3/ad6n/97ubz3Edz39wkuqT2Pa8jNzT9xO+43cKWlxUAa7rsB195U14vjbv56oJMrrz3gHdxZSa+GPb8Td1DRC3em3kbc/6P4wZYZi+TPmgFj/Cci03FncgRyde+REJFrgF6qGtGRsjGxwkoExuRBRKqJSGuvquR43KmY7wYdlzEFza6uMyZvxXBnz9TGVfW8jmsHMCauWNWQMcYkOKsaMsaYBBdzVUNVqlTRlJSUoMMwxpiYMmfOnC2qWjW312IuEaSkpDB79uygwzDGmJgiImvyes2qhowxJsFZIjDGmARnicAYYxJczLUR5ObAgQOkp6ezd+/eoEPxTYkSJahevTpFixYNOhRjTJyJi0SQnp5O2bJlSUlJQYIZHtdXqsrWrVtJT0+ndu3aQYdjjIkzvlUNiRtUepOILM7jdRGRJ0RkpYgsFJEWh7uuvXv3Urly5bhMAgAiQuXKleO6xGOMCY6fbQQTcOOj5qULrhfM+rhRtsYeycriNQmExPv2GWOC41vVkKrOEJGUfGbpAbzkDYzynYhUEJFqhzDEnTEmKKqQmQlZWZCd7abh90PTvXth3z73OPym+vfnQrft26FYMff+0DoyM91twwYoX969PzyWWJ6mp8PRR+f5UQPsZj+bySCle284+eR85z0cQbYRHMdf+1tP9577WyIQkX64UgM1a9aMSnCHKikpiaZNm5KZmUnt2rWZOHEiFSpUAGDJkiVce+21rF+/nuzsbC677DJuu+22P47yP/roI26//XYyMjIoXrw4Z555JqNGjQpwa0yByM6G3bth/353O3AAdu50z4d2bKHbtm0g4u7v3+92eKGd4YED7rZvH6xdC1Wq5L3jzcqCn392O5bQjjXnfOH39+2DPXv+urMNrXPVKqha9c94w3fMBw4E/enGnzxK/V+kKFd1h/L7YPbshhSJs0QQMVUdB4wDSEtLK5S95JUsWZL58+cDcPnllzN69GiGDx/Onj17OPfccxk7diydOnUiIyODCy64gDFjxjBw4EAWL17MoEGDmDJlCg0bNiQrK4tx48YFuzGJIjsbduyALVvgp59g0SJISnI74n373I6wYkW30wvtyFesgKOO+nPnvnev27nv3euWtXEjFC/+507TLyVLuliLFHHT8Psi8OWXUK+eey70fPg0/H2VKkHRopCc7B4nJ7tbq1YukVWv/vfXkpLcduZcd871JCVBmTJ/Phb5c/153UIJsUKFv68z/HH4jjN0P1anSUl/+4q3793OTZ/cxHPznqNepXo82v05iqT4MxRGkIlgPX8dA7a691zMO/XUU1m4cCEAr776Kq1bt6ZTJzeCYqlSpXjqqado164dAwcO5MEHH2T48OE0bNgQcCWLa66Jy2FR/aUKv/4Ka9bA6tXu/pIlkJEBmze74veuXW6nX7So21Hv3BnZsitXdkfnRYu6HdXSpW4nW7Soe/7YY6FcOShRwiWHUqVcsihe3CWH2rXdfKEj/BIloGzZP3dooVt2ttsph+YtVszt8IsW/fOWcwdo4lJWdhanPX8ay7cu5+bTbuaudndRsmheA7cduSATwWRgkIi8DrQCdhRI+8B114F3ZF5gmjeHxx6LaNasrCw+//xz+vbtC7hqoZNOOukv89StW5ddu3bx+++/s3jxYoYOHVqw8cayUDVJRgb89pvbea9Y4XbAu3fD1q0wb57bQc6f755ftw5+/z3vZZYu7Y5qa9SABg3ctHx5t2M9cACaNnU765o13VFoyZJuJ5zLUZoxftqasZVKJSuRVCSJe8+8lxrla5B2bJrv6/UtEYjIa0A7oIqIpAN3AkUBVPVp3NijXXFj7GYAV/gVSzTs2bOH5s2bs379eho1asRZZ50VdEiFQ3Y2bNrkjtJXrIAff3RH74sXu6PnrVtdFUxWlksAkShRwu3AmzRx9dtt2rh68/Ll4dRTITUVqlVzj4vH5RCzJs6oKq8seoUhHw/h/g73c9VJV3F+o/Ojtn4/zxq66CCvKzCwwFcc4ZF7QQu1EWRkZNC5c2dGjx7N4MGDSU1NZcaMGX+Zd9WqVZQpU4Zy5crRuHFj5syZwwknnBBI3AVq71747jt45x2YO9ft4DfkUcgrVcpNjz4aGjd2R/Y1a8Ixx7gj8bp13ZF8UpKreqlWzb2nQgXbuZu4sm7HOvpP6c/UFVM5pfoptK7ZOuoxxERjcSwpVaoUTzzxBOeddx4DBgzgkksuYeTIkXz22Wd07NiRPXv2MHjwYG6++WYAbrrpJv7xj3/Qpk0bGjRoQHZ2NuPGjaN///4Bb0ketm2D5cvdkfyaNa5aZv58+PRTd1QfrkkTSEuDE06AOnUgJcVVzYR29sYkuNcWvcbVH15NlmbxWOfHGNRyEElFov/fsETggxNPPJFmzZrx2muv0bt3b95//32uvfZaBg4cSFZWFr1792bQoEEANGvWjMcee4yLLrqIjIwMRIRu3boFvAW4qpeFC91O/5NP4NtvXbVOTsnJcNxxcM45rh6+XTt3q5rr+BfGmDAVS1akVfVWjOs2jtoVg+s+JubGLE5LS9OcA9MsW7aMRo0aBRRR9Pi+nT/8AB9+6Kp2Fi50jbMhZcpAixauGqdDB3dUn5LiqmyKWCe2xkQiMzuTR799lP1Z+xl+xnDAtQ9Eo+cAEZmjqrm2PFuJIFHt2wcTJ7rz55csgZkz3SmX4HbuLVvC5Ze7nX9Kijvd0Rhz2BZsXEDfyX2Zs2EOFza+8I8EUBi6j7FEkEi2b4fXXoPPPnNH/SH167vqnLQ0d6psx44BBWhM/NmXuY97ZtzD/V/fT6WSlXir51tc0OiCQpEAQuImEUSreBWUw67CW7ECxo6FL75wV85mZ7vne/SA7t2hd293zrwxxhcrflvBA18/wMVNL+aRTo9QuVTloEP6m7hIBCVKlGDr1q1x2xV1aDyCEiVKRP6mL7+E//zHTUP69IELLoDOnd3FVMYYX+zav4v3f3ifS5pdQpOjmvDDoB+oU7FO0GHlKS4SQfXq1UlPT2fz5s1Bh+Kb0Ahl+Zo+HSZPdjv/0NXVQ4e6BNC4sXVNYEwUfPrTp/T7sB9rtq+hRbUWNKraqFAnAYiTRFC0aNHEHblr3z4YNgzee8/1Ogmuv5r+/eH2293FWMYY323bs40bP7mR8fPH06ByA77q8xWNqsbG2YxxkQgSTlYWvPwyPPqo6wAt1CXwdddBv36QAKfSGlOYZGVn0Xp8a37c+iPD2gzjjrZ3UCL5EKpyA2aJIJbs2+d2/sOGucfJya6+/+qrXcOvMSaqtmRs+aOTuJEdRlKzfE1aVDvsUXcDY1cCxYKvv4auXV1na8OGuT557rnHdbj24YeWBIyJMlXlpQUv0eDJBjw39zkAzmt4XkwmAbASQeE2cyZcfz1884173Lo19O3rGn+t4deYQKzZvoarP7yaaT9N47Qap3FGrTOCDumIWSIojDZscFU+ixa5x+ecAy+95BqBjTGBeXnhy1wz5RpUlSe7PMmAkwdQRGK/YiX2tyCe/Por3HabO9Nn0SKXABYtctU/lgSMCVzVUlVpXaM1SwYsYVDLQXGRBMBKBIXHkiVuUJWdO13/PrfdBudHb2AKY8zfHcg6wKhvR3Eg6wC3t72dzvU606lup7i7cDU+0lksy8yEu+5yfferwtSpMGeOJQFjAjZvwzxaPdeKYZ8PY+mWpX908xJvSQCsRBCs5cuhWzdYudKN1PXNN24AF2NMYPZm7mXEVyN48OsHqVKqCm9f+Db/aPSPoMPylZUIgpCd7a4HaNLEJYFHHnENxJYEjAncyt9W8vA3D3PZCZexbOCyuE8CYCWC6MvKgk6dXG+gzZrBhAlw4olBR2VMQtu1fxfvLnuX3if0pslRTVg+aHmgI4ZFm5UIomntWjj5ZJcErr7adQxnScCYQE1bOY3GYxpz+XuXs2zzMoCESgJgiSB6MjMhNRXmzXPVQmPH2kVhxgRoa8ZWLn/vcs5+5WxKFS3Ff6/4b8x0ElfQrGooWoYMcWMAjxrlOoczxgQm1Encyt9WMvz04dx2xm0x1UlcQbNEEA3Tp8OYMW7Ad0sCxgRm8+7NVC5VmaQiSTzQ8QFqVahF82OaBx1W4KxqyG/r1rkO44oXh88/hyL2kRsTbarKC/NeoMFTDXh2zrMA9GjYw5KAx0oEftq7F9q0cb2Efv+9ayMwxkTV6u2r6fdBPz5d9Smn1zyd9rXbBx1SoWOJwE+dO7szha66Clq2DDoaYxLOxAUTuWbKNYgIY7qO4eq0q+Omf6CCZInAL6++CjNmuCuHx40LOhpjEtLRZY7mjFpn8HS3p6lZvmbQ4RRaEuo/I1akpaXp7Nmzgw4jf7t3Q40arj0gPd0NKGOM8d2BrAM8+PWDZGkWd7S9I+hwChURmaOqabm9ZiUCP3TpAtu2uVKBJQFjomLuhrn8+/1/s+DXBVzc9GJUNS47iPODVZYVtOefh//+F668Ei66KOhojIl7ew7s4ZbPbqHlsy35dfevvPuvd3nlH69YEjgEviYCETlbRJaLyEoRuSWX12uKyJciMk9EFopIVz/j8d2ECa5hOC0NHn886GiMSQirtq3ikW8foU/zPiwdsJTzGp4XdEgxx7dEICJJwGigC5AKXCQiOc+fvA14U1VPBHoBY/yKx3fz57vxhE86yV0vUKpU0BEZE7d+3/c7E+ZPAKDxUY1Zce0Knjv3OSqWrBhsYDHKzxJBS2Clqq5S1f3A60CPHPMoUM67Xx74xcd4/KMKPXu6+++/D+XK5T+/MeawTV0xlSZjmtB3ct8/OomrVaFWwFHFNj8TwXHAurDH6d5z4e4CLhWRdGAqcG1uCxKRfiIyW0Rmb9682Y9Yj8y4cW5cge7d3XjDxpgCtyVjC73f7c05r55D2eJl+frfXydsJ3EFLejG4ouACapaHegKTBT5+9UeqjpOVdNUNa1q1apRDzJfO3fCffe5+6++GmwsxsSpUCdxry9+nTvOuIO5/eZySvVTgg4rbvh5+uh6oEbY4+rec+H6AmcDqOq3IlICqAJs8jGugjVwIKxZAy+8YO0CxhSwX3f9StXSVUkqksTDZz1MrQq1aHZ0s6DDijt+lghmAfVFpLaIFMM1Bk/OMc9aoAOAiDQCSgCFsO4nD9Onw8SJcP310KdP0NEYEzdUlefnPs/xTx3PuDnuyvzux3e3JOAT30oEqpopIoOAaUASMF5Vl4jICGC2qk4GhgLPisj1uIbjPhorlzpnZ7supStVgv/8J+hojIkbq7at4qoPruKLn7+gba22dKzTMeiQ4p6vVxar6lRcI3D4c3eE3V8KtPYzBt+8+SYsWAAjR0Lp0kFHY0xceHH+iwyYOoAkSeLpc57mqpOusk7iosC6mDgcqu6q4Vq14IYbgo7GmLhxbNljObP2mYw9ZyzVy1UPOpyEYYngcHz2mZuec44bcMYYc1j2Z+3n/v/dT7Zmc1e7uzir7lmcVfesoMNKOFbmOlT797t+hABu+VuvGcaYCM1aP4uTxp3EndPvZNW2VcRK82A8skRwqCZOdIPNDBvmupo2xhySjAMZ3PjJjZzy/Cls27ONyb0m89L5L1kncQGy8QgORXY2VK4MIrB5MyQlBROHMTFsyaYltBjXgiuaX8EDHR+gfInyQYeUEGw8goJyzz2wfbs7bdSSgDER27F3B+8se4crTryCxkc1ZuW1K6lR3krUhYVVDUUqMxNGj4ayZWHUqKCjMSZmTPlxCo3HNObKD67khy0/AFgSKGQsEURq2jTYtAkefdQNQWmMydfm3Zu55J1L6PZaNyqWrMi3fb+lYZWGQYdlcmFVQ5GaMMFNzzsvyCiMiQlZ2Vm0eaENP2/7mbvb3c0tbW6hWFKxoMMyebBEEIn0dJg0CZo2dY3Fxphcbdy1kaNKH0VSkSRGdRpFSoUUmhzVJOiwzEFEXMchIonbtebw4W56773BxmFMIZWt2Twz+xkaPNmAZ2Y/A0C3Bt0sCcSIgyYCETlNRJYCP3iPTxCR2B1S8nAsc6Mg0a1bsHEYUwit/G0lHV7qQP8p/Tn5uJPpXK9z0CGZQxRJ1dCjQGe8LqRVdYGInOFrVIXJunUwaxZcfbW7fsAY84cX5r3AgKkDKJZUjGe7P0vfE/vahWExKKI2AlVdl+PLzfInnELottvc9Jprgo3DmEKoZvmadK7bmdFdR3NcuZwj0ZpYEUkiWCcipwEqIkWBIcAyf8MqJH75BV56CTp3hhNOCDoaYwK3L3Mf9/3vPrI1mxHtR9ChTgc61OkQdFjmCEXSWNwfGIgbeH490BwY4GNMhcfIkW46dGiwcRhTCHyf/j0njTuJu7+6m7U71loncXEkkhLB8ap6SfgTItIa+NqfkAqR+fPdtKONkGQS1+79u7n9y9t57LvHOK7ccXx40Yec0+CcoMMyBSiSEsGTET4XXzZsgK+/dv0KWeOXSWBrdqxhzKwx9E/rz5IBSywJxKE8SwQicipwGlBVRMKH4SqHG4M4vn3/vZt27RpsHMYEYPve7UxaOokrW1xJatVUVg5eaSOGxbH8qoaKAWW8ecqGPf878E8/gyoUJk5001atgo3DmCh7/4f3uWbKNWzavYk2NdvQsEpDSwJxLs9EoKpfAV+JyARVXRPFmAqH776DOnWgXLmgIzEmKjbt3sTgjwbzxpI3aHZ0MyZfNNk6iUsQkTQWZ4jIQ0BjoEToSVU907eogvbuu+7U0YcfDjoSY6IiKzuL1uNbs3bHWu5pfw83t76ZoklFgw7LREkkieAV4A2gG+5U0suBzX4GFbgXXnDT0NjExsSpX3b+wjFljiGpSBKPn/04KRVSSK2aGnRYJsoiOWuosqo+DxxQ1a9U9d9A/JYGsrPhgw+gQwcob0PomfiUrdmMnTWWhk815OnZTwPQtX5XSwIJKpISwQFvukFEzgF+ASr5F1LApkxx07Ztg43DGJ/8uPVHrvrgKmasmUHHOh3pUq9L0CGZgEWSCO4RkfLAUNz1A+WA6/wMKlBjvI5Vr78+2DiM8cHzc59n0EeDKJFcgvHnjqdP8z7WSZw5eCJQ1Q+9uzuA9vDHlcXxJzMTPv7YDUBTpkzQ0RhT4FIqpNClXhdGdx1NtbLVgg7HFBL5XVCWBFyI62PoY1VdLCLdgFuBksCJ0Qkxiv73Pze94opg4zCmgOzL3Md/ZvwHgHvOvMc6iTO5yq9E8DxQA5gJPCEivwBpwC2q+l4UYou+t9920549g43DmALwzbpv6Du5Lz9s+YF/N/83qmrVQCZX+SWCNKCZqmaLSAlgI1BXVbdGJ7QATJwI9epBdbuK0sSuXft3Mfzz4Tw580lqlK/Bx5d8bKOGmXzld/roflXNBlDVvcCqQ00CInK2iCwXkZUickse81woIktFZImIvHooyy9QO3a4m3UpYWLc2h1reWbOMww8eSCLr1lsScAcVH4lgoYistC7L0Bd77EAqqrN8luw18YwGjgLSAdmichkVV0aNk99YBjQWlW3ichRR7AtR+ZDr038rLMCC8GYw7VtzzbeWvoW/U7qR2rVVFYNWcWxZY8NOiwTI/JLBI2OcNktgZWqugpARF4HegBLw+a5ChitqtsAVHXTEa7z8M2c6aYdrCHNxJZ3l73LgKkD2Lx7M21rteX4KsdbEjCHJL9O5460o7njgHVhj9OBnPUuDQBE5Gtc19Z3qerHORckIv2AfgA1a9Y8wrDy8NlnULo0HGt/IBMbNu7ayLUfXcukpZNofkxzplw8heOrHB90WCYGRTR4vc/rrw+0A6oDM0SkqapuD59JVccB4wDS0tIKfny8Zctg6VK46iooEkmvG8YEKys7i9NfOJ11O9Yx8syR3HjajdZJnDlsfiaC9bjTT0Oqe8+FSwe+V9UDwM8i8iMuMczyMa6/e/99N+3TJ6qrNeZQpf+ezrFljyWpSBJPnP0EtSvWtq6izRGL6PBXREqKyKGWOWcB9UWktogUA3oBk3PM8x6uNICIVMFVFa06xPUcuU8/ddNTTon6qo2JRLZm8+T3T9LwqYaMnTUWgC71u1gSMAXioIlARLoD84GPvcfNRSTnDv1vVDUTGARMA5YBb6rqEhEZISLnerNNA7aKyFLgS+CmQK5TmDULkpOtWsgUSj9s+YEzXjiDwR8Ppk3NNnRr0C3okEyciaRq6C7cGUDTAVR1vojUjmThqjoVmJrjuTvC7itwg3cLjiq0aBFoCMbk5rm5zzFo6iBKFS3Fi+e9SO9mve3qYFPgIuqGWlV35PjxFXyDbVB27IBdu6BHj6AjMeZv6lasS/fju/NUl6c4uszRQYdj4lQkiWCJiFwMJHkXgA0GvvE3rCj67DM3rVEj//mMiYK9mXsZ8dUIAEZ2GEn72u1pX7t9wFGZeBdJpfi1uPGK9wGv4rqjvs7HmKLrq6/c9Nxz85/PGJ99vfZrmj/dnPv+dx+bd2/G1Zwa479ISgQNVXU4MNzvYAKxxrtuzoalNAHZuW8nt35+K6NnjaZWhVpMu3Qanep2Cjosk0AiSQSjROQYYBLwhqou9jmm6Fq3zgahMYFK/z2d5+Y9x7Utr+XeDvdSppj9Hk10HbRqSFXb40Ym2ww8IyKLROQ23yOLlvR08KvbCmPysDVj6x/XAzSq2ohVg1fxeJfHLQmYQER04ryqblTVJ4D+uGsK7sj/HTFCFTZvhpSUoCMxCUJVmbR0EqljUhn88WCWb1kOYMNGmkBFckFZIxG5S0QW4Qav/wbXXUTs++EHN7Urik0UbNi5gQvevICeb/WkRrkazL5qtnUSZwqFSNoIxgNvAJ1V9Ref44mu0Kmj55wTbBwm7oU6iVu/cz0PdnyQ60+9nuQiQff5aIxz0F+iqp4ajUACsWCBmzZpEmwcJm6t27GO48odR1KRJEZ3HU3tirVpULlB0GEZ8xd5Vg2JyJvedJGILAy7LQobuSy2/eIVcIoVCzYOE3eysrN44vsnaDj6z07iOtfrbEnAFEr5lQiGeNP47eFq3TprKDYFbtnmZfSd3Jdv07+lS70udD++e9AhGZOvPEsEqrrBuztAVdeE34AB0QnPRzt2wJIlcN55QUdi4si4OeNo/kxzftz6IxPPn8iUi6dQs7ydnmwKt0hOH81tNPcuBR1I1K1d604fPfHEoCMxcaR+pfqc3/B8lg5cyqXNLrWeQk1MyLNqSESuwR3518nRJlAW+NrvwHw3yxsErVmzYOMwMW3PgT3cNf0uRIT7O95vncSZmJRfG8GrwEfAfcAtYc/vVNXffI0qGqZMcdNGjYKNw8SsGWtmcOXkK1nx2wr6n9QfVbUSgIlJ+VUNqaquBgYCO8NuiEgl/0PzWWammxYvHmwcJub8vu93BkwZQNsJbcnSLD6/7HPGdhtrScDErIOVCLoBc3AD0YT/yhWo42Nc/luzBjp2DDoKE4N+2fkLE+ZP4IZTbmBE+xGULlY66JCMOSJ5JgJV7eZNIxqWMuYsWACXXx50FCZGbMnYwptL3mTAyQNoWKUhPw/52UYMM3Ejkr6GWotIae/+pSLyiIjE9vlw+/a5aVZWsHGYQk9VeWPxG6SOTuW6j6/jx60/AlgSMHElktNHxwIZInICMBT4CZjoa1R++81r67aGYpOPX3b+wnlvnEevt3tRq0It5vSbY1cGm7gUSa9XmaqqItIDeEpVnxeRvn4H5quNG920Tmw3cxj/ZGVnccYLZ7B+53oePuthhpwyxDqJM3Erkl/2ThEZBvQGTheRIkBRf8Py2c6dbnrUUcHGYQqdNdvXUL1cdZKKJDHmnDHUqViHepXqBR2WMb6KpGroX7iB6/+tqhtxYxE85GtUflu71k2tsznjycrO4pFvH6HR6EaMne06ietUt5MlAZMQIhmqciPwClBeRLoBe1X1Jd8j81PofO8qVYKNwxQKizct5rTxpzH0k6F0qNOB8xqeF3RIxkRVJGcNXQjMBHoCFwLfi8g//Q7MVz/95KY2aH3Ce3r207R4pgWrtq3i1X+8yuRek6leLj4G4DMmUpG0EQwHTlbVTQAiUhX4DJjkZ2C+Wr/eTa1EkLBC3UE0qtKIno178ljnx6haumrQYRkTiEgSQZFQEvBsJcJB7wutVavctESJYOMwUZdxIIM7vryDJEnigbMeoG1KW9qmtA06LGMCFckO/WMRmSYifUSkDzAFmOpvWD4rXhzKlQs6ChNl01dPp9nYZoz6dhS79u9CVYMOyZhCIZIxi28SkX8Abbynxqnqu/6G5bO9e22c4gSyY+8Obv70ZsbNHUfdinX54rIvrKtoY8LkNx5BfeBhoC6wCLhRVddHKzBfZWRAyZJBR2GiZMOuDby86GVuPPVG7m5/N6WKlgo6JGMKlfyqhsYDHwIX4HogffJQFy4iZ4vIchFZKSK35DPfBSKiIpJ2qOs4LKtWWUNxnNu8ezNPfu9+sg2rNGT1kNU81OkhSwLG5CK/qqGyqvqsd3+5iMw9lAWLSBIwGjfUZTowS0Qmq+rSHPOVBYYA3x/K8g9bdjb8+quVCOKUqvLa4tcY/NFgft/3O53rdaZB5QZ2RpAx+civRFBCRE4UkRYi0gIomePxwbQEVqrqKlXdD7wO9Mhlvv8ADwB7Dzn6w7F9u5umpERldSZ61u1YR/fXunPJO5dQr1I95l09zzqJMyYC+ZUINgCPhD3eGPZYgTMPsuzjgHVhj9OBVuEzeAmlhqpOEZGb8lqQiPQD+gHUrHmEPWBv2eKmdtZQXMnMzqTdi+3YuGsjj3Z+lGtbXktSkaSgwzImJuQ3MI2vp1V4ndc9AvQ52LyqOg4YB5CWlnZk5/yFeh490oRiCoXV21dTo1wNkosk80y3Z6hTsQ51KlqvssYcCj8vDFsP1Ah7XN17LqQs0ASYLiKrgVOAyb43GIfGKrbuJWJaZnYmD3/zMI1GN2LMrDEAdKzT0ZKAMYfBzw7WZwH1RaQ2LgH0Ai4OvaiqO4A/Tt0Rkem4U1Rn+xgT7N/vphUq+Loa45+Fvy6k7+S+zP5lNj2O78EFqRcEHZIxMc23RKCqmSIyCJgGJAHjVXWJiIwAZqvqZL/Wna/Q6GTFiweyenNkxswaw5CPh1CxREXe+Ocb9EztiYR6kzXGHJaDJgJx/7JLgDqqOsIbr/gYVZ15sPeq6lRydEehqnfkMW+7iCI+UhkZbppso03FklAncU2OakKvJr14tPOjVCll14IYUxAi2RuOAbJxZwmNAHYCbwMn+xiX/6xqKCbs3r+b2764jeQiyTzU6SHOqHUGZ9Q6I+iwjIkrkTQWt1LVgXjn+avqNiB2h/ba612uYFVDhd7nqz6n6dimPPb9Y+zL2medxBnjk0hKBAe8q4QV/hiPINvXqPy0cqWb2pXFhdb2vdu58ZMbeX7e89SvVJ8ZfWZweq3Tgw7LmLgVSYngCeBd4CgRuRf4HzDS16j8VLasm9rpo4XWr7t+5fXFr/N/rf+PBf0XWBIwxmeRdEP9iojMAToAApynqst8j8wvS5e6AWnsTJNCJbTzH3LKEI6vcjyrr1ttjcHGREkkZw3VBDKAD8KfU9W1fgbmmw0bICsr6CiMR1V5ZdErDPl4CLv276Jr/a7Ur1zfkoAxURRJG8EUXPuAACWA2sByoLGPcfnnp5+gdu2gozDA2h1r6f9hfz5a+RGnVj+V5899nvqV6wcdljEJJ5Kqoabhj72O4gb4FpHfNm2y0ckKgczsTNpNaMem3Zt44uwnGHDyAOskzpiAHPJVVao6V0RaHXzOQqyO9UcTlFXbVlGrfC2SiyTzbPdnqVupLikVUoIOy5iEFkkbwQ1hD4sALYBffIvIT6G2gRo18p/PFLjM7ExGfTOKO6ffyYNnPcjgVoPpUKdD0GEZY4isRFA27H4mrs3gbX/C8dmBA25aLHavh4tF8zfOp+/kvszdMJfzG55Pz9SeQYdkjAmTbyLwLiQrq6o3Rikef4X6GSriZ+/bJtxTM5/i+mnXU7lkZSb1nGQ9hRpTCOWZCEQk2etBtHU0A/LVrl1BR5AwQp3ENTu6GZc0vYRHOj9CpZKVgg7LGJOL/EoEM3HtAfNFZDLwFrA79KKqvuNzbAVvxw43tTYC3+zav4vhnw+naFJRHu70sHUSZ0wMiKSOpASwFdf7aDeguzeNPZs2uWl27HaVVJh98tMnNBnThCdnPsmBrAPWSZwxMSK/EsFR3hlDi/nzgrKQ2PyHhxJArVrBxhFntu3Zxg2f3MCE+RM4vvLxzLhiBm1qtgk6LGNMhPJLBElAGf6aAEJiMxHYWUO+2LR7E5OWTmJYm2Hc0fYOSiSXCDokY8whyC8RbFDVEVGLJBpC4xVbIjhiG3dt5LVFr3H9qde7TuKGrKZyqcpBh2WMOQz5tRHEX/eca71+8uz00cOmqrw4/0VSR6cy7PNhrNi6AsCSgDExLL89Yvxd9lm6tJvaMJWHZfX21Zz9ytn0eb8PqVVTmd9/vnUSZ0wcyLNqSFV/i2YgURGqGrJhKg9ZZnYm7V9sz5aMLYzuOpr+af0pIlayMiYeHHKnczFt/Xo3tTaCiK38bSW1K9QmuUgy488dT52KdahVwc66MiaeJNYhXei8dhum8qAOZB1g5H9H0nhMY0bPGg1A+9rtLQkYE4cSq0Swdi2UKgVFiwYdSaE2d8Nc+k7uy/yN8+mZ2pN/Nf5X0CEZY3yUWIkgMxOSbPCT/Dzx/RPcMO0GqpauyjsXvsP5jc4POiRjjM8SKxHs3WvDVOYh1EncicecyGUnXMaoTqOoWLJi0GEZY6IgsRLBkiV26mgOO/ftZNjnwyieVJxRnUdxeq3TOb3W6UGHZYyJosRqLK5UCbZtCzqKQuPjlR/TZGwTxswag6LWSZwxCSqxSgT79kFqatBRBG5rxlZu+OQGXlrwEo2qNOLrf3/NqTVODTosY0xAEqtEsHChXUwGbN2zlXeXvcvtZ9zOvKvnWRIwJsH5mghE5GwRWS4iK0Xkllxev0FElorIQhH5XET8O0ld1XVDnZnp2yoKsw07N/DwNw+jqjSo3IA1161hRPsRFE+2xGhMovMtEXjjHY8GugCpwEUikrNeZh6QpqrNgEnAg37F80cCqFbNt1UURqrK+HnjaTS6Ebd/eTsrf1sJYGcEGWP+4GeJoCWwUlVXqep+4HWgR/gMqvqlqnojyvMdUN23aEL9DKWk+LaKwubnbT/T6eVO9J3clxOOOYEF/RdYJ3HGmL/xs7H4OGBd2ON0oFU+8/cFPsrtBRHpB/QDqFmz5uFFk2BjEWRmZ3LmS2eyNWMrY88ZS7+T+lknccaYXBWKs4ZE5FIgDWib2+uqOg4YB5CWlnZ45zju2uWmWVmH9fZYsWLrCupUrENykWRe6PECdSvWpUb5GkGHZYwpxPw8RFwPhO+BqnvP/YWIdASGA+eq6j7fotmzx01DYxLEmQNZB7hnxj00GduEp2Y+BUC7lHaWBIwxB+VniWAWUF9EauMSQC/g4vAZRORE4BngbFXd5GMsrnsJgKpVfV1NEGb/Mpu+k/uy8NeF9GrSi4uaXhR0SMaYGOJbIlDVTBEZBEwDkoDxqrpEREYAs1V1MvAQUAZ4S0QA1qrqub4EtHmzL4sN2uPfPc4Nn9zAMWWO4f1e73Pu8f58fMaY+OVrG4GqTgWm5njujrD7Hf1c/1+EximOk6qhUCdxacem0ffEvjx41oNUKFEh6LCMMTGoUDQWR0WokbhUqWDjOEK/7/ud//v0/yiRXIJHz36U1jVb07pm66DDMsbEsMQ5nzA7201jeDyCqSum0nhMY8bNHUdykWTrJM4YUyASr0RQJPZy35aMLVz38XW8sugVGldtzKSek2hVPb9LMowxJnKJlwhisESwbc82PvjxA+5seye3nn4rxZIS46I4Y0x0JE4iiLGqofW/r+eVRa9w02k3Ub9yfdZct8Yag40xvoi9epLDFSNVQ6rKs3OeJXVMKndNv4uftv0EYEnAGOObwr1XLEgxUDX0028/0eGlDvT7sB8tqrVg4TULqVepXtBhGWPiXOJUDf3yi5sW0jNtMrMz6fBSB37b8xvPdHuGK1tcaZ3EGWOiInESQblyblq2bLBx5LB8y3LqVqpLcpFkXjzvRepWqkv1cv71xm2MMTklziFnqCTgurII3P6s/dw9/W6ajm3K6JmjAWib0taSgDEm6hKnRBBSCBLBzPUz6Tu5L4s3LebiphdzSbNLgg7JGJPAEicRFJK2gce+e4yhnwylWplqfHDRB3Rr0C3okIwxCS7xEkFAJYJQJ3Etj2vJVS2u4oGOD1C+RPlAYjHGmHCWCHy2Y+8Obv70ZkoWLcljZz/GaTVO47Qap0U1BmOMyY81Fvvog+UfkDomlefmPUfxpOLWSZwxplCyEoEPNu/ezJCPh/Da4tdoelRT3vvXe5x83Mm+r9cYYw6HJQIf7Ni3g6krpnJ3u7u5pc0t1kmcMaZQs0RQQNbtWMfLC1/mlja3UK9SPdZct8Yag40xMcHaCI5Qtmbz9OynaTymMff8954/OomzJGCMiRWJkwhCCjARrNi6gjNfPJNrplxDy+NasuiaRdZJnDEm5iRe1VAByczO5KyJZ7F973aeP/d5rmh+BVIIrlo2xphDlXiJ4Ah31ss2L6N+5fokF0lm4vkTqVupLseWPbYAAjTGmGAkTtXQESaCfZn7uPPLO2n2dDOemvkUAKfXOt2SgDEm5lmJIALfpX9H38l9Wbp5Kb2b9aZ3s94FHJwxxgTHEsFBjPpmFDd9ehPVy1Vn6sVT6VK/iw/BGWNMcCwR5CFbsykiRTi1xqn0T+vP/R3vp1zxcj4GaIwxwbBEkMP2vdsZOm0opYqW4smuT1onccaYuGeNxWHe++E9Uken8uKCFylbvKx1EmeMSQiJUyIIySURbNq9iUFTB/HW0rdofkxzPrz4Q1pUaxFAcMYYE32JkwjyObr/fd/vfLrqU+49815uOu0miiYVjWJgxhgTrMRLBF6JYO2OtUxcMJFbT7+VepXqsfa6tZQtXjbAAI0xJhi+thGIyNkislxEVorILbm8XlxE3vBe/15EUnwLxksE2ShjZo2h8ZjGjPzfyD86ibMkYIxJVL4lAhFJAkYDXYBU4CIRSc0xW19gm6rWAx4FHvArHlRZXhnavdmVgVMHcmr1U1kyYIl1EmeMSXh+lghaAitVdZWq7gdeB3rkmKcH8KJ3fxLQQXzquS0zO5POvWHRlqW80OMFpl06jZQKKX6syhhjYoqfbQTHAevCHqcDrfKaR1UzRWQHUBnYEj6TiPQD+gHUrFnzsIJJbpjKy8vaU/e28VSrnHJYyzDGmHgUE9cRqOo4VU1T1bSqVase3kJ69KDNhC8sCRhjTA5+JoL1QI2wx9W953KdR0SSgfLAVh9jMsYYk4OfiWAWUF9EaotIMaAXMDnHPJOBy737/wS+ULuc1xhjosq3NgKvzn8QMA1IAsar6hIRGQHMVtXJwPPARBFZCfyGSxbGGGOiyNcLylR1KjA1x3N3hN3fC/T0MwZjjDH5i4nGYmOMMf6xRGCMMQnOEoExxiQ4SwTGGJPgJNbO1hSRzcCaw3x7FXJctZwAbJsTg21zYjiSba6lqrlekRtzieBIiMhsVU0LOo5osm1ODLbNicGvbbaqIWOMSXCWCIwxJsElWiIYF3QAAbBtTgy2zYnBl21OqDYCY4wxf5doJQJjjDE5WCIwxpgEF5eJQETOFpHlIrJSRG7J5fXiIvKG9/r3IpISQJgFKoJtvkFElorIQhH5XERqBRFnQTrYNofNd4GIqIjE/KmGkWyziFzofddLROTVaMdY0CL4bdcUkS9FZJ73++4aRJwFRUTGi8gmEVmcx+siIk94n8dCEWlxxCtV1bi64bq8/gmoAxQDFgCpOeYZADzt3e8FvBF03FHY5vZAKe/+NYmwzd58ZYEZwHdAWtBxR+F7rg/MAyp6j48KOu4obPM44BrvfiqwOui4j3CbzwBaAIvzeL0r8BEgwCnA90e6zngsEbQEVqrqKlXdD7wO9MgxTw/gRe/+JKCDiEgUYyxoB91mVf1SVTO8h9/hRoyLZZF8zwD/AR4A9kYzOJ9Ess1XAaNVdRuAqm6KcowFLZJtVqCcd7888EsU4ytwqjoDNz5LXnoAL6nzHVBBRKodyTrjMREcB6wLe5zuPZfrPKqaCewAKkclOn9Ess3h+uKOKGLZQbfZKzLXUNUp0QzMR5F8zw2ABiLytYh8JyJnRy06f0SyzXcBl4pIOm78k2ujE1pgDvX/flC+DkxjCh8RuRRIA9oGHYufRKQI8AjQJ+BQoi0ZVz3UDlfqmyEiTVV1e5BB+ewiYIKqjhKRU3GjHjZR1eygA4sV8VgiWA/UCHtc3Xsu13lEJBlXnNwalej8Eck2IyIdgeHAuaq6L0qx+eVg21wWaAJMF5HVuLrUyTHeYBzJ95wOTFbVA6r6M/AjLjHEqki2uS/wJoCqfguUwHXOFq8i+r8finhMBLOA+iJSW0SK4RqDJ+eYZzJwuXf/n8AX6rXCxKiDbrOInAg8g0sCsV5vDAfZZlXdoapVVDVFVVNw7SLnqursYMItEJH8tt/DlQYQkSq4qqJVUYyxoEWyzWuBDgAi0giXCDZHNcromgxc5p09dAqwQ1U3HMkC465qSFUzRWQQMA13xsF4VV0iIiOA2ao6GXgeV3xciWuU6RVcxEcuwm1+CCgDvOW1i69V1XMDC/oIRbjNcSXCbZ4GdBKRpUAWcJOqxmxpN8JtHgo8KyLX4xqO+8TygZ2IvIZL5lW8do87gaIAqvo0rh2kK7ASyACuOOJ1xvDnZYwxpgDEY9WQMcaYQ2CJwBhjEpwlAmOMSXCWCIwxJsFZIjDGmARnicAUSiKSJSLzw24p+cy7qwDWN0FEfvbWNde7QvVQl/GciKR692/N8do3Rxqjt5zQ57JYRD4QkQoHmb95rPfGafxnp4+aQklEdqlqmYKeN59lTAA+VNVJItIJeFhVmx3B8o44poMtV0ReBH5U1Xvzmb8PrtfVQQUdi4kfViIwMUFEynjjKMwVkUUi8reeRkWkmojMCDtiPt17vpOIfOu99y0ROdgOegZQz3vvDd6yFovIdd5zpUVkiogs8J7/l/f8dBFJE5H7gZJeHK94r+3ypq+LyDlhMU8QkX+KSJKIPCQis7w+5q+O4GP5Fq+zMRFp6W3jPBH5RkSO967EHQH8y4vlX17s40Vkpjdvbj22mkQTdN/bdrNbbjfcVbHzvdu7uKvgy3mvVcFdVRkq0e7ypkOB4d79JFx/Q1VwO/bS3vP/B9yRy/omAP/07vcEvgdOAhYBpXFXZS8BTgQuAJ4Ne295bzodb8yDUExh84RiPB940btfDNeLZEmgH3Cb93xxYDZQO5c4d4Vt31vA2d7jckCyd78j8LZ3vw/wVNj7RwKXevcr4PoiKh309223YG9x18WEiRt7VLV56IGIFAVGisgZQDbuSPhoYGPYe2YB471531PV+SLSFjdYydde1xrFcEfSuXlIRG7D9VPTF9d/zbuqutuL4R3gdOBjYJSIPICrTvrvIWzXR8DjIlIcOBuYoap7vOqoZiLyT2++8rjO4n7O8f6SIjLf2/5lwKdh878oIvVx3SwUzWP9nYBzReRG73EJoKa3LJOgLBGYWHEJUBU4SVUPiOtRtET4DKo6w0sU5wATROQRYBvwqapeFME6blLVSaEHItIht5lU9UdxYx10Be4Rkc9VdUQkG6Gqe0VkOtAZ+BduoBVwo01dq6rTDrKIParaXERK4frfGQg8gRuA50tVPd9rWJ+ex/sFuEBVl0cSr0kM1kZgYkV5YJOXBNoDfxtzWdw4zL+q6rPAc7jh/r4DWotIqM6/tIg0iHCd/wXOE5FSIlIaV63zXxE5FshQ1ZdxnfnlNmbsAa9kkps3cB2FhUoX4Hbq14TeIyINvHXmSt1oc4OBofJnV+qhroj7hM26E1dFFjINuFa84pG4XmlNgrNEYGLFK0CaiCwCLgN+yGWedsACEZmHO9p+XFU343aMr4nIQly1UMNIVqiqc3FtBzNxbQbPqeo8oCkw06uiuRO4J5e3jwMWhhqLc/gENzDQZ+qGXwSXuJYCc8UNWv4MBymxe7EsxA3M8iBwn7ft4e/7EkgNNRbjSg5FvdiWeI9NgrPTR40xJsFZicAYYxKcJQJjjElwlgiMMSbBWSIwxpgEZ4nAGGMSnCUCY4xJcJYIjDEmwf0/eg7odF3AaesAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.9481182499999999\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAm2UlEQVR4nO3de7xd07n/8c83Ozv3m1woIhdpcpIgEnZdW1HHJXU0btUmLa0epS6hv1b1clCOXihHqz2ibShaijrqkhJU0WqVEiUhCRVBJKgIQSSRZOf5/THmkm3bsVd29txrrb2+79drveZlzTXHM3NZzxpjzDmGIgIzM6teHUodgJmZlZYTgZlZlXMiMDOrck4EZmZVzonAzKzKORGYmVU5JwKrGJI+J+kPRRz3c0lntkVMbUHSc5L2zdbPlnR1qWOy9sWJwFpF9mW1UtJySf+SdKWkHq1ZRkT8JiL2L+K44yPiu61ZdoGkkPR2dp2LJf1IUk0eZbWEpF6SLpK0MIvxmWy7f6ljs/LlRGCt6ZMR0QPYCagDzmh8gKSObR5V69sxu87xwGeA/yxxPABI6gTcDWwHTAB6AbsDS4FdWnC+9vB3ZUVwIrBWFxGLgduB7eHdX9EnSXoaeDrbd5CkxyQtk/Q3SWMKn5e0jaQbJS2RtFTSxdn+oyX9NVuXpB9LekXSm5Iel1Qo70pJ32twvmMlzZf0mqTpkrZq8F5IOl7S01ksUyWpyOucD9wPjG1wvpZc1zBJ92T7XpX0G0l9NvKPHeDzwCDg0IiYGxHrIuKViPhuRMxocL0fbhDTu39WkvaWtEjSNyW9DFwhaZ6kgxoc3zGLf6dse7fsOpdJmiVp7xbEbSXmRGCtTtI2wIHAow12HwLsCoyWNA64HPgy0A/4BTBdUuesmeVW4HlgCLA1cF0TxewP7AWMAHoDnyb98m0cyz7Audn7W2bnbXy+g4CPAGOy4w4o8jpHAh8D5mfbLb0uZTFuBYwCtgHOLiaGRvYF7oiI5S34bMGHgL7AYOA44FpgcoP3DwBejYh/SNoauA34XvaZrwO/kzRgE8q3EnAisNZ0s6RlwF+BPwM/aPDeuRHxWkSsJH3B/CIi/h4R9RHxK+AdYDdSE8ZWwGkR8XZErIqIvzZR1hqgJzASUETMi4iXmjjuc8DlEfGPiHgH+Dawu6QhDY45LyKWRcRC4F4a/MLfgH9IehuYB/wJuCTb36Lrioj5EXFXRLwTEUuAH5GanTZWP6CpP4ONsQ44K4tlJXANMFFSt+z9z5KSA8CRwIyImJHVPu4CZpJ+BFgFcSKw1nRIRPSJiMERcWL2RVLwQoP1wcCpWXPCsix5bEP6otwGeD4i1n5QQRFxD3AxMBV4RdI0Sb2aOHQr0q/wwueWk2oOWzc45uUG6yuAHgCS5mQdrsslfazBMTtlx3yGVMvpvinXJWkLSddlnc9vAlcDLencXUqq9WyKJRGxqrCRNX/NAz6ZJYOJpOQA6XqPaHS9H22FGKyNORFYW2k4zO0LwPezpFF4dYuIa7P3BhXTURkRP42InYHRpCai05o47EXSFxYAkrqTfjkvLuL820VEj+z1l0bvRURcDzwAfGcTr+sHpD+fHSKiF+mXdlH9FI38ETggu8YNWQF0a7D9oUbvNzUccaF56GBgbpYcIF3TVY2ut3tEnNeC2K2EnAisFC4Fjpe0a9bp213Sf0jqCTxEat44L9vfRdKejU8g6SPZ52uBt4FVpGaNxq4FvihprKTOpC/dv0fEc610LecBx0r60CZcV09gOfBG1u7eVEIrxlWkL+ffSRopqYOkfpL+S1KhueYx4LOSaiRNoLgmqOtIfTInsL42AKnm8klJB2Tn65J1OA9sYfxWIk4E1uYiYiZwLKlp53VSZ+vR2Xv1wCeBDwMLgUWkJpjGepG+eF8nNf0sBS5ooqw/AmcCvyN9EQ8DJrXitTwO3Edq+2/pdf03qbnpDVLn640tjOUdUofxk8BdwJukBNQf+Ht22FeyOJaR+k9uLuK8L5FqPnsAv22w/wVSLeG/gCWkJHQa/l6pOPLENGZm1c2Z28ysyjkRmJlVOScCM7Mq50RgZlblKm5Qqf79+8eQIUNKHYaZWUV55JFHXo2IJof/qLhEMGTIEGbOnFnqMMzMKoqk5zf0npuGzMyqnBOBmVmVcyIwM6tyFddHYGbWUmvWrGHRokWsWrWq+YMrVJcuXRg4cCC1tbVFf8aJwMyqxqJFi+jZsydDhgyhyInoKkpEsHTpUhYtWsTQoUOL/lxuTUOSLleaRvCJDbwvST9VmkJwdmHqOzOzvKxatYp+/fq1yyQAIIl+/fptdI0nzz6CK0kTaG/IJ4Dh2es44Gc5xmJmBtBuk0BBS64vt6ahiLiv0XSAjR0M/DrS8KcPSuojacsNTDe46f76V/jDHzb9PD16wMknQ9euGz6mvh7WrIHVq9cvI2DLLaGd/yM0s8pTyj6CrXnv9IWLsn3vSwSSjiPVGhg0aFDLSnvgAfje91r22YLCkN3f/CZsu+37v+zXrEmvdU3Nj5LZbTd46SU4/vj0mXfeWf9atgw+8hHo3z9tr14Ny5fDuHHQvfv641evhlWrYOhQ6N0btt4aOnfetGszszZRU1PDDjvswNq1axk6dChXXXUVffr0AWDOnDmcfPLJLF68mHXr1vH5z3+eM844491f+bfffjtnnnkmK1asoHPnzuyzzz5ceOGFmxxTRXQWR8Q0YBpAXV1dyyZQOO209NoUb74JJ5yQvuw7d4baWujUKS0brje17+KLoV8/uO++dK5vfzsta2rSudatS1/uV1/dsti23RZuuSXVOFasgJ494d/+zTUQszLTtWtXHnvsMQC+8IUvMHXqVE4//XRWrlzJxIkT+dnPfsb+++/PihUrOPzww7nkkks46aSTeOKJJ5gyZQq33XYbI0eOpL6+nmnTprVKTKVMBItJE3oXDKSIeWRLqlcv+M1vWvbZ449fv/7WW+sTQE3N+v0LF6ZaQefOKYF07gxPPAFvv53WC69OneDJJ1Pt4E9/gmuugQULYIcd3l9u9+7p81ttBYMHp/VjjoFTTmnZdZhZq9l9992ZPXs2ANdccw177rkn+++/PwDdunXj4osvZu+99+akk07i/PPP5/TTT2fkyJFAqlmccMIJrRJHKRPBdGCKpOuAXYE3cusfKDc9eza9f9Cg9Gpoq62aPnaXXdLy2GPhssvgqqtSgujeHV57DW66KX12wQJYuhSefhpefDF95itfSa8BA1IiGjIkJYnttoNnn4Vdd03rH/1oq1yuWVn6f/8Psl/mrWbsWLjooqIOra+v5+677+aYY44BUrPQzjvv/J5jhg0bxvLly3nzzTd54oknOPXUU1s33kxuiUDStcDeQH9Ji4CzgFqAiPg5MAM4kDSv6wrgi3nF0u517QrHHffefV/+ctPHzpkDhx+e+h5eeik1Sb38Mjz44Ppjrrhi/fqwYTBiREoKy5alpPLqq7DFFnDooanZ64031r8WLkzHH3+8m6XMmrBy5UrGjh3L4sWLGTVqFPvtt1+pQ8r1rqHJzbwfwEl5lW8bsN12qVmpoQh4/fV0R9STT6bmqMsug3vvhWeeSa/bb4cuXVI/RsGll264nBNPTE1p22yTks9ee8Hzz6fmq/794YUXYO3a1PH9v/8LjX4JmeWuyF/ura3QR7BixQoOOOAApk6dyimnnMLo0aO5r9CHmFmwYAE9evSgV69ebLfddjzyyCPsuOOOrR5TxU1eX1dXFx6Guo299hp065YSQUSqRdx6a2pW6t37va/Fi1OV+5574GMfS4nlX/9af65OnVKCePXVpss68kj45CdTh3yHDqmDfcCAtF1IQj16wMqV6TwdOqSaR48eqc+kT590vFkT5s2bx6hRo0oaQ48ePVi+fDkAjz76KIcccgjPPPMMa9asYbvttmPatGnsu+++rFy5kiOOOIIDDjiAk08+mdmzZ3PYYYcxY8YMRowYwbp165g2bRrHN+x/zDR1nZIeiYi6pmKqiLuGrMT69l2/LqUEcPDBTR/bpw/cdVfx577tNvjqV1MfBqS7plp651TBllumONatgyOOSLffDhqU+mbeeSclrNpa+NCHUiIxK5Fx48YxZswYrr32Wo466ihuueUWTj75ZE466STq6+s56qijmDJlCgBjxozhoosuYvLkyaxYsQJJHHTQQa0Sh2sEVj7eegvmz0+1hjfeSLWAp55KtYKuXVMtomvXdOfUokXpC33NmlTj6N8/9WHMnJm+7G+6qbgyt9gC6upSE9YWW8CSJSlJdO2absPt3j31j2y5ZepU32yzPP8ELGflUCNoC64RWOXq2TP9em9o/PiWn2/NGpg9O9U2Xn011WwWLkzL2bPh5ptTU9Ztt73/s7W16fONdeqU+jVGjEjPaYwYAZ/9bLrrqm9fd5BbRXIisPartjZ1Qm+oI/rii9Ny7dr1DwnW16cv844d0zMXDz2Uah8LF6aE8vzzqa/kjjvgn/9Mny882dmzZ7qTauedYfhw+MQn8r9Gs1bgRGDWsWN6wXv7DLp3h49/fMOfq6+H3/0O5s5NNYybboJf/zq9ICWUujrYfvs0DMigQXDYYampy0omItr1wHMtae53IjBrqZoa+PSn12+vW5eaoB56CL71rXTb7MMPp1fBccelZHPwwWncqeHDU2IYNiwlC8tVly5dWLp0absdirowH0GXLl026nPuLDbL29tvp+cmpk2DRx5ZP97UhvTrlzq/v/zl1My0446pdtLRv9s2VTXPUPZBncVOBGZtLSL1NSxYkDqfZ81Kt8x27pyajx58cP3ttA317p0ezOvRIw0DMmlSutPJrAhOBGaV5sUXU4K48cbUOf344/DnP7//uPHj04i4hx6akorZBjgRmLUXa9em5yYuuijdrdTw/++wYTBmTGpWOvdcd0rbe3xQIvBjlWaVpGPH1Kl8wQWpc3r+fNh99/TeihXpzqVLL03JQIKBA9NYTs8+W9q4raw5EZhVsmHD4G9/SzWDF19MT2Sfey6MHJkecFu8OM09se22KTEMHZrm1PigWfSs6jgRmLUnvXqlW1fnzUtDhi9dCtdfn56CBnjuuTSwX01NuiPpxz9OHddW1ZwIzNqzvn3TwHtPPplqDY8+CtkgZvzjH/C1r6UxlLbYAj73uTROk1UdJwKzajJ2bOoziEi1g6lT06B6r7ySpjzt0gVGj4bPfAZ+//s0kJ+1e75ryMxSYjjvvPTl/8AD73+/b980s92xx6aBAf1wW8Xx7aNmtnGeew6uvDLNVlcYwbWhTp1Sp/OnPlWK6KwFfPuomW2cIUPg7LPhhhvSKKvr1sHdd68fW2n16tT30KMHnHNOGkbDKpZrBGa28ZYsgT33fG9NoW/f9Jo8OTUjjRnj+RnKiJuGzCwf69bBGWekL/z77oO//nX9e9tvDwcdBIcckobjrqkpWZjmRGBmbaW+HqZPTyOt3nHHe98bODDdmfSxj5UmtirnPgIzaxs1NWkAvNtvh+XLYcaM9IBb375ppre99krzMVx5ZXrfyoITgZnlo3v3NF3nueemJ5yffTbti4AvfjFN7bnLLvDb35Y60qrnRGBmbWPIkFQLeO01+J//SQ+yPfxwmlehpibdhXTLLaWOsio5EZhZ29psMzj1VPjLX9IkPN26pU7nG25IHcsSfPazHjG1DTkRmFnp7LpregYhItUW9t8/7b/22jRi6lZbwQ9+8N55F6zVORGYWXno3h3uvDPVDu6/P91y+tJLcPrpqYP5+98vdYTtlhOBmZUXCfbYI/UfrFmTnkeA9LzCuHFpGO2m5nS2FnMiMLPy1bFjmq/5qaegTx947LE0xtGIEWkk1W9/O03GY5vEicDMyt+IEfD666mv4Ior0r5Zs9KIqX36pPkWrMVyTQSSJkh6StJ8Sd9q4v3Bku6WNFvSnyQNzDMeM2sHjj46JYQ1a+BLX0r7Ro1aP8+CbbTcEoGkGmAq8AlgNDBZ0uhGh/0P8OuIGAOcA5ybVzxm1s507AiXXgo33pi2TzkldSp/4xuwdm1pY6swedYIdgHmR8SCiFgNXAcc3OiY0cA92fq9TbxvZvbBDj003YJ6wAFp+4ILoLY23Zp6552lja1C5JkItgZeaLC9KNvX0CzgsGz9UKCnpH6NTyTpOEkzJc1csmRJLsGaWQXr1i0Ncrd6NXzlK2mehIceggkTYMCANPOabVCpO4u/DoyX9CgwHlgM1Dc+KCKmRURdRNQNGDCgrWM0s0pRWwsXXQRvvZXuMKqpgVdfhYkTYdgwuOkm9yM0Ic9EsBjYpsH2wGzfuyLixYg4LCLGAadn+5blGJOZVYsdd0x9Bc88k7YXLIDDDkv9CLNmlTa2MpNnIngYGC5pqKROwCRgesMDJPWXVIjh28DlOcZjZtVo223T08ozZ6ZaAaRnEAYOTNNsLlxY0vDKQW6JICLWAlOAO4F5wPURMUfSOZImZoftDTwl6Z/AFoCfITez1ifBzjunJ5KnToUuXWDxYjjrLBg8OD2tXMU8Q5mZVad169JMaieckLYHDoQ5c6BXr9LGlRPPUGZm1liHDnD88bBqVdpetAh6967KiXKcCMysunXunGoHU6em7UmT0q2nVcSJwMxMghNPhGuuSdu77ppuNa0STgRmZgWTJ69/+Oyww+BHPyptPG3EicDMrKGDDoIHHkjrp54KF15Y2njagBOBmVlju+2W5kEA+PrX4TvfSf0I7ZQTgZlZU7bfHubNS+vf/S5st127HZ7CicDMbENGjoT6ehg/Pk1+8633TavSLjgRmJl9kA4d4JZb0vr558MXvlDaeHLgRGBm1pzevWHu3PTMwa9/vf6Zg3bCicDMrBijRqWnjwGmTFlfS2gHnAjMzIrVvz/cfntaP+QQePbZkobTWpwIzMw2xoQJcNppaX3ffdvFnUROBGZmG+v889OQFAsWwLnnljqaTeZEYGbWEoUEcPrpMH9+aWPZRE4EZmYt0asX3HZbWh8+PE2LWaGcCMzMWurAA+HQQ9P65puXNpZN4ERgZrYpbrwxPXn8+utwww2ljqZFnAjMzDbVZZel5cknV+RdRE4EZmab6sMfhqOOgpdfhs99rtTRbDQnAjOz1nDFFWl57bUwa1ZpY9lITgRmZq2hpgbuvz+tjxtX2lg2khOBmVlr2WMP2Guv1E9w772ljqZoTgRmZq3pJz9Jy332gTlzShtLkZwIzMxa09ix6zuMt98eXnqppOEUw4nAzKy1XX01nHJKWh8/vrSxFMGJwMwsDz/5CeyyCzz9dNnfReREYGaWl0suSctDDilpGM1xIjAzy8vOO8Nuu8Fzz8Ef/lDqaDbIicDMLE+FGc0OOKC0cXwAJwIzszz16QP77ZfWv/GNkoayIbkmAkkTJD0lab6kbzXx/iBJ90p6VNJsSQfmGY+ZWUnceisMHgwXXFCWg9Lllggk1QBTgU8Ao4HJkkY3OuwM4PqIGAdMAi7JKx4zs5Lp1AlOOCGtX399aWNpQp41gl2A+RGxICJWA9cBBzc6JoBe2Xpv4MUc4zEzK52jj07LwpPHZaSoRCBpT0l3SfqnpAWSnpW0oJmPbQ280GB7UbavobOBIyUtAmYAJxcZt5lZZdliC5g0CR54AObNK3U071FsjeCXwI+AjwIfAeqy5aaaDFwZEQOBA4GrJL0vJknHSZopaeaSJUtaoVgzsxK44AKorYWDGzeOlFaxieCNiLg9Il6JiKWFVzOfWQxs02B7YLavoWOA6wEi4gGgC9C/8YkiYlpE1EVE3YABA4oM2cyszAwcmB4ue/rpVDMoE8UmgnslXSBpd0k7FV7NfOZhYLikoZI6kTqDpzc6ZiHw7wCSRpESgX/ym1n7df75712WgY5FHrdrtqxrsC+AfTb0gYhYK2kKcCdQA1weEXMknQPMjIjpwKnApZK+mp3v6IgyvLfKzKy1DBkCEybAzTeXOpJ3FZUIIuLjLTl5RMwgdQI33PedButzgT1bcm4zs4o1fjzccQf88IfwzW+WOpqi7xrqLelHhQ5bSRdK6p13cGZm7dJXv5qWv/pVaePIFNtHcDnwFvDp7PUmcEVeQZmZtWudO6dawbx5sGJFqaMpOhEMi4izsofDFkTEfwPb5hmYmVm7tk/Wxfrgg6WNg+ITwUpJHy1sSNoTWJlPSGZmVeCgg9Ly6qtLGwfF3zV0AvCrrF9AwGvA0XkFZWbW7o0bl5Z//GNp46D4u4YeA3aU1CvbfjPPoMzM2j0JdtgBHn8cXn4ZPvShkoXygU1Dko7Mll+T9DXgS8CXGmybmVlLfSe7m77EzUPN9RF0z5Y9N/AyM7OWKow5dNppJZ2n4AObhiLiF9nyv9smHDOzKlJbm54p+PGP05PGhx5akjCKfaDsfEm9JNVKulvSkkKzkZmZbYKzzkrLEk5jWezto/tnHcQHAc8BHwZOyysoM7Oq0bs39OwJ//oXrFtXkhCKTQSFJqT/AP4vIt7IKR4zs+pz5pnw1lswf35Jii82Edwq6UlgZ+BuSQOAVfmFZWZWRfbYIy2///2SFK9iR32W1Jc0QU29pG5Ar4h4OdfomlBXVxczZ85s62LNzPJTXw8ds4aXnO4ekvRIRNQ19d4H3jUkaZ+IuEfSYQ32NTzkxtYJ0cysitXUpESwdm1Jim/uyeLxwD3AJ5t4L3AiMDNrHRMmwK23lqTo5p4jOCtbfrFtwjEzq1K77JISwWuvQd++bVp0sc8R/EBSnwbbm0n6Xm5RmZlVm97ZXF8l6AMt9q6hT0TEssJGRLwOHJhLRGZm1eij2Uj/t9/e5kUXmwhqJHUubEjqCnT+gOPNzGxjbL99Wi5e3OZFFzsfwW9Izw8Upqf8IlAek22ambUHnTql5fPPt3nRxc5H8ENJs4B9s13fjYg78wvLzKwKde4M//hHmxdbbI0AYB6wNiL+KKmbpJ4R8VZegZmZVZ2dd4a//a3Niy32rqFjgRuAX2S7tgZuzikmM7PqNGJEWtbXt2mxxXYWnwTsCbwJEBFPA5vnFZSZWVUaNSot585t02KLTQTvRMTqwoakjqQni83MrLV85CNpecstbVpssYngz5L+C+gqaT/g/4Df5xeWmVkV2nXXtLz55jYttthE8E1gCfA48GVgBnBGXkGZmVWlbt3SskuXNi222buGJNUAcyJiJHBp/iGZmVWxffaBe+5Jw1G/d7Tn3DRbI4iIeuApSYPaIB4zs+q2005peWfbPapVbNPQZsCcbOL66YVXnoGZmVWlE09My8Kk9m2g2AfKzsw1CjMzS4YOTcva2jYrsrkZyroAxwMfJnUU/zIiip5CR9IE4CdADXBZRJzX6P0fAx/PNrsBm0dEn6KjNzNrj8aPh3nz2qy45pqGfgXUkZLAJ4ALiz1x1sk8NfvcaGCypNENj4mIr0bE2IgYC/wvnvHMzCwNNfHKK+nVBppLBKMj4siI+AXwKeBjG3HuXYD5EbEgexjtOuDgDzh+MnDtRpzfzKx92m23tHzooTYprrlEsKawsjFNQpmtgRcabC/K9r2PpMHAUNL8yE29f5ykmZJmLlmyZCPDMDOrMLvvnpZtNBJpc4lgR0lvZq+3gDGFdUlvtmIck4AbsltV3ycipkVEXUTUDRgwoBWLNTMrQ4XvuUWL2qS4D0wEEVETEb2yV8+I6NhgvVcz514MbNNge2C2rymTcLOQmVlSmKTmppvapLhinyNoiYeB4ZKGSupE+rJ/37MHkkaSnlN4IMdYzMwqhwTjxsGrr8LajW2V33i5JYKsT2EKcCdpUpvrI2KOpHMkTWxw6CTguojwaKZmZgXjx6flE0/kXtTGzFC20SJiBmmAuob7vtNo++w8YzAzq0iHHw4XXQT/+lfuReXZNGRmZi3Vs2daPvlk7kU5EZiZlaP+/dOya9fci3IiMDMrR927p+WsWbkX5URgZlaOevdOy802y70oJwIzs3IkQYcO8Le/5V6UE4GZWblatw6eey73YpwIzMzK1eabwwsvNH/cJnIiMDMrV4MGQX2TQ7C1KicCM7NyNXJkmsQ+Z04EZmblauts5P533sm1GCcCM7Ny1aVLWr78cq7FOBGYmZWr4cPTMucOYycCM7NyVWgaWrgw12KcCMzMylVhvKGc7xxyIjAzK1f9+qVlziOQOhGYmZWrQo2gpibXYpwIzMzKVW1tmr94zZpci3EiMDMrZ6tXw7x5uRbhRGBmVu4efzzX0zsRmJmVs27doE+fXItwIjAzK2d77AErV+ZahBOBmVk5W7vWfQRmZlUt52YhcCIwMytvHfL/mnYiMDMrZ2PGpOW6dbkV4URgZlbOOnVKyxwfKnMiMDMrZ4VEsGpVbkU4EZiZlbNly9LyrbdyK8KJwMysnG27bVquXZtbEU4EZmblrLY2Ld1HYGZWpSo9EUiaIOkpSfMlfWsDx3xa0lxJcyRdk2c8ZmYVpzAXwfPP51ZEx7xOLKkGmArsBywCHpY0PSLmNjhmOPBtYM+IeF3S5nnFY2ZWkQrzFlfoXUO7APMjYkFErAauAw5udMyxwNSIeB0gIl7JMR4zs8pTmK7yqadyKyLPRLA18EKD7UXZvoZGACMk3S/pQUkTmjqRpOMkzZQ0c8mSJTmFa2ZWhnr1SsuePXMrotSdxR2B4cDewGTgUkl9Gh8UEdMioi4i6gYMGNC2EZqZlVL37mlZoU1Di4FtGmwPzPY1tAiYHhFrIuJZ4J+kxGBmZgBduqTlY4/lVkSeieBhYLikoZI6AZOA6Y2OuZlUG0BSf1JT0YIcYzIzqyyF20dzHI46t0QQEWuBKcCdwDzg+oiYI+kcSROzw+4ElkqaC9wLnBYRS/OKycysIg0bBkvz+2rM7fZRgIiYAcxotO87DdYD+Fr2MjOzpnTpAqtX53b6UncWm5lZc2prK/fJYjMzawWdOlXsXUNmZtYaIuDZZ3M7vROBmVm522wz6Nw5t9M7EZiZlbu+feGdd3I7vROBmVm569QJnn46t9M7EZiZlbslS0DK7fROBGZm5W7cuNRhHJHL6Z0IzMzKXWEC+7ffzuX0TgRmZuVuzJi0fOONXE7vRGBmVu4Kw0vkNB+LE4GZWbkbPDgt6+tzOb0TgZlZuSs8TJbTswROBGZm5a5wt9BLL+VyeicCM7Nyt9lmadm1ay6ndyIwMyt3HbKvaj9HYGZWpQpPFa9bl8vpnQjMzMpdIRG4RmBmVqXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuXcNGRmVuWcCMzMqlyhaej++/M5fS5nzUiaIOkpSfMlfauJ94+WtETSY9nrS3nGY2ZWkQYPhtNOgyOOyOX0HXM5KyCpBpgK7AcsAh6WND0i5jY69LcRMSWvOMzMKl6HDnD++fmdPrczwy7A/IhYEBGrgeuAg3Msz8zMWiDPRLA18EKD7UXZvsYOlzRb0g2StmnqRJKOkzRT0swlS5bkEauZWdUqdWfx74EhETEGuAv4VVMHRcS0iKiLiLoBAwa0aYBmZu1dnolgMdDwF/7AbN+7ImJpRLyTbV4G7JxjPGZm1oQ8E8HDwHBJQyV1AiYB0xseIGnLBpsTgXk5xmNmZk3I7a6hiFgraQpwJ1ADXB4RcySdA8yMiOnAKZImAmuB14Cj84rHzMyapshp7Iq81NXVxcyZM0sdhplZRZH0SETUNfVeqTuLzcysxCquRiBpCfB8Cz/eH3i1FcOpBL7m6uBrrg6bcs2DI6LJ2y4rLhFsCkkzN1Q1aq98zdXB11wd8rpmNw2ZmVU5JwIzsypXbYlgWqkDKAFfc3XwNVeHXK65qvoIzMzs/aqtRmBmZo04EZiZVbl2mQiKmBmts6TfZu//XdKQEoTZqoq45q9JmpsN+X23pMGliLM1NXfNDY47XFJIqvhbDYu5Zkmfzv6u50i6pq1jbG1F/NseJOleSY9m/74PLEWcrUXS5ZJekfTEBt6XpJ9mfx6zJe20yYVGRLt6kcY1egbYFugEzAJGNzrmRODn2fok0ixpJY8952v+ONAtWz+hGq45O64ncB/wIFBX6rjb4O95OPAosFm2vXmp426Da54GnJCtjwaeK3Xcm3jNewE7AU9s4P0DgdsBAbsBf9/UMttjjaCYmdEOZv3cBzcA/y5JbRhja2v2miPi3ohYkW0+SBoWvJIVOwPed4EfAqvaMricFHPNxwJTI+J1gIh4pY1jbG3FXHMAvbL13sCLbRhfq4uI+0iDcG7IwcCvI3kQ6NNoJOeN1h4TQTEzo717TESsBd4A+rVJdPkodja4gmNIvygqWbPXnFWZt4mI29oysBwV8/c8Ahgh6X5JD0qa0GbR5aOYaz4bOFLSImAGcHLbhFYyG/v/vVm5DUNt5UnSkUAdML7UseRJUgfgR1Tf0OYdSc1De5NqffdJ2iEilpUyqJxNBq6MiAsl7Q5cJWn7iFhX6sAqRXusETQ7M1rDYyR1JFUnl7ZJdPko5pqRtC9wOjAx1s8MV6mau+aewPbAnyQ9R2pLnV7hHcbF/D0vAqZHxJqIeBb4JykxVKpirvkY4HqAiHgA6EIanK29Kur/+8Zoj4mg2ZnRsu0vZOufAu6JrBemQhUzG9w44BekJFDp7cbQzDVHxBsR0T8ihkTEEFK/yMSIqOTJLIr5t30zqTaApP6kpqIFbRhjayvmmhcC/w4gaRQpESxp0yjb1nTg89ndQ7sBb0TES5tywnbXNBTFzYz2S1L1cT6pU2ZS6SLedEVe8wVAD+D/sn7xhRExsWRBb6Iir7ldKfKa7wT2lzQXqAdOi4iKre0Wec2nApdK+iqp4/joSv5hJ+laUjLvn/V7nAXUAkTEz0n9IAcC84EVwBc3ucwK/vMyM7NW0B6bhszMbCM4EZiZVTknAjOzKudEYGZW5ZwIzMyqnBOBWRMk1Ut6TNITkn4vqU8rn/+57D5/JC1vzXObbSwnArOmrYyIsRGxPelZk5NKHZBZXpwIzJr3ANmgXpKGSbpD0iOS/iJpZLZ/C0k3SZqVvfbI9t+cHTtH0nElvAazDWp3TxabtSZJNaThC36Z7ZoGHB8RT0vaFbgE2Af4KfDniDg0+0yP7Pj/jIjXJHUFHpb0u0p+0tfaJycCs6Z1lfQYqSYwD7hLUg9gD9YP0wHQOVvuA3weICLqSUObA5wi6dBsfRvSAHBOBFZWnAjMmrYyIsZK6kYa5+Yk4EpgWUSMLeYEkvYG9gV2j4gVkv5EGhDNrKy4j8DsA2Szup1CGthsBfCspCPg3bljd8wOvZs0BSiSaiT1Jg1v/nqWBEaShsI2KztOBGbNiIhHgdmkCVA+BxwjaRYwh/XTJn4F+Likx4FHSHPn3gF0lDQPOI80FLZZ2fHoo2ZmVc41AjOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwM6tyTgRmZlXOicDMrMr9f0nsvVKGIn4xAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.9460696474624111\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.2297\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by SML (Standardized Max Logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "  0%|          | 0/98 [00:00<?, ?it/s]Exception ignored in: <function _releaseLock at 0x7fd46bb75820>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.9/logging/__init__.py\", line 227, in _releaseLock\n",
      "    def _releaseLock():\n",
      "KeyboardInterrupt: \n",
      "100%|██████████| 98/98 [00:25<00:00,  3.84it/s]\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.sml import SML\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "sml = SML(config, model, id_train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:05<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:05<00:00, 17.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 0.17674805338695232 id variance: 1.4942061282549082\n",
      "ood mean score: 3.8848704739479807 ood variance: 3.5674267128303514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=sml,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/fUlEQVR4nO3dd5gUVdbA4d9hGHJOiuQMAyLiiAERkIyYE2Z2URYR0DV9KorKuq4JxQAoKmJAdMVVR0HRVRFXRQmSgyASBYEhM4QJ5/vj1kgzTmhgqmu6+7zP0091V1VXnarpuafqVtW9oqoYY4yJX8WCDsAYY0ywLBEYY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEEONEZLGIdAo6jqJCRO4VkZcDWvcEEXk4iHUXNhG5WkQ+O8rv2m+yiLFEEEEislpE9onIHhHZ5BUM5fxcp6q2VNXpfq4jm4iUFJF/ichabztXiMidIiKRWH8u8XQSkfWh41T1EVW9waf1iYgMFZFFIrJXRNaLyLsicqIf6ztaIvKgiLx5LMtQ1Ymq2j2Mdf0p+R3tb1JESnixr/D272oRGS8i9Y90WeZwlggi7zxVLQe0AU4G7gk2nCMnIsXzmPQu0AXoDZQHrgUGAM/4EIOISFH7/T4D3AIMBaoATYEPgHMLe0X5/A18F+C6JwPnA1cBFYGTgDm439wRCXL/FUmqaq8IvYDVQNeQz48DU0I+nw58B+wA5gOdQqZVAV4FfgO2Ax+ETOsDzPO+9x3QOuc6gROAfUCVkGknA1uBRO/zX4Gl3vKnAfVC5lXgZmAF8Gsu29YF2A/UyTH+NCATaOx9ng78C/gR2AV8mCOm/PbBdOCfwLfetjQG/uLFvBtYBfzNm7esN08WsMd7nQA8CLzpzVPf267rgbXevhgWsr7SwGve/lgK3AWsz+Nv28Tbznb5/P0nAKOBKV68PwCNQqY/A6zz9sscoEPItAdxBeGb3vQbgHbA996+2gg8D5QI+U5L4HNgG/A7cC/QEzgIpHv7ZL43b0XgFW85G4CHgQRvWj9vnz8NpHrT+gH/86aLN22zF9tCoBXuICDdW98e4KOc/wdAghfXL94+mUOO35A3X1fv7/mnafn8f+X2t+7v/a1nAJ8Ag3MsYz5wsfe+ecj+Ww5cHnQZ4lvZFHQA8fTK8Q9Q2/uHecb7XMv7J+uNO1Pr5n2u7k2fArwDVAYSgY7e+JO9f8DTvH+q6731lMxlnV8CN4bE8wTwgvf+AmAl0AIoDtwHfBcyr3r/FFWA0rls26PA13ls9xoOFdDTvYKmFa6wfi/kn7WgfTDd+ydu6cWYiDvaboQrjDoCaUBbb/5O5Ci48ygcXsIV+icBB4AWodvk7fPawIKcywtZ7kBgTQF//wne9rTz4p8IvB0y/RqgqjftdmATUCok7nTgQm/flAZOwSXO4t62LAVu9eYvjyvUbwdKeZ9Py7kPQtb9PvCi9zepgUvU2X+zfkAGMMRbV2kOTwQ9cAV4Je/v0AKoGbLND+fzf3An7v+gmffdk4CqR/L7ym25+fytX/e2sTRwHfBtyPxJuKRa0ptnHe5AoziHDpqSgi5H/HgVtVPrePCBiOzG/cg2Aw94468BpqrqVFXNUtXPgdlAbxGpCfQCBqrqdlVNV9Wvve8NAF5U1R9UNVNVX8MVZqfnsu63gCvBVa0Afb1x4Aqyf6nqUlXNAB4B2ohIvZDv/0tVt6nqvlyWXQ1X8ORmozc92xuqukhV9wL3A5eLSEJ++yDkuxNUdbGqZnj7YYqq/qLO18BnQIc84sjLQ6q6T1Xn444IT/LGXw484u3z9cCz+Syjaj7bH+p9Vf3R28cTcVWEAKjqm6qa6m3bSFyB1Czku9+r6gfevtmnqnNUdaY3/2pcQd7Rm7cPsElVR6rqflXdrao/5BaQiByH28e3qupeVd2MO8LvGzLbb6r6nLeunH//dFyiaQ6I9xsKZ1+AO7O5T1WXe3/D+aqamst84e7fgjzobeM+XPIL/Y1fDfxHVQ/g9t9qVX3V2+afcActlxVCDEWOJYLIu1BVy+OOVptzqICsB1wmIjuyX8BZQE2gDrBNVbfnsrx6wO05vlcHVw2S03vAGV5iORtXbfJNyHKeCVnGNtwRWq2Q76/LZ7u2erHmpqY3PbflrMEd2Vcj/32Qawwi0ktEZorINm/+3hyedMKxKeR9GpB9Af+EHOvLb/tTyXv7w1kXInKHiCwVkZ3etlTk8G3Jue1NReRj78aDXbjknT1/HVx1Szjq4f4GG0P2+4u4M4Nc1x1KVb/EVUuNBjaLyDgRqRDmusONM9z9W5A/tkNVd+POtLMT3pW45Axun5yW47d4NXB8IcRQ5FgiCIh39DoBeNIbtQ53pFwp5FVWVR/1plURkUq5LGod8M8c3yujqpNyWed23BHzFbgLbm+rqoYs5285llNaVb8LXUQ+m/Rf3D9OndCRInIa7p/9y5DRofPUxR1Rbi1gH/wpBhEpiUtuTwLHqWolYCougRUUbzg24qqEcos7py+A2iKSfDQrEpEOuGsQlwOVvW3ZyaFtgT9vz1hgGdBEVSvg6tqz518HNMxjdTmXsw53FlktZL9XUNWW+Xzn8AWqPquqp+CqV5riqnwK/J637kYFzAPu99VORGrnM89eoEzI59wK7ZzxTAKuFJEzcFVoX4XE9XWO32I5Vb0pjFijjiWCYI0CuonISbiLgOeJSA8RSRCRUt7tj7W90+xPgDEiUllEEkXkbG8ZLwEDReQ0706asiJyroiUz2Odb+HqRi/lULUQwAvAPSLSEkBEKopI2KfBqvpfXGH4noi09LbhdG+7xqrqipDZrxGRJBEpA4wAJqtqZn77II/VlsBVn2wBMkSkFxB6S+PvQFURqRjuduTwb9w+qSwitYDBec3obd8YYJIXcwkv/r4icncY6yqPq4ffAhQXkeFAQUfV5XEXZ/eISHMgtJD6GKgpIreKu623vJeUwe2X+tl3XXm/r8+AkSJSQUSKiUgjEelIGETkVO/3l4grjPfjzjaz15VXQgJ4GfiHiDTxfr+tRaRqzpm839fnwPsicoqIFPe2aaCI/NWbbR7Q1/v/SMb9xgsyFXf0PwJ4R1Wz4/4YaCoi13rLS/S2s0UYy4w6lggCpKpbcBevhqvqOtwF23txhcE63FFV9t/oWtyR8zLctYVbvWXMBm7EnZpvx13w7ZfPalNwd7hs8urEs2N5H3gMeNurZliEuy5xJC7BHVF9irtL5E3cnShDcsz3Bu5saBPuKGyoF0NB++Aw3qn9UFyBvR13lpMSMn0Z7ohvlXd6n1t1WX5GAOuBX3FHpJNxR855GcqhKpIduCqPi4CPwljXNNx++xlXXbaf/KuiAO7AbfNu3AHBO9kTvH3TDTgPt59XAJ29ye96w1QRmeu9vw6XWJfg9uVkwq+KqeCtf7sXeyruRgRwf/8kb/9/kMt3n8L9/T7DJbVXcBdyc3MpruB+B3e2tAhIxv1twF1vauTF8RCHH+jkyrse8B/cXUlvhYzfjTuo6Iu7U28T7v+jZEHLjEZyqGbAGP+JyHTcnRyBPN17LETkJqCvqoZ1pGxMtLAzAmPyICI1RaS9V1XSDHcr5vtBx2VMYbOn64zJWwnc3TMNcFU9b+OuAxgTU6xqyBhj4pxVDRljTJyLuqqhatWqaf369YMOwxhjosqcOXO2qmr13KZFXSKoX78+s2fPDjoMY4yJKiKyJq9pVjVkjDFxzhKBMcbEOUsExhgT56LuGkFu0tPTWb9+Pfv37w86FN+UKlWK2rVrk5iYGHQoxpgYExOJYP369ZQvX5769esjwXSP6ytVJTU1lfXr19OgQYOgwzHGxBjfqobEdSq9WUQW5TFdRORZEVkpIgtEpO3Rrmv//v1UrVo1JpMAgIhQtWrVmD7jMcYEx89rBBNw/aPmpReuFcwmuF62xh7LymI1CWSL9e0zxgTHt6ohVZ0hIvXzmeUC4HWvY5SZIlJJRGoeQRd3xhgDqpCZCenpcPAgpKVBVtahV2bm4cPs9wcPwoEDUKzY4fPn99q6FcqWdesMfWVl/XlcuOM3b4Zy5UDk0Pjs9Xnv92btZ4vupf5518Kppxb6LgzyGkEtDm9vfb037k+JQEQG4M4aqFu3bkSCO1IJCQmceOKJZGRk0KBBA9544w0qVaoEwOLFixkyZAgbNmwgKyuL6667jvvuu++Po/xPPvmE+++/n7S0NEqWLMk555zDyJEjA9waE5NUXWF54IB77dnjCsTsV0bG4Z937ICEBPed0EJ27173/TVrXKEYWriGFrqZmfD771C8OJQsmXeBnP1+5Uo4/ni3nvR02LnTFZKVKh2+zNDXrl1B71XffdkAbjwPKh6A2XNaUCzGEkHYVHUcMA4gOTm5SLaSV7p0aebNmwfA9ddfz+jRoxk2bBj79u3j/PPPZ+zYsXTv3p20tDQuueQSxowZw80338yiRYsYPHgwU6ZMoXnz5mRmZjJu3LhgN8b4Jy3NFcA7dhw6Ij140B1pFivm3h88CGvXQmrqoaPVjIxDBXVGBvzyC9SokXtBvm8frF8PlSu7gnT7dlcY797t1lfYEhJcnAkJh17Zn0XcttWpA4mJh08rVuzw9+XLw+rV0KQJlCgBtWpB/fpQujRUqXL48rNfqm6f1q3rvpOY6PZt7dq5ryP0fUKCS2rVqh2aVtAL3LBkSbdt2a9ixQ7/nNe4vMYXL+5iD5m+4+Au7vzyHl6e9wqNqzTm6fNeplh9f7rCCDIRbODwPmBre+Oi3hlnnMGCBQsAeOutt2jfvj3du7seFMuUKcPzzz9Pp06duPnmm3n88ccZNmwYzZs3B9yZxU03xWS3qEVfRgZs2wb797vCJC3NHfXu2uUK5e3bXWEq4sZnF0bZhXB2Qbxx46FCY+lSqFjRHeHu2+fmOVIlS7qCIiHBDYsXdzH8/rsrNLPHhxaQpUq5dbVu7dZ73HEucVSq5KaVLOm2r3p19z63ZWRluWnly7tCqkQJ97lChUPLKFXq0LaaQpOZlcmZL3dkeepy7jrzLh7s9CClE/PquO3YBZkIUoDBIvI2cBqws1CuD9x6K3hH5oWmTRsYNSqsWTMzM/niiy/o378/4KqFTjnllMPmadSoEXv27GHXrl0sWrSI22+/vXDjjWcZGfDbb67g3rPHDXfsOFT1sHu3K+g3b3bjV6929bO7drmj6HCUL+9eqanQqJErIEML6WrVYMsWV0ifdJJLIM2aHSpUK1Z0BWhWliugS5RwLxF3FF+ypCt4q1Vzha4VtHEjNS2VKqWrkFAsgX+e80/qVKxD8gnJvq/Xt0QgIpOATkA1EVkPPAAkAqjqC7i+R3vj+thNA/7iVyyRsG/fPtq0acOGDRto0aIF3bp1Czqk2LFliyuk9+93R8Gqrurk++9dgQ7uKHzFCndEHo46ddxRev36rirmlFOgZk1XADds6KojVF31RI0aroCuUsUV2MYUMlVl4sKJ3PLpLTza5VFuPOVGLmpxUcTW7+ddQ1cWMF2Bmwt9xWEeuRe27GsEaWlp9OjRg9GjRzN06FCSkpKYMWPGYfOuWrWKcuXKUaFCBVq2bMmcOXM46aSTAok7cOnp7sj6999dfe2vv8L8+fDNN+4I+eef3fTclC7tCuamTV2dcI0aULWqOwpv0cJ9v3Ztd/Rdterhdc3GFBHrdq5j4JSBTF0xldNrn077uu0jHkNUXCyOJmXKlOHZZ5/lwgsvZNCgQVx99dU88sgj/Pe//6Vr167s27ePoUOHctdddwFw5513cvHFF3PWWWfRtGlTsrKyGDduHAMHDgx4SwpBeror2BcvdtUjmzfDrFmuYE5JcdUoO3bk/t0mTdwRea9erlBv0uRQ3XRiojuir13bFfbGRKlJCyfxt4//RqZmMqrHKAa3G0xCscgfqFgi8MHJJ59M69atmTRpEtdeey0ffvghQ4YM4eabbyYzM5Nrr72WwYMHA9C6dWtGjRrFlVdeSVpaGiJCnz59At6CI3DgACxb5uraly2DTz91BX9WVu5H8qVLu2nNm7vkcNVVcMIJrt68cWN3xN66tSvwjYlxlUtX5rTapzGuzzgaVA6u+Zio67M4OTlZc3ZMs3TpUlq0aBFQRJET6Haqujr4d95x9fM//ujGLVx4+Hw1a7pqmHr1oF07V11Tq5Yr5GvXdhdLjYlTGVkZPP390xzMPMiws4cB7vpAJFoOEJE5qprrlWc7IzB/lpHh7ryaNQsWLXJ323z22eHztGnjLszecosr5Nu2dUf5VaoEEbExRd78TfPpn9KfORvncHnLy/9IAEWh+RhLBMZZsgSmTHFH/HPmHBpfurS79bF/f1cvf+mlrs7ebmk0JiwHMg7w8IyHefTbR6lSugrvXvYul7S4pEgkgGwxkwgidXoVlEKvwsvIgK++gpdfhn//+9D48uXhnHPgwguhRw93tG+FvjFHbcW2FTz27WNcdeJVPNX9KaqWqRp0SH8SE4mgVKlSpKamxmxT1Nn9EZQ61guoaWnwyivwySfuBe5WygsvdBdoL70UWrWyO3GMOUZ7Du7hw2UfcnXrq2lVoxXLBi+jYeWGQYeVp5hIBLVr12b9+vVs2bIl6FB8k91D2RFZuNDdxTNzpiv49+07NK1jR+jWDYYMcbdlGmMKxee/fM6AjwewZsca2tZsS4vqLYp0EoAYSQSJiYnWc1e2X3+FMWPg3Xddezjgbs/s2dMV+FdcAd2720NVxhSy7fu2c8dndzB+3niaVm3K1/2+pkX16LibMSYSQdw7cADefNPV9Wff3dO+PQwd6hJAUlKw8RkT4zKzMmk/vj0/p/7MPWfdw/COwylVPHqehbFEEM22boUXXoARI9xTvNWqweDBMHAgtGwZdHTGxLytaVv/aCTukS6PULdiXdrWPOpedwNjt4NEo9274Z57XBPC99/vHt4aO9a11/Pcc5YEjPGZqvL6/Ndp+lxTXp77MgAXNr8wKpMA2BlBdJk5Ex57DD74wH1OSnJnAxdfbHf6GBMha3as4W8f/41pv0zjzDpncna9s4MO6ZhZIijqsrLgww/hoYdcq5wAnTrB5Ze7KiBLAMZEzJsL3uSmKTehqjzX6zkGnTqIYhL9FSuWCIqyt96Cq692748/Hv7xD/fZ7pAyJhDVy1SnfZ32vNjnRepVqhd0OIXGEkFR9OOPcNtt8O23rsnlIUNcEihTJujIjIkr6ZnpjPx+JOmZ6dzf8X56NO5B90bdY+7BVUsERcmWLXDvva7ZB4DOnWHyZGvIzZgA/LTxJ/qn9OenTT/Rt1XfItVIXGGL/sqtWKAKkya5dn1efhmuv97dGvrll5YEjImw/Rn7ufeLezn1pVP5bfdvvHf5e0y6ZFJMJoBsdkYQtPR017jbV19B2bKu8O/cOeiojIlbK7et5MnvnuS6k65jZPeRVC5dOeiQfGdnBEH6+mvXv+5XX0Hfvq6HL0sCxkTcnoN7eGP+GwC0qtGK5YOXM/6C8XGRBMASQTB27YI77nC3gS5dCq++6qqGjrRROWPMMZu2chotx7Tk+g+uZ+mWpQCBdhsZBEsEkfbJJ9C0KYwcCX36uJ7A+vULOipj4k5qWirXf3A9PSf2pExiGb75yzdR00hcYbNrBJGSkeGafv7uOyhe3LUOeumlQUdlTFzKbiRu5baVDOswjPvOvi+qGokrbJYIIiE93XX+8t13rrvHn35yHbwbYyJqy94tVC1TlYRiCTzW9THqVapHm+PbBB1W4KxqyG/798Mll8DUqfDAA7B2rSUBYyJMVXn1p1dp+nxTXprzEgAXNL/AkoDHzgj8tGkT9OrlrgM8+KBLBMaYiFq9YzUDPhrA56s+p0PdDnRuYHfm5WSJwC8bNkCXLrB8OTzzjOskxhgTUW/Mf4ObptyEiDCm9xj+lvy3mGgkrrBZIvDDli1w+umwcSO8955rJtoYE3HHlTuOs+udzQt9XqBuxbpBh1NkWSIobFu3wplnujOC116zJGBMBKVnpvP4t4+TqZkM7zic7o26071R96DDKvIsERSmrCy49VZYuRLef9/dKWSMiYi5G+fy1w//yvzf53PViVf90UicKZhVlhWme++FiROhf39LAsZEyL70fdz937tp91I7ft/7O+9f8T4TL55oSeAI+JoIRKSniCwXkZUicncu0+uKyFci8pOILBCR3n7G46tly1w3kuecAy+9FHQ0xsSNVdtX8dT3T9GvTT+WDFrChc0vDDqkqONbIhCRBGA00AtIAq4UkaQcs90H/FtVTwb6AmP8isdXqvCXv0BCArzwgnUfaYzPdh3YxYR5EwBoWaMlK4as4OXzX46bRuIKm59nBO2Alaq6SlUPAm8DF+SYR4EK3vuKwG8+xuOfG25wHcvfcw80aRJ0NMbEtKkrptJqTCv6p/T/o5G4WOo2Mgh+JoJawLqQz+u9caEeBK4RkfXAVGBIbgsSkQEiMltEZm/ZssWPWI/eU0/B+PGuT4GHHgo6GmNi1ta0rVz7/rWc+9a5lC9Znm//+m3cNhJX2IK+WHwlMEFVawO9gTdE/vy0h6qOU9VkVU2uXr16xIPM0+uvw//9n+tYPiUFigW9O42JTdmNxL296G2Gnz2cuQPmcnrt04MOK2b4efvoBqBOyOfa3rhQ/YGeAKr6vYiUAqoBm32Mq3Bs3w6DB7tWRefOhRIlgo7ImJjz+57fqV62OgnFEniy25PUq1SP1se1DjqsmOPnIewsoImINBCREriLwSk55lkLdAEQkRZAKaCI1f3kYehQ2L0b3noLatYMOhpjYoqq8srcV2j2fDPGzRkHwHnNzrMk4BPfzghUNUNEBgPTgARgvKouFpERwGxVTQFuB14Skb/jLhz3U1X1K6ZCs38/fPABNGwIV14ZdDTGxJRV21dx40c38uWvX9KxXke6NuwadEgxz9cni1V1Ku4icOi44SHvlwDt/YzBF/fcA3v2uCYkjDGF5rV5rzFo6iASJIEXzn2BG0+50RqJiwBrYuJIpabCqFFw1lnWjpAxheyE8idwToNzGHvuWGpXsD68I8USwZF66ik3vP/+YOMwJgYczDzIo/97lCzN4sFOD9KtUTe6NeoWdFhxx865jsSBA/D881CxoutrwBhz1GZtmMUp407hgekPsGr7KqLh8mCssjOCI9GrF+za5TqeT0gIOhpjolJaehrDvxrO0zOfpma5mqT0TeG8ZucFHVZcs0QQroMH4auv3PtLLw02FmOi2K/bf+W5H5/jxrY38ljXx6hYqmLQIcU9SwTheuwxNxw9Otg4jIlCO/fv5D9L/8NfTv4LLWu0ZOWQldSpWKfgL5qIsGsE4dizB4YPh1q14Kabgo7GmKgy5ecptBzTkhs+uoFlW5cBWBIoYiwRhGO49+jDffdZE9PGhGnL3i1c/Z+r6TOpD5VLV+b7/t/TvFrzoMMyubCqoYIsW+aqg6pUgQEDgo7GmKiQmZXJWa+exa/bf+WhTg9x91l3UyLB2uMqqiwRFGTQIHeh+OOPrXVRYwqwac8mapStQUKxBEZ2H0n9SvVpVaNV0GGZAoRdsolIGT8DKZLmznV3CnXtCmecEXQ0xhRZWZrFi7NfpOlzTXlx9osA9Gnax5JAlCgwEYjImSKyBFjmfT5JRKKzS8kj9dZbbmhtChmTp5XbVtLl9S4MnDKQU2udSo/GPYIOyRyhcKqGngZ64DUhrarzReRsX6MqCrKyYORI6NYNTjgh6GiMKZJe/elVBk0dRImEErx03kv0P7k/YjdURJ2wrhGo6rocf9xMf8IpQmbOdMOzYz/nGXO06lasS49GPRjdezS1KuTsidZEi3ASwToRORNQEUkEbgGW+htWEfDII2542WXBxmFMEXIg4wD/+t+/yNIsRnQeQZeGXejS0NrdinbhXCweCNyM63h+A9AGGORjTMFThSlToH17aNYs6GiMKRJ+WP8Dp4w7hYe+foi1O9daI3ExJJwzgmaqenXoCBFpD3zrT0hFwH//64bt2gUbhzFFwN6De7n/q/sZNXMUtSrU4uMrP+bcpucGHZYpROGcETwX5rjYMWGCGz7wQKBhGFMUrNm5hjGzxjAweSCLBy22JBCD8jwjEJEzgDOB6iJyW8ikCrg+iGPXzz+7DukrWquIJj7t2L+DyUsmc0PbG0iqnsTKoSutx7AYlt8ZQQmgHC5ZlA957QJitx3mHTtg9my46qqgIzEmEB8u+5Ck0UkM/HjgH43EWRKIbXmeEajq18DXIjJBVddEMKZgfetd+mhlT0Sa+LJ572aGfjKUdxa/Q+vjWpNyZYo1EhcnwrlYnCYiTwAtgVLZI1X1HN+iCtKXX7ph797BxmFMBGVmZdJ+fHvW7lzLw50f5q72d5GYkBh0WCZCwkkEE4F3gD64W0mvB7b4GVSgvvnGdUNZo0bQkRjju992/8bx5Y4noVgCz/R8hvqV6pNUPSnosEyEhXPXUFVVfQVIV9WvVfWvQGyeDQDMmgUnnxx0FMb4KkuzGDtrLM2fb84Ls18AoHeT3pYE4lQ4ZwTp3nCjiJwL/AZU8S+kAG3f7obNrV7UxK6fU3/mxo9uZMaaGXRt2JVejXsFHZIJWDiJ4GERqQjcjnt+oAJwq59BBeb7793w4ouDjcMYn7wy9xUGfzKYUsVLMf788fRr088aiTMFJwJV/dh7uxPoDH88WRx7shPBKacEG4cxPqlfqT69GvdidO/R1CxfM+hwTBGR3wNlCcDluDaGPlXVRSLSB7gXKA3EXkX69OluWNvumTax4UDGAf4x4x8APHzOw9ZInMlVfmcErwB1gB+BZ0XkNyAZuFtVP4hAbJG3cCGUKmVdUpqY8N267+if0p9lW5fx1zZ/RVWtGsjkKr9EkAy0VtUsESkFbAIaqWpqZEKLsJkzYedO66DeRL09B/cw7IthPPfjc9SpWIdPr/7Ueg0z+crv0PegqmYBqOp+YNWRJgER6Skiy0VkpYjcncc8l4vIEhFZLCJvHcnyC9WsWW54882BhWBMYVi7cy0vznmRm0+9mUU3LbIkYAqU3xlBcxFZ4L0XoJH3WQBV1db5Ldi7xjAa6AasB2aJSIqqLgmZpwlwD9BeVbeLSHBPca1e7YaNGwcWgjFHa/u+7by75F0GnDKApOpJrLplFSeUty5WTXjySwQtjnHZ7YCVqroKQETeBi4AloTMcyMwWlW3A6jq5mNc59HbuBHKlYMyZQILwZij8f7S9xk0dRBb9m6hY72ONKvWzJKAOSL5NTp3rA3N1QLWhXxeD5yWY56mACLyLa5p6wdV9dOcCxKRAcAAgLp16x5jWHn4+GN7othElU17NjHkkyFMXjKZNse3YcpVU2hWzXrUM0curM7rfV5/E6ATUBuYISInquqO0JlUdRwwDiA5Obnw+8dThd274fjjC33RxvghMyuTDq92YN3OdTxyziPcceYd1kicOWp+JoINuNtPs9X2xoVaD/ygqunAryLyMy4xzPIxrj/LfpCsc+eIrtaYI7V+13pOKH8CCcUSeLbnszSo3MCaijbHLKwb5kWktIgc6TnnLKCJiDQQkRJAXyAlxzwf4M4GEJFquKqiVUe4nmOXfcdQhw4RX7Ux4cjSLJ774TmaP9+csbPGAtCrSS9LAqZQFJgIROQ8YB7wqfe5jYjkLND/RFUzgMHANGAp8G9VXSwiI0TkfG+2aUCqiCwBvgLuDOQ5hddec8P69SO+amMKsmzrMs5+9WyGfjqUs+qeRZ+mfYIOycQYUc2/yl1E5uCanZ6uqid74xaq6okRiO9PkpOTdfbs2YW7UBE47jjYtKlwl2vMMXp57ssMnjqYMollGNVzFNe2vtaeDjZHRUTmqGpybtPCaoZaVXfm+PEV/gXboOzc6YYXXhhoGMbkplHlRpzX7Dye7/U8x5U7LuhwTIwKJxEsFpGrgATvAbChwHf+hhVBW7e6ofVBYIqA/Rn7GfH1CAAe6fIInRt0pnMDu4nB+Cuci8VDcP0VHwDewjVHfauPMUXWL7+4YU1rktcE69u139LmhTb863//YsveLRRUbWtMYQnnjKC5qg4DhvkdTCAWLnRDe5jMBGT3gd3c+8W9jJ41mnqV6jHtmml0b9Q96LBMHAknEYwUkeOBycA7qrrI55gi63//c01LNG0adCQmTq3ftZ6Xf3qZIe2G8M8u/6RciXJBh2TiTIFVQ6raGdcz2RbgRRFZKCL3+R5ZpKSkQItjbVbJmCOTmpb6x/MALaq3YNXQVTzT6xlLAiYQYT1QpqqbVPVZYCDumYLhfgYVMQcOQFaWtThqIkZVmbxkMkljkhj66VCWb10OYN1GmkCF80BZCxF5UEQW4jqv/w7XXET0S/WeXWtmDXUZ/23cvZFL/n0Jl717GXUq1GH2jbOtkThTJIRzjWA88A7QQ1V/8zmeyMpOBA0bBhuHiXnZjcRt2L2Bx7s+zt/P+DvFiwXd5qMxToG/RFU9IxKBBGK5Oy2nVKlg4zAxa93OddSqUIuEYgmM7j2aBpUb0LSq3ZhgipY8q4ZE5N/ecKGILAh5LQzpuSy6LfA24/TTg43DxJzMrEye/eFZmo8+1Ehcj8Y9LAmYIim/M4JbvGHstnC1xOssrU6d/Ocz5ggs3bKU/in9+X799/Rq3Ivzmp0XdEjG5CvPMwJV3ei9HaSqa0JfwKDIhOez1auhevWgozAxZNyccbR5sQ0/p/7MGxe9wZSrplC3ok+96hlTSMK5fbRbLuN6FXYggZgzB9q2DToKE0OaVGnCRc0vYsnNS7im9TXWUqiJCnlWDYnITbgj/4Y5rgmUB771OzDf7dnjhrVj405YE4x96ft4cPqDiAiPdn3UGokzUSm/awRvAZ8A/wLuDhm/W1W3+RpVJGR3T9mmTaBhmOg1Y80Mbki5gRXbVjDwlIGoqp0BmKiUX9WQqupq4GZgd8gLEanif2g+mz/fDTt1CjQME312HdjFoCmD6DihI5mayRfXfcHYPmMtCZioVdAZQR9gDq4jmtBfuQLR/RTW3r1uaM1LmCP02+7fmDBvAredfhsjOo+gbImyQYdkzDHJMxGoah9v2CBy4UTQd99BjRr2MJkJy9a0rfx78b8ZdOogmldrzq+3/Go9hpmYEU5bQ+1FpKz3/hoReUpEov9+uOLF4eDBoKMwRZyq8s6id0gancStn97Kz6k/A1gSMDElnNtHxwJpInIScDvwC/CGr1FFwrRpdqHY5Ou33b9x4TsX0ve9vtSrVI85A+bYk8EmJoXT6lWGqqqIXAA8r6qviEh/vwPzXWYmpKUFHYUpojKzMjn71bPZsHsDT3Z7kltOv8UaiTMxK5xf9m4RuQe4FuggIsWARH/D8ll2h/Xt2wcbhyly1uxYQ+0KtUkolsCYc8fQsHJDGlexGwpMbAunaugKXMf1f1XVTbi+CJ7wNSq/rVvnhtYPgfFkZmXy1PdP0WJ0C8bOdo3EdW/U3ZKAiQvhdFW5CZgIVBSRPsB+VX3d98j8tGaNGzZqFGwcpkhYtHkRZ44/k9s/u50uDbtwYfMLgw7JmIgK566hy4EfgcuAy4EfRORSvwPzVXarozWte8B498LsF2j7YltWbV/FWxe/RUrfFGpXsGZHTHwJ5xrBMOBUVd0MICLVgf8Ck/0MzFfp6W5oD5PFrezmIFpUa8FlLS9jVI9RVC9rLdGa+BROIiiWnQQ8qYTZ6X2RtXw5JCZCyZJBR2IiLC09jeFfDSdBEnis22N0rN+RjvU7Bh2WMYEKp0D/VESmiUg/EekHTAGm+huWz0qWPHRWYOLG9NXTaT22NSO/H8meg3tQ1aBDMqZICKfP4jtF5GLgLG/UOFV939+wfPbLL9CiRdBRmAjZuX8nd31+F+PmjqNR5UZ8ed2X1lS0MSHy64+gCfAk0AhYCNyhqhsiFZivMjJcExMmLmzcs5E3F77JHWfcwUOdH6JMYpmgQzKmSMmvamg88DFwCa4F0ueOdOEi0lNElovIShG5O5/5LhERFZHkI13HUfn5Z2gQm23pGWfL3i0894P7yTav1pzVt6zmie5PWBIwJhf5HRaXV9WXvPfLRWTukSxYRBKA0biuLtcDs0QkRVWX5JivPHAL8MORLP+YJCbCtujvW8f8maoyadEkhn4ylF0HdtGjcQ+aVm1qdwQZk4/8zghKicjJItJWRNoCpXN8Lkg7YKWqrlLVg8DbwAW5zPcP4DFg/xFHf7QyM6FVq4itzkTGup3rOG/SeVz9n6tpXKUxP/3tJ2skzpgw5HdGsBF4KuTzppDPCpxTwLJrAetCPq8HTgudwUsodVR1iojcmdeCRGQAMACgbt1CaAF7yxa7dTTGZGRl0Om1Tmzas4mnezzNkHZDSCiWEHRYxkSF/Dqm8fW2Cq/xuqeAfgXNq6rjgHEAycnJx3bPX2amG+7efUyLMUXD6h2rqVOhDsWLFefFPi/SsHJDGlaO7s7zjIk0Px8M2wDUCflc2xuXrTzQCpguIquB04EU3y8YZ7c8euKJvq7G+CsjK4Mnv3uSFqNbMGbWGAC6NuxqScCYo+DnPZSzgCYi0gCXAPoCV2VPVNWdQLXszyIyHXeL6mwfY4Jly9wwwaoNotWC3xfQP6U/s3+bzQXNLuCSpEuCDsmYqOZbIlDVDBEZDEwDEoDxqrpYREYAs1U1xa9152vPHjdMjsydqqZwjZk1hls+vYXKpSrzzqXvcFnSZYhI0GEZE9UKTATi/suuBhqq6givv+LjVfXHgr6rqlPJ0RyFqg7PY95OYUV8rLL7IihdOiKrM4Uju5G4VjVa0bdVX57u8TTVylQr+IvGmAKFc0YwBsjC3SU0AtgNvAec6mNc/kn0OlerUiXYOExY9h7cy31f3kfxYsV5ovsTnF3vbM6ud3bQYRkTU8K5WHyaqt6Md5+/qm4HSvgalZ8OHnTDEtG7CfHii1VfcOLYExn1wygOZB6wRuKM8Uk4ZwTp3lPCCn/0R5Dla1R+Wr3aDe05giJrx/4d3PHZHbzy0ys0qdKEGf1m0KFeh6DDMiZmhXNG8CzwPlBDRP4J/A94xNeo/FSqlBuWLx9sHCZPv+/5nbcXvc3/tf8/5g+cb0nAGJ+F0wz1RBGZA3QBBLhQVZf6Hplfli51ZwPW+miRkl3433L6LTSr1ozVt662i8HGREg4dw3VBdKAj0LHqepaPwPzTdmycOBA0FEYj6oyceFEbvn0FvYc3EPvJr1pUrWJJQFjIiicw+IpuOsDApQCGgDLgZY+xuWfFSugWbOgozDA2p1rGfjxQD5Z+Qln1D6DV85/hSZVmwQdljFxJ5yqocPaYvAaihvkW0R+E4GdO4OOIu5lZGXQaUInNu/dzLM9n2XQqYOskThjAnLEFeWqOldETit4ziJqyRJo1y7oKOLWqu2rqFexHsWLFeel816iUZVG1K9UP+iwjIlr4VwjuC3kYzGgLfCbbxH5bdu2Q88SmIjJyMpg5HcjeWD6Azze7XGGnjaULg27BB2WMYbwzghC77PMwF0zeM+fcHyW/UDScccFG0ecmbdpHv1T+jN341wuan4RlyVdFnRIxpgQ+SYC70Gy8qp6R4Ti8VdamhtaE9QR8/yPz/P3aX+naumqTL5ssrUUakwRlGciEJHiXgui7SMZkK9WrXLDWrWCjSMOZDcS1/q41lx94tU81eMpqpS29p2MKYryOyP4EXc9YJ6IpADvAnuzJ6rqf3yOrfBlNy9RuXKgYcSyPQf3MOyLYSQmJPJk9yetkThjokA4TUyUAlJxrY/2Ac7zhtEn+xpBzZrBxhGjPvvlM1qNacVzPz5Hema6NRJnTJTI74yghnfH0CIOPVCWLTr/w7OfKLa+CArV9n3bue2z25gwbwLNqjZjxl9mcFbds4IOyxgTpvwSQQJQjsMTQLboTATZVUPWBHWh2rx3M5OXTOaes+5heMfhlCpeKuiQjDFHIL9EsFFVR0QskkjIPhOwawTHbNOeTUxaOIm/n/F310jcLaupWqZq0GEZY45CftcIYq8j2B073LCUHbEeLVXltXmvkTQ6iXu+uIcVqSsALAkYE8XySwSx99jnxo1uaIngqKzesZqeE3vS78N+JFVPYt7AedZInDExIM+qIVXdFslAIqJsWTe0vgiOWEZWBp1f68zWtK2M7j2agckDKSbh3HRmjCnq4qtEPHgQKlQIOoqosnLbShpUakDxYsUZf/54GlZuSL1K9YIOyxhTiOLrkG7LFkhMDDqKqJCemc4j3zxCyzEtGT1rNACdG3S2JGBMDIqvM4Jdu9zL5Gvuxrn0T+nPvE3zuCzpMq5oeUXQIRljfBRfiaB0aXuquADP/vAst027jeplq/Ofy//DRS0uCjokY4zP4isRpKdDpUpBR1EkZTcSd/LxJ3PdSdcxsvtIKpe25y2MiQfxlQh27bJrBDnsPrCbe764h5IJJRnZYyQd6nWgQ70OQYdljImg+LpYvHYtZGQEHUWR8enKT2k1thVjZo1BUWskzpg4FV+JYONGqFgx6CgCl5qWyvUfXE+vib0om1iWb//6LU/1eAqR2HuY3BhTsPiqGipfHkqWDDqKwKXuS+X9pe9z/9n3M6zDMEoWt31iTDzz9YxARHqKyHIRWSkid+cy/TYRWSIiC0TkCxHx9yb1jAxo3NjXVRRVG3dv5MnvnkRVaVq1KWtuXcOIziMsCRhj/EsEXn/Ho4FeQBJwpYgk5ZjtJyBZVVsDk4HH/YoHcHcNxdnFYlVl/E/jaTG6Bfd/dT8rt60EsDuCjDF/8POMoB2wUlVXqepB4G3ggtAZVPUrVfV6lGcmUNvHeGDnzrjqi+DX7b/S/c3u9E/pz0nHn8T8gfOtkThjzJ/4eY2gFrAu5PN64LR85u8PfJLbBBEZAAwAqFu37tFFk5nphttiry293GRkZXDO6+eQmpbK2HPHMuCUAdZInDEmV0XiYrGIXAMkAx1zm66q44BxAMnJyUd3j2P2baONGh3V16PFitQVNKzckOLFivPqBa/SqHIj6lSsE3RYxpgizM9DxA1AaAlU2xt3GBHpCgwDzlfVA75Fk57uhjF6jSA9M52HZzxMq7GteP7H5wHoVL+TJQFjTIH8PCOYBTQRkQa4BNAXuCp0BhE5GXgR6Kmqm32MJaYTwezfZtM/pT8Lfl9A31Z9ufLEK4MOyRgTRXxLBKqaISKDgWlAAjBeVReLyAhgtqqmAE8A5YB3vYeZ1qrq+b4EdMA72YixTmmemfkMt312G8eXO54P+37I+c382X3GmNjla6moqlOBqTnGDQ9539XP9R9m7143TEvLf74okd1IXPIJyfQ/uT+Pd3ucSqUqBR2WMSYKxdbhcX4OHnTDetHdscquA7v4v8//j1LFS/F0z6dpX7c97eu2DzosY0wUi5/7CbOrhqL4OYKpK6bSckxLxs0dR/Fixa2ROGNMoYifM4KtW90wCgvPrWlbufXTW5m4cCItq7dk8mWTOa12fo9kGGNM+OInEWRfJC5bNtg4jsL2fdv56OePeKDjA9zb4V5KJETvWY0xpuiJn0SQ/WRx6dLBxhGmDbs2MHHhRO48806aVG3CmlvX2MVgY4wv4ucaQVaWGyYkBBtHAVSVl+a8RNKYJB6c/iC/bP8FwJKAMcY38ZcIihXdTf5l2y90eb0LAz4eQNuabVlw0wIaV4nPZrONMZETP1VDRTwRZGRl0OX1Lmzbt40X+7zIDW1vsEbijDERET+JIPsaQRFLBMu3LqdRlUYUL1ac1y58jUZVGlG7gr+tcRtjTKiiVSr6qYhdIziYeZCHpj/EiWNPZPSPowHoWL+jJQFjTMTFzxlBEaoa+nHDj/RP6c+izYu46sSruLr11UGHZIyJY5YIImzUzFHc/tnt1CxXk4+u/Ig+TfsEGo8xxsRPIshuhjqgRJDdSFy7Wu24se2NPNb1MSqWqhhILMYYEyp+EsFmr7uDCDcxsXP/Tu76/C5KJ5ZmVM9RnFnnTM6sc2ZEYzDGmPwEX2EeKeXLu2G5chFb5UfLPyJpTBIv//QyJRNKWiNxxpgiKX7OCLILYdcBjq+27N3CLZ/ewqRFkzixxol8cMUHnFrrVN/Xa4wxR8MSgQ92HtjJ1BVTeajTQ9x91t3WSJwxpkiLv0Tg08XidTvX8eaCN7n7rLtpXKUxa25dYxeDjTFRIX6uEWTfPlrIZwRZmsULs1+g5ZiWPPzNw380EmdJwBgTLeInEfhQNbQidQXnvHYON025iXa12rHwpoXWSJwxJurEX9VQISWCjKwMur3RjR37d/DK+a/wlzZ/QSJw/cEYYwqbJYIjtHTLUppUbULxYsV546I3aFSlESeUP6EQAjTGmGBY1VCYDmQc4IGvHqD1C615/sfnAehQr4MlAWNM1LMzgjDMXD+T/in9WbJlCde2vpZrW19byMEZY0xw4i8RHOHtoyO/G8mdn99J7Qq1mXrVVHo16eVDcMYYE5z4SQRHePtolmZRTIpxRp0zGJg8kEe7PkqFkhV8DNAYY4IRP4kgzKqhHft3cPu02ymTWIbnej9njcQZY2KeXSwO8cGyD0gancRr81+jfMny1kicMSYu2BkBsHnvZgZPHcy7S96lzfFt+Piqj2lbs22EAzTGmGBYIgB2HdjF56s+55/n/JM7z7yTxITECAdnjDHBidtEsHbnWt6Y/wb3driXxlUas/bWtZQvWT7AAI0xJhi+XiMQkZ4islxEVorI3blMLyki73jTfxCR+r4F4yWCLJQxs8bQckxLHvnfI380EmdJwBgTr3xLBCKSAIwGegFJwJUikpRjtv7AdlVtDDwNPOZXPKiyvCp0evdcbp56M2fUPoPFgxZbI3HGmLjn5xlBO2Clqq5S1YPA28AFOea5AHjNez8Z6CI+tdyWkZlOj2th4dYlvHrBq0y7Zhr1K9X3Y1XGGBNV/LxGUAtYF/J5PXBaXvOoaoaI7ASqAltDZxKRAcAAgLp16x5VMMWbJ/Hm0s40um88NavWP6plGGNMLIqK5whUdZyqJqtqcvXq1Y9uIRdcwFkTvrQkYIwxOfiZCDYAdUI+1/bG5TqPiBQHKgKpPsZkjDEmBz8TwSygiYg0EJESQF8gJcc8KcD13vtLgS/VHuc1xpiI8u0agVfnPxiYBiQA41V1sYiMAGaragrwCvCGiKwEtuGShTHGmAjy9YEyVZ0KTM0xbnjI+/3AZX7GYIwxJn9RcbHYGGOMfywRGGNMnLNEYIwxcc4SgTHGxDmJtrs1RWQLsOYov16NHE8txwHb5vhg2xwfjmWb66lqrk/kRl0iOBYiMltVk4OOI5Jsm+ODbXN88GubrWrIGGPinCUCY4yJc/GWCMYFHUAAbJvjg21zfPBlm+PqGoExxpg/i7czAmOMMTlYIjDGmDgXk4lARHqKyHIRWSkid+cyvaSIvONN/0FE6gcQZqEKY5tvE5ElIrJARL4QkXpBxFmYCtrmkPkuEREVkai/1TCcbRaRy72/9WIReSvSMRa2MH7bdUXkKxH5yft99w4izsIiIuNFZLOILMpjuojIs97+WCAibY95paoaUy9ck9e/AA2BEsB8ICnHPIOAF7z3fYF3go47AtvcGSjjvb8pHrbZm688MAOYCSQHHXcE/s5NgJ+Ayt7nGkHHHYFtHgfc5L1PAlYHHfcxbvPZQFtgUR7TewOfAAKcDvxwrOuMxTOCdsBKVV2lqgeBt4ELcsxzAfCa934y0EVEJIIxFrYCt1lVv1LVNO/jTFyPcdEsnL8zwD+Ax4D9kQzOJ+Fs843AaFXdDqCqmyMcY2ELZ5sVqOC9rwj8FsH4Cp2qzsD1z5KXC4DX1ZkJVBKRmseyzlhMBLWAdSGf13vjcp1HVTOAnUDViETnj3C2OVR/3BFFNCtwm71T5jqqOiWSgfkonL9zU6CpiHwrIjNFpGfEovNHONv8IHCNiKzH9X8yJDKhBeZI/98L5GvHNKboEZFrgGSgY9Cx+ElEigFPAf0CDiXSiuOqhzrhzvpmiMiJqrojyKB8diUwQVVHisgZuF4PW6lqVtCBRYtYPCPYANQJ+VzbG5frPCJSHHc6mRqR6PwRzjYjIl2BYcD5qnogQrH5paBtLg+0AqaLyGpcXWpKlF8wDufvvB5IUdV0Vf0V+BmXGKJVONvcH/g3gKp+D5TCNc4Wq8L6fz8SsZgIZgFNRKSBiJTAXQxOyTFPCnC99/5S4Ev1rsJEqQK3WUROBl7EJYForzeGArZZVXeqajVVra+q9XHXRc5X1dnBhFsowvltf4A7G0BEquGqilZFMMbCFs42rwW6AIhIC1wi2BLRKCMrBbjOu3vodGCnqm48lgXGXNWQqmaIyGBgGu6Og/GqulhERgCzVTUFeAV3+rgSd1Gmb3ARH7swt/kJoBzwrnddfK2qnh9Y0McozG2OKWFu8zSgu4gsATKBO1U1as92w9zm24GXROTvuAvH/aL5wE5EJuGSeTXvuscDQCKAqr6Auw7SG1gJpAF/OeZ1RvH+MsYYUwhisWrIGGPMEbBEYIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCKJBHJFJF5Ia/6+cy7pxDWN0FEfvXWNdd7QvVIl/GyiCR57+/NMe27Y43RW072flkkIh+JSKUC5m8T7a1xGv/Z7aOmSBKRPaparrDnzWcZE4CPVXWyiHQHnlTV1sewvGOOqaDlishrwM+q+s985u+Ha3V1cGHHYmKHnRGYqCAi5bx+FOaKyEIR+VNLoyJSU0RmhBwxd/DGdxeR773vvisiBRXQM4DG3ndv85a1SERu9caVFZEpIjLfG3+FN366iCSLyKNAaS+Oid60Pd7wbRE5NyTmCSJyqYgkiMgTIjLLa2P+b2Hslu/xGhsTkXbeNv4kIt+JSDPvSdwRwBVeLFd4sY8XkR+9eXNrsdXEm6Db3raXvXJ74Z6Knee93sc9BV/Bm1YN91Rl9hntHm94OzDMe5+Aa2+oGq5gL+uN/z9geC7rmwBc6r2/DPgBOAVYCJTFPZW9GDgZuAR4KeS7Fb3hdLw+D7JjCpknO8aLgNe89yVwrUiWBgYA93njSwKzgQa5xLknZPveBXp6nysAxb33XYH3vPf9gOdDvv8IcI33vhKuLaKyQf+97RXsK+aamDAxY5+qtsn+ICKJwCMicjaQhTsSPg7YFPKdWcB4b94PVHWeiHTEdVbyrde0RgnckXRunhCR+3Dt1PTHtV/zvqru9WL4D9AB+BQYKSKP4aqTvjmC7foEeEZESgI9gRmqus+rjmotIpd681XENRb3a47vlxaRed72LwU+D5n/NRFpgmtmITGP9XcHzheRO7zPpYC63rJMnLJEYKLF1UB14BRVTRfXomip0BlUdYaXKM4FJojIU8B24HNVvTKMddypqpOzP4hIl9xmUtWfxfV10Bt4WES+UNUR4WyEqu4XkelAD+AKXEcr4HqbGqKq0wpYxD5VbSMiZXDt79wMPIvrgOcrVb3Iu7A+PY/vC3CJqi4PJ14TH+wagYkWFYHNXhLoDPypz2Vx/TD/rqovAS/juvubCbQXkew6/7Ii0jTMdX4DXCgiZUSkLK5a5xsROQFIU9U3cY355dZnbLp3ZpKbd3ANhWWfXYAr1G/K/o6INPXWmSt1vc0NBW6XQ02pZzdF3C9k1t24KrJs04Ah4p0eiWuV1sQ5SwQmWkwEkkVkIXAdsCyXeToB80XkJ9zR9jOqugVXME4SkQW4aqHm4axQVefirh38iLtm8LKq/gScCPzoVdE8ADycy9fHAQuyLxbn8BmuY6D/qut+EVziWgLMFddp+YsUcMbuxbIA1zHL48C/vG0P/d5XQFL2xWLcmUOiF9ti77OJc3b7qDHGxDk7IzDGmDhnicAYY+KcJQJjjIlzlgiMMSbOWSIwxpg4Z4nAGGPinCUCY4yJc/8PBNQXp56FW+0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUROC score: 0.944366345\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('SML AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnL0lEQVR4nO3de5xXVb3/8dcb5CI3MS5eAAUVRShEGUXFW3g3Q/NkSSrZ8eQhEc1jniz15yW7nEozk1Qqjx5LzVOmaJrhpbQTmkMqgqQiKkJeEFSEAeXy+f2x9sg4Dsx3YPb3Mt/38/H4Pvb1u9dnDzqfWWvtvZYiAjMzq17tSh2AmZmVlhOBmVmVcyIwM6tyTgRmZlXOicDMrMo5EZiZVTknAqsYkk6U9McCzrtW0oXFiKkYJL0k6ZBs/WJJvyx1TNa2OBFYq8h+Wa2QtEzS65JukNStNcuIiF9FxGEFnDchIr7VmmXXkxSSlmf3uVDSFZLa51HWxpDUQ9KVkuZnMb6QbfcudWxWvpwIrDV9OiK6AXsANcAFjU+QtFnRo2p9u2X3eSDweeBfSxwPAJI6Ag8Aw4AjgB7APsBiYK+NuF5b+LeyAjgRWKuLiIXAvcDH4YO/oidKeh54Ptt3tKQnJb0t6a+Shtd/X9IASbdLWiRpsaSrs/2nSPpLti5JP5L0hqSlkp6WVF/eDZIua3C9L0uaK2mJpKmStm1wLCRNkPR8FstkSSrwPucC/weMaHC9jbmvHSU9mO17U9KvJPVs4Y8dYDywHfCZiHgmItZGxBsR8a2IuKfB/e7UIKYPflaSDpK0QNLXJb0G/LekOZKObnD+Zln8e2Tbe2f3+bakpyQdtBFxW4k5EVirkzQAOAp4osHuY4FRwFBJuwPXA/8O9AKuA6ZK6pQ1s9wNvAwMBPoBtzZRzGHAAcDOwBbA50h/+TaOZQzw3ez4Ntl1G1/vaGBPYHh23uEF3ucQYH9gbra9sfelLMZtgV2BAcDFhcTQyCHAHyJi2UZ8t97WwMeA7YHTgFuAcQ2OHw68GRF/l9QP+D1wWfadrwG/ldRnE8q3EnAisNZ0h6S3gb8Afwa+0+DYdyNiSUSsIP2CuS4iHouINRFxI/AesDepCWNb4NyIWB4RKyPiL02UtQroDgwBFBFzIuLVJs47Ebg+Iv4eEe8B3wD2kTSwwTnfi4i3I2I+8BAN/sJfj79LWg7MAf4E/DTbv1H3FRFzI2JaRLwXEYuAK0jNTi3VC2jqZ9ASa4GLslhWADcDYyV1yY5/gZQcAE4C7omIe7LaxzSglvRHgFUQJwJrTcdGRM+I2D4iTs9+kdR7pcH69sA5WXPC21nyGED6RTkAeDkiVm+ooIh4ELgamAy8IWmKpB5NnLot6a/w+u8tI9Uc+jU457UG63VANwBJs7MO12WS9m9wzh7ZOZ8n1XK6bsp9SdpK0q1Z5/NS4JfAxnTuLibVejbFoohYWb+RNX/NAT6dJYOxpOQA6X6Pb3S/+7VCDFZkTgRWLA2HuX0F+HaWNOo/XSLiluzYdoV0VEbEVRExEhhKaiI6t4nT/kn6hQWApK6kv5wXFnD9YRHRLfs80uhYRMRtwHTg/23ifX2H9PP5RET0IP2lXVA/RSP3A4dn97g+dUCXBttbNzre1HDE9c1DxwDPZMkB0j3d1Oh+u0bE9zYidishJwIrhZ8BEySNyjp9u0r6lKTuwN9IzRvfy/Z3ljS68QUk7Zl9vwOwHFhJatZo7BbgS5JGSOpE+qX7WES81Er38j3gy5K23oT76g4sA97J2t2bSmiFuIn0y/m3koZIaiepl6RvSqpvrnkS+IKk9pKOoLAmqFtJfTJfYV1tAFLN5dOSDs+u1znrcO6/kfFbiTgRWNFFRC3wZVLTzlukztZTsmNrgE8DOwHzgQWkJpjGepB+8b5FavpZDPygibLuBy4Efkv6RbwjcEIr3svTwMOktv+Nva9LSM1N75A6X2/fyFjeI3UY/wOYBiwlJaDewGPZaWdlcbxN6j+5o4Drvkqq+ewL/LrB/ldItYRvAotISehc/Hul4sgT05iZVTdnbjOzKudEYGZW5ZwIzMyqnBOBmVmVq7hBpXr37h0DBw4sdRhmZhVlxowZb0ZEk8N/VFwiGDhwILW1taUOw8ysokh6eX3H3DRkZlblnAjMzKqcE4GZWZWruD4CM7ONtWrVKhYsWMDKlSubP7lCde7cmf79+9OhQ4eCv+NEYGZVY8GCBXTv3p2BAwdS4ER0FSUiWLx4MQsWLGDQoEEFfy+3piFJ1ytNIzhrPccl6SqlKQRn1k99Z2aWl5UrV9KrV682mQQAJNGrV68W13jy7CO4gTSB9vocCQzOPqcB1+QYi5kZQJtNAvU25v5yaxqKiIcbTQfY2DHA/0Qa/vRRST0lbbOe6QY33V/+An/8Y2HnjhwJxxyTSxhmZuWmlH0E/fjw9IULsn0fSQSSTiPVGthuu+02rrTp0+Gyy5o/LwJ69IDvfAdeew223homTlx3bEPZdu1aWL4cli2Dd9/98GflSjjkEGjfHjp3ho4dmy575Uqoq4MuXWDzzTfuXs2sbLVv355PfOITrF69mkGDBnHTTTfRs2dPAGbPns2kSZNYuHAha9euZfz48VxwwQUf/JV/7733cuGFF1JXV0enTp0YM2YMl19++aYHFRG5fYCBwKz1HLsb2K/B9gNATXPXHDlyZOTqP/8zIv1KXvcZMSKia9e0fsklEeefH/Fv/xYxdmzEqFERAwdGdOsWIX30u+v77LlnxLBh6bt9+6brN/7+9tunYx07pm0p4owzIiZMiPj85yMmTYqYNSti7dp8fyZmbcQzzzxT6hCia9euH6yPHz8+LrvssoiIqKurix122CHuu+++iIhYvnx5HHHEEXH11VdHRMTTTz8dO+ywQ8yZMyciIlavXh0//elPmyyjqfsEamM9v1dLWSNYSJrQu15/CphHNneXXAInngh9+8LMmfD1r6dawbvvwgsvwEUXQbt20KcPbLVV+gweDL17p5pEt27Qvfu6T/32vfem69fWwptvwhZbwIAB0LVr+nTpsm45d246p0ePVCt47z248caUHq69Nn138eJ0vZ/8JC379IFFi2DvvVOs8+fD6NHwxBPwgx/AYYel/UuXpmUE1NSk+MysJPbZZx9mzpwJwM0338zo0aM57LDDAOjSpQtXX301Bx10EBMnTuT73/8+559/PkOGDAFSzeIrX/lKq8RRykQwFThD0q3AKOCdyKt/oCU6d4bhw9P61lunX6CQmn2eeQY+9rG0v10L+9n33HPT4rrhhg9vv/km3H03XHcdDByYYtt++5R0nn4a+veHP/whnTt+/IavfeCB6TN4MLzzTkoWr7ySksoee6R77ts33XNL79usXH31q/Dkk617zREj4MorCzp1zZo1PPDAA5x66qlAahYaOXLkh87ZcccdWbZsGUuXLmXWrFmcc845rRtvJrdEIOkW4CCgt6QFwEVAB4CIuBa4BziKNK9rHfClvGJpFe3awcc/Xuoo1undG045JX025NFH4Z57UvKqr6WsWgX33w9TpqRz/vzn9GnsmgIe5Bo8OCWLefNgyBB48cVUy9hpp3Rs0qQN96uYVZkVK1YwYsQIFi5cyK677sqhhx5a6pByfWpoXDPHA5iYV/mW2Xvv9Gns+ONTbSICXn01dYz36JE+3bvD44+nZqUlS9JxSE1PffvCs8+uSxyvvQa//nVq1nr88Y+Wc9ZZaXniifCnP8GECemz5ZbrahdOFFYKBf7l3to233xznnzySerq6jj88MOZPHkyZ555JkOHDuXhhx/+0Lnz5s2jW7du9OjRg2HDhjFjxgx22223Vo+p4iavr6mpCQ9DXebq6mDOHDj/fLjvvubPHzcu9VfU1aWnrt5+G3r2TAls221hxYp1x+rqUt9K794pKfXp40RiBZszZw677rprSWPo1q0by5YtA+CJJ57g2GOP5YUXXmDVqlUMGzaMKVOmcMghh7BixQqOP/54Dj/8cCZNmsTMmTM57rjjuOeee9h5551Zu3YtU6ZMYcKECR8po6n7lDQjImqaislDTFjr69IlvYtR30dRb9Ys+PGPYc2a1Mfx6KOplnHLLemzsfbbb12/zpZbpiaqCHjjDdhuu9QU1qED7LZb6u/o1cvJw8rC7rvvzvDhw7nllls4+eSTufPOO5k0aRITJ05kzZo1nHzyyZxxxhkADB8+nCuvvJJx48ZRV1eHJI4++uhWicM1Aiut+ieZ6p+e6tAhPSU1e3ZqmurTZ90TVe3apdrC/Pnw/vupn+OOO9J3Vq1qedlduqQaxqBBqZZx3HGpWWynnVIi69q1te/WSqwcagTF4BqBVZb6DuyGOnVKHdB7NDP81MQGXUxr1sDq1fDSSykpdOqUkkbnzinRzJmTmpjmzEkd2wMHpv6Nhx9OHdwvvpjWG9tpp9QEteee6UmsQYPSZ6edUqe4n6KyNsCJwNqG9u3TZ5ddmj4+evSGv//qq+mN8PnzYdo0eOQRWLgwvdMxdy789a9Nf2/IkJRUjj46fbbffpNuw6wUnAjMALbZJi0HD4aDD/7wsbVr0wt8Tz+dhgD5/e9TbWPevNTP8Y9/pP6QrC0XSH0Wn/xkuu5BB8Fee7lfokxERJseeG5jmvudCMyaU/8m+Zgxafuooz58/J//TIMa/vrXqUP8uefSW+nZG6MfMnx4SiD9+6cayJAhcNppcMQRH20is1bXuXNnFi9e3GaHoo5sPoLOnTu36HvuLDbLw5o1qQ9izhy46abU3LTjjrBgQWo+mj49dXg3JKWBCbfeOnVW9+qV3vQeMSL1l3gQwk1WzTOUbaiz2InArJSefhpuuw1+9avUYb0hw4enF/T22y89KdWCGajMnAjMKsXatakW8f77qUnq+efTY7K3357euWhsiy1g//3h2GPTS3i9esEnPpHep5DcL2EfcCIwawsefxweeAD69YPJk+GxxzZ8ft++qfP72mth2DAnhSrnRGDWls2cmTqs166Fhx5K70t06gRXXPHh8zp3Tk1Ko0enzurhw9P7EPvsk2oT1qY5EZhVozVrUkf1nXemdyFmzVr/W9hbbpk6o+ufjDruuNRB7fci2gwnAjNbZ+VKeP31NNHSSy+lz7e+lZqcFjYxN9QBB6Rmpv32gzPPdBNThXIiMLPCvfhimrDlJz9JEx69/vqHj++2W0oKZ52V+iCsIjgRmNmmWbIEvvnN9P7DW2+lGewg1SK22SbNcHfeeWl4cCtLG0oEHjHLzJr3sY+lp4+eeiqNx3TnnanTecWKNA/35Zent6832wwuvLDU0VoLORGYWcuNHZuG1Vi8OL3zcMMNqbN5zRq47LI0revpp5c6SiuQE4GZbZoOHeCLX0xzO6xcCePHp/kkrrkmjaV0zTVpKHArW04EZtZ6OnWCG29MTUYHHJDmtz799PQG9Nix8Le/lTpCa4ITgZm1vs6d4c9/TlOSnnVW2nfXXTBqVBpQ76KLPvo0kpWME4GZ5adXL7jyyjSH9J13woknwt//DpdemkZZPeCANGyGlZQTgZkVx9ix8MtfwvLl654seuSRNPS2BIceCt/7XjpuReVEYGbF1aVLqhFEpGEvunRJ+++/H77xjTQX9Iknpgl8rCicCMysdIYNSzWAiPQY6sSJaf/NN6eJfLbYAr72taaH4LZW40RgZuWhQwe4+uo0iurPf56mBF2xIr2s1rdvalp6771SR9kmORGYWXmR4NRT4fe/T7WEa65JtYO77oIBA+Dii1MNwlqNE4GZlbcJE9Iw2mefnZqILrkkDXmxZEmpI2sznAjMrDJccUUaJnuvveDtt9Ojqccfn5KEbRInAjOrHNtum6bo/PnP0/ZvfpOGwu7ePY19ZBvFicDMKs+pp6ZO5euvT9vLlsH++0PXrm4y2gi5JgJJR0h6VtJcSec1cXx7SQ9IminpT5L65xmPmbUhEnzpS6nj+NVX0766utRk1KMHvPNOaeOrILklAkntgcnAkcBQYJykoY1O+yHwPxExHLgU+G5e8ZhZG7b11qmGcNVVaQa1d9+Fnj3hggtKHVlFyLNGsBcwNyLmRcT7wK3AMY3OGQo8mK0/1MRxM7PCSDBpUppm8+yz075vfxvat4dVq0oaWrnLMxH0A15psL0g29fQU8Bx2fpngO6SejW+kKTTJNVKql3kNwzNrDlXXJHmXoZUU+jYER56qLQxlbFSdxZ/DThQ0hPAgcBCYE3jkyJiSkTURERNnz59ih2jmVWigQNTEjj33LQ9Zgx89auljKhs5ZkIFgIDGmz3z/Z9ICL+GRHHRcTuwPnZvrdzjMnMqokE3/9+ejsZ4Mc/hn33TZ3K9oE8E8HjwGBJgyR1BE4ApjY8QVJvSfUxfAO4Psd4zKxaTZiQniLq2ROmT0+PmU6bVuqoykZuiSAiVgNnAPcBc4DbImK2pEsljc1OOwh4VtJzwFbAt/OKx8yqXI8e8NZbMHkybLUVHHbYunkRqpyiwgZvqqmpidra2lKHYWaV7B//gF13TeuTJqXHTts4STMioqapY6XuLDYzK74hQ+CPf0zrP/kJ/OpXpY2nxJwIzKw6HXoozJiR1k86Cf72t9LGU0JOBGZWvfbYIw1iBzBqFEyduuHz2ygnAjOrbnvtBbffntaPOSa9jFZlM6E5EZiZfeYzMGdOWj/nHNh999LGU2ROBGZmkDqQlyyBbbZJSWHsWFjzkYEO2iQnAjOzeltuCc8/n144u+su6N27KuZHdiIwM2uoa9c0jPWIEWlKzHbt2vyQFE4EZmaNSenR0l12SdsHH1zaeHLmRGBm1pR27dZ1ID/6KJx1VmnjyZETgZnZ+kgwe3Zav+oqGDSotPHkxInAzGxDhg6FlSvT+ksvwYMPbvD0SuREYGbWnE6dYNastH7wwfDPf5Y2nlbmRGBmVohhw9YNTjdiRElDaW1OBGZmhfrCF2DkSFi0CH74w1JH02qcCMzMWqJ++Opzz4WXXy5tLK3EicDMrCU+9jG4+OK0PnDgukdMK5gTgZlZS110EfzgB2l96NCKH4bCicDMbGN87WswZkxa32uv0sayiZwIzMw21rRpadTS2lrYd99SR7PRnAjMzDZWu3Ywc2Zanz4dXnihtPFsJCcCM7NN0aED3HNPWt9jj4qcw8CJwMxsUx15JIweDUuXwmmnlTqaFnMiMDNrDY88AltvDddfD2+8UepoWsSJwMysNUhw7bVp/cQTSxtLCzkRmJm1lmOOgZ12gvvvh+XLSx1NwZwIzMxa01e/mpYVNBaRE4GZWWs65ZS0/O53SxpGSzgRmJm1pq5d05DV770Ht99e6mgK4kRgZtbapk1Ly7PProhxiJwIzMxa2zbbpBFK589PyaDM5ZoIJB0h6VlJcyWd18Tx7SQ9JOkJSTMlHZVnPGZmRfPNb6blj3+c3jEoY7klAkntgcnAkcBQYJykoY1OuwC4LSJ2B04AfppXPGZmRdWhQ3qMFNYNWV2m8qwR7AXMjYh5EfE+cCtwTKNzAuiRrW8BtK0Zoc2suh18MIwaBXfdBbNnlzqa9SooEUgaLWmapOckzZP0oqR5zXytH/BKg+0F2b6GLgZOkrQAuAeYtJ7yT5NUK6l20aJFhYRsZlYefvSjtDz22JKGsSGF1gh+AVwB7AfsCdRky001DrghIvoDRwE3SfpITBExJSJqIqKmT58+rVCsmVmR7LMPDBgAc+fCU0+VOpomFZoI3omIeyPijYhYXP9p5jsLgQENtvtn+xo6FbgNICKmA52B3gXGZGZWGe68My3rO5DLTKGJ4CFJP5C0j6Q96j/NfOdxYLCkQZI6kjqDpzY6Zz5wMICkXUmJwG0/Zta27L47HHVUmrdg5cpSR/MRmxV43qhsWdNgXwBj1veFiFgt6QzgPqA9cH1EzJZ0KVAbEVOBc4CfSTo7u94pERXw9oWZWUsdfXRKBL/7HYwbV+poPkSV9nu3pqYmamtrSx2GmVnLvPEGbLUV9OoFb75Z9OIlzYiImqaOFfrU0BaSrqh/ckfS5ZK2aN0wzczasL594bOfhcWL181zXCYK7SO4HngX+Fz2WQr8d15BmZm1SZddlpYTJpQ2jkYK7SPYMSL+pcH2JZKezCEeM7O2a5ddoKYGpk8vdSQfUmiNYIWk/eo3JI0GVuQTkplZG7bttmlZRi/HFpoIvgJMlvSSpJeBq4HyqtuYmVWC009Py8svL20cDRTUNBQRTwK7SeqRbS/NMygzszbr8MPTRPd/+UupI/nABhOBpJMi4peS/qPRfgAi4oocYzMza5smTYKrroJnn039BiXWXNNQ12zZfT0fMzNrqZNPTsv/+q/SxpHZYI0gIq7LlpcUJxwzsyowcmRavv56aePIFPpC2fcl9ZDUQdIDkhZJOinv4MzM2iQJvvAFeOihUkcCFP7U0GFZB/HRwEvATsC5eQVlZtbm7bMPrFixbhazEio0EdQ3IX0K+N+IeCeneMzMqsNhh6XljTeWNg4KTwR3S/oHMBJ4QFIfoPzGUjUzqxSDB6flrbeWNg4KTAQRcR6wL1ATEauA5Xx0/mEzMytU9hg+221X2jho/j2CMRHxoKTjGuxreMrteQVmZtbmffGLqWno3Xehe+meyG/uzeIDgQeBTzdxLHAiMDPbeMOHp+XPfw5nn12yMJp7j+CibPml4oRjZlZFxo+Hc86Bxx4raRiFvkfwHUk9G2xvKemy3KIyM6sGvXql5d//XtIwCn1q6MiIeLt+IyLeAo7KJSIzs2ohwY47wiuvlDSMQhNBe0md6jckbQ502sD5ZmZWiDFjYOVKuL10Xa6FJoJfkd4fOFXSqcA0oPRvQZiZVbof/jAt77uvZCEoIgo7UToCOCTbnBYRJYm6pqYmamtrS1G0mVk++vSBN9+EAn8fbwxJMyKipqljhdYIAOYAf4iIrwGPSPIw1GZmreFTn0rLEs1lXOhTQ18GfgNcl+3qB9yRU0xmZtVl/Pi0fOONkhRfaI1gIjAaWAoQEc8DffMKysysqmy/fVqW6DHSQhPBexHxfv2GpM1IbxabmdmmGjAgLX/725IUX2gi+LOkbwKbSzoU+F/grvzCMjOrIh07wqhRMHt2rh3G61NoIvg6sAh4Gvh34B7ggryCMjOrOsOGpeXixUUvutlEIKk9MCcifhYRx0fEZ7N1Nw2ZmbWWT34yLWfPLnrRzSaCiFgDPCup9INmm5m1VXvumZZ3Fb/VvdCmoS2B2dnE9VPrP819SdIRkp6VNFfSeU0c/5GkJ7PPc5LebmH8ZmZtwy67wIgRMGVK0Ytubj6Cehe29MJZk9Jk4FBgAfC4pKkR8Uz9ORFxdoPzJwG7t7QcM7M2o317qKsrerHNzVDWGZgA7ETqKP5FRKwu8Np7AXMjYl52rVtJ01s+s57zxwEXFXhtM7O258ADYcaMohfbXNPQjUANKQkcCVzegmv3AxqOrbog2/cRkrYHBpFmQzMzq07t26flyy8XtdjmEsHQiDgpIq4DPgvsn1McJwC/yTqmP0LSaZJqJdUuWrQopxDMzEps96x1/KGHilpsc4lgVf1KC5qE6i0EBjTY7p/ta8oJwC3ru1BETImImoio6dOnTwvDMDOrEAcfnJaPPFLUYpvrLN5N0tJsXaQ3i5dm6xERPTbw3ceBwZIGkRLACcAXGp8kaQjpqaTSDLtnZlYu6v/Qfeutohbb3OT17Tf2whGxWtIZwH1Ae+D6iJgt6VKgNiLqHz89AbjVL6iZWdWTYORIeP75ohZb6OOjGyUi7iENR9Fw3/9rtH1xnjGYmVWUHXeE226D999PYxAVQUsmpjEzs7ztumtavvZa0Yp0IjAzKyf1ieBvfytakU4EZmblpCabVvjuu4tWpBOBmVk52XHHtCxih7ETgZlZOXrllebPaSVOBGZm5ebTn4bXXy9acU4EZmblZuDA9PjomiZH3Wl1TgRmZuWme/e0XLGiKMU5EZiZlZutt07LlSuLUpwTgZlZuencOS2LNJG9E4GZWbmpH1piyZKiFOdEYGZWbnbYIS1nzy5KcU4EZmblZvDgtCzStJVOBGZm5WarrdJy1qyiFOdEYGZWbqS0fOGFohSX63wEZma2kfr1W/f0UM5cIzAzK0f77lu0GoETgZlZOXr33aIV5URgZlaO6oejLkJCcCIwMytHw4en5Zw5uRflRGBmVo523jktFy7MvSgnAjOzcrTttmlZhBFInQjMzMrR5punpZuGzMyqVN++RSvKicDMrBx16gQ9e8Kbb+ZelBOBmVm56tYtTVmZMycCM7NyFQFvvZV7MU4EZmblSoLf/S73YpwIzMzK1bvvQrv8f007EZiZlavjjoO1a2H16lyLcSIwMytX9VNWPvlkrsXkmggkHSHpWUlzJZ23nnM+J+kZSbMl3ZxnPGZmFaV+vKHly3MtJrdEIKk9MBk4EhgKjJM0tNE5g4FvAKMjYhjw1bziMTOrOPVTVr70Uq7F5Fkj2AuYGxHzIuJ94FbgmEbnfBmYHBFvAUTEGznGY2ZWWbp0ScvN8p1MMs9E0A94pcH2gmxfQzsDO0v6P0mPSjqiqQtJOk1SraTaRYsW5RSumVmZ6dkzLevqci2m1J3FmwGDgYOAccDPJPVsfFJETImImoio6dOnT3EjNDMrlW7d0vLpp3MtJs9EsBAY0GC7f7avoQXA1IhYFREvAs+REoOZmdXXCNasybWYPBPB48BgSYMkdQROAKY2OucOUm0ASb1JTUXzcozJzKxySDBwIMyfn2sxuSWCiFgNnAHcB8wBbouI2ZIulTQ2O+0+YLGkZ4CHgHMjYnFeMZmZVZwOHWDWrFyLUETkWkBrq6mpidra2lKHYWZWHKNHw1//mgag2wSSZkRETVPHSt1ZbGZmGxIBPXrkWoQTgZlZOdtjj9Q8lCMnAjOzctapEyzOt+vUicDMrJzVz1D2zju5FeFEYGZWzvplAzLkOKqCE4GZWTkbNCgtV63KrQgnAjOzclbfUZzjJPZOBGZm5axjx7TMcU4CJwIzs3JWP2fxkiX5FZHblc3MbNP175+Wc+fmVoQTgZlZOdtmm7Rsl9+vaycCM7NyVj9LWY4vlTkRmJmVs803T8vp03MrwonAzKyc1TcJ5ThvsROBmVm522039xGYmVW1Ll38QpmZWVWTYObM3C7vRGBmVu4WLlz3hnEOnAjMzMrd/vvDK6/kdnknAjOzcrdokTuLzcyq2u6753p5JwIzs3K3dm365PTkkBOBmVm5qx9mYunSXC7vRGBmVu622iot33wzl8s7EZiZlbv6R0dzmq7SicDMrNz16pWWq1fncnknAjOzclc/4JwTgZlZlXIiMDOrcvWJYOXKXC7vRGBmVu4i0jKnWcqcCMzMyl3v3mnZvn0ul881EUg6QtKzkuZKOq+J46dIWiTpyezzb3nGY2ZWkeofH33vvVwun9vcZ5LaA5OBQ4EFwOOSpkbEM41O/XVEnJFXHGZmFa9z57Ssq8vl8nnWCPYC5kbEvIh4H7gVOCbH8szM2qYBA+BTn4J+/XK5fH6zIUM/oOEA2guAUU2c9y+SDgCeA86OiI8Mui3pNOA0gO222y6HUM3MyliHDnD33bldvtSdxXcBAyNiODANuLGpkyJiSkTURERNnz59ihqgmVlbl2ciWAgMaLDdP9v3gYhYHBH1vR8/B0bmGI+ZmTUhz0TwODBY0iBJHYETgKkNT5C0TYPNscCcHOMxM7Mm5NZHEBGrJZ0B3Ae0B66PiNmSLgVqI2IqcKakscBqYAlwSl7xmJlZ0xT1b6xViJqamqitrS11GGZmFUXSjIioaepYqTuLzcysxJwIzMyqnBOBmVmVq7g+AkmLgJc38uu9gXwm/Sxfvufq4HuuDptyz9tHRJMvYlVcItgUkmrX11nSVvmeq4PvuTrkdc9uGjIzq3JOBGZmVa7aEsGUUgdQAr7n6uB7rg653HNV9RGYmdlHVVuNwMzMGnEiMDOrcm0yERQwV3InSb/Ojj8maWAJwmxVBdzzf0h6RtJMSQ9I2r4Ucbam5u65wXn/IikkVfyjhoXcs6TPZf/WsyXdXOwYW1sB/21vJ+khSU9k/30fVYo4W4uk6yW9IWnWeo5L0lXZz2OmpD02udCIaFMf0kinLwA7AB2Bp4Chjc45Hbg2Wz+BNG9yyWPP+Z4/CXTJ1r9SDfecndcdeBh4FKgpddxF+HceDDwBbJlt9y113EW45ynAV7L1ocBLpY57E+/5AGAPYNZ6jh8F3AsI2Bt4bFPLbIs1gkLmSj6GdbOh/QY4WJKKGGNra/aeI+KhiKif+fpR0kRBlazQObG/BfwXsLKYweWkkHv+MjA5It4CiIg3ihxjayvkngPoka1vAfyziPG1uoh4mDQs//ocA/xPJI8CPRvN7dJibTERNDVXcuMZnz84JyJWA+8AvYoSXT4KueeGTiX9RVHJmr3nrMo8ICJ+X8zAclTIv/POwM6S/k/So5KOKFp0+Sjkni8GTpK0ALgHmFSc0Eqmpf+/NyvPyeutDEk6CagBDix1LHmS1A64guqb7GgzUvPQQaRa38OSPhERb5cyqJyNA26IiMsl7QPcJOnjEbG21IFVirZYI2h2ruSG50jajFSdXFyU6PJRyD0j6RDgfGBsrJsrulI1d8/dgY8Df5L0EqktdWqFdxgX8u+8AJgaEasi4kXgOVJiqFSF3POpwG0AETEd6EwanK2tKuj/95Zoi4mg2bmSs+0vZuufBR6MrBemQhUyP/TuwHWkJFDp7cbQzD1HxDsR0TsiBkbEQFK/yNiIqOTp7Qr5b/sOUm0ASb1JTUXzihhjayvknucDBwNI2pWUCBYVNcrimgqMz54e2ht4JyJe3ZQLtrmmoShsruRfkKqPc0mdMieULuJNV+A9/wDoBvxv1i8+PyLGlizoTVTgPbcpBd7zfcBhkp4B1gDnRkTF1nYLvOdzgJ9JOpvUcXxKJf9hJ+kWUjLvnfV7XAR0AIiIa0n9IEcBc4E64EubXGYF/7zMzKwVtMWmITMzawEnAjOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwa4KkNZKelDRL0l2Serby9V/KnvNH0rLWvLZZSzkRmDVtRUSMiIiPk941mVjqgMzy4kRg1rzpZIN6SdpR0h8kzZD0iKQh2f6tJP1O0lPZZ99s/x3ZubMlnVbCezBbrzb3ZrFZa5LUnjR8wS+yXVOACRHxvKRRwE+BMcBVwJ8j4jPZd7pl5/9rRCyRtDnwuKTfVvKbvtY2ORGYNW1zSU+SagJzgGmSugH7sm6YDoBO2XIMMB4gItaQhjYHOFPSZ7L1AaQB4JwIrKw4EZg1bUVEjJDUhTTOzUTgBuDtiBhRyAUkHQQcAuwTEXWS/kQaEM2srLiPwGwDslndziQNbFYHvCjpePhg7tjdslMfIE0BiqT2krYgDW/+VpYEhpCGwjYrO04EZs2IiCeAmaQJUE4ETpX0FDCbddMmngV8UtLTwAzS3Ll/ADaTNAf4HmkobLOy49FHzcyqnGsEZmZVzonAzKzKORGYmVU5JwIzsyrnRGBmVuWcCMzMqpwTgZlZlfv/RN2kurz8qvgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUPR score: 0.9439973436004492\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('SML AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML fpr95 score: 0.2555\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('SML fpr95 score:', fpr95)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}