2.3 Question answering with diff translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73031965",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "from training import compute_metrics_QA\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "dataset_name = {}\n",
    "dataset_name['fr'] = 'fquad'\n",
    "dataset_name['en'] = 'squad'\n",
    "data_path = '/data/desponds/data/Question_answering/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f042582",
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing import preprocessing_question_answering\n",
    "datasets, tokenized = preprocessing_question_answering(dataset_name, data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "837812ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from training import get_trainers_question_answering\n",
    "trainers = get_trainers_question_answering(data_path, tokenized, langs = ['en'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5567a3e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from training import get_models_question_answering\n",
    "models = get_models_question_answering(trainers)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bde3b6db",
   "metadata": {},
   "source": [
    "## Translation fr to en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cc515df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from translation import translate_fr_en_qa\n",
    "from transformers import pipeline\n",
    "import pickle\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "\n",
    "save_path = '/data/desponds/data/Question_answering/'\n",
    "data_path_trans = \"/data/desponds/data/translation_models/\"\n",
    "translators_paths = {}\n",
    "translators_paths['t5-small-1'] = {'fr_en' : \"model_t5_small_1_8_epoch_1/checkpoint-795/\", 'en_fr': \"en_fr_model_t5_small_1_8_epoch_1/checkpoint-795\"}\n",
    "translators_paths['t5-small-3'] = {'fr_en' : \"model_4_4_epoch_5/checkpoint-15890/\", 'en_fr': \"en_fr_model_t5_small_4_4_epoch_5/checkpoint-31775\"}\n",
    "translators_paths['t5-base'] = {'fr_en' : \"model_t5_base_4_4_epoch_3/checkpoint-12710/\", 'en_fr': \"en_fr_model_t5_base_4_4_epoch_3/checkpoint-19065\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a7ab99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Translation from fr to en \n",
    "\n",
    "for name in translators_paths:\n",
    "    translator = pipeline(\"translation\", model=f\"{data_path_trans}{translators_paths[name]['fr_en']}\")\n",
    "\n",
    "    translated_fr_en = datasets['fr']['validation'].map(lambda examples : translate_fr_en_qa(examples, translator), batched = True, batch_size = 512)\n",
    "    with open(f'{save_path}/{name}.pickle', 'wb') as handle:\n",
    "            pickle.dump(translated_fr_en, handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d3a5e8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "translated_fr_en"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a12fad2",
   "metadata": {},
   "source": [
    "## Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ea5a419",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prepare dataset for evaluation\n",
    "from preprocessing import preprocess_validation_examples_QA\n",
    "def remove_trans(example):\n",
    "    example['context'] = example['context']['translation_text']\n",
    "    \n",
    "    example['question'] = example['question']['translation_text']\n",
    "    return example\n",
    "\n",
    "datasets_en, tokenized_en = {}, {}\n",
    "for name in translators_paths:\n",
    "    with open(f'{save_path}/{name}.pickle', 'rb') as handle:\n",
    "        datasets_en[name] = pickle.load(handle)\n",
    "        datasets_en[name] = datasets_en[name].map(remove_trans)\n",
    "        tokenized_en[name] = datasets_en[name].map(lambda examples : preprocess_validation_examples_QA(examples, 'en'), \n",
    "                                                batched=True, remove_columns=datasets['fr'][\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fecf4c42",
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = {} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "389b9ba8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "metrics, all_predicted_answers = {}, {}\n",
    "for name in translators_paths:\n",
    "    \n",
    "    translator = pipeline(\"translation\", model=f\"{data_path_trans}{translators_paths[name]['en_fr']}\")\n",
    "    \n",
    "    predictions, _, _ = models['en'].predict(tokenized_en[name])\n",
    "    start_logits, end_logits = predictions\n",
    "    metric, predicted_answers, theoretical_answers = compute_metrics_QA(start_logits, end_logits, \n",
    "               tokenized_en[name], datasets_en[name],\n",
    "               need_translation = True, base_answers = datasets['fr'][\"validation\"],\n",
    "               translator = translator)\n",
    "    \n",
    "    metrics[name] = metric\n",
    "    all_predicted_answers[name] = predicted_answers\n",
    "with open('/data/desponds/data/Question_answering/translators_predicted_answers.pickle', 'wb') as handle:\n",
    "    pickle.dump(all_predicted_answers, handle)\n",
    "print(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c744ad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_predicted_answers['t5-small-1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a901f054",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "results_qa = pd.DataFrame() \n",
    "\n",
    "results_qa['question'] = [th['question'] for th in datasets['fr'][\"validation\"]]\n",
    "results_qa['theoretical_answers'] = [th['answers']['text'][0] for th in datasets['fr'][\"validation\"]]\n",
    "for name in translators_paths :\n",
    "    results_qa[name] = [th['prediction_text'] for th in all_predicted_answers[name]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "8de42df6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>theoretical_answers</th>\n",
       "      <th>t5-small-1</th>\n",
       "      <th>t5-small-3</th>\n",
       "      <th>t5-base</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Que concerne principalement les documents ?</td>\n",
       "      <td>La Vierge aux rochers</td>\n",
       "      <td>Objekte der Spéculation für die Forscher</td>\n",
       "      <td>Objekte der Spéculation</td>\n",
       "      <td>Verdachtsobjekte</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Par quoi sont décrit les deux tableaux ?</td>\n",
       "      <td>documents contemporains</td>\n",
       "      <td>La Vierge zu den Rössern, aber sie bleiben Obj...</td>\n",
       "      <td>certes</td>\n",
       "      <td>certes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Quels types d'objets sont les deux tableaux au...</td>\n",
       "      <td>objets de spéculations</td>\n",
       "      <td>Objekte der Spéculation für die Forscher</td>\n",
       "      <td>Objekte der Spéculation</td>\n",
       "      <td>Verdachtsobjekte</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sur quelle jambe les personnages se tiennent-t...</td>\n",
       "      <td>droite</td>\n",
       "      <td>the characters hold en support on their jambe ...</td>\n",
       "      <td>Unterstützung auf dem rechten Bein</td>\n",
       "      <td>threw him</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Quel pied avancent les personnages ?</td>\n",
       "      <td>gauche</td>\n",
       "      <td>hair longs and boucles, they are vêtus</td>\n",
       "      <td>Rechtes Bein</td>\n",
       "      <td>links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>De quelle couleur est l'ange qui joue de la li...</td>\n",
       "      <td>vert</td>\n",
       "      <td>jambe droite et avancer leur pied gauche vers ...</td>\n",
       "      <td>weiß</td>\n",
       "      <td>gris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>D'où sont issus les deux panneaux ?</td>\n",
       "      <td>atelier de Léonard de Vinci</td>\n",
       "      <td>die beiden panneaux stammen aus Léonard von Vi...</td>\n",
       "      <td>Léonard de Vinciatelier</td>\n",
       "      <td>Werkstatt von Léonard de Vinci</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Lors de quel processus des craquelures apparai...</td>\n",
       "      <td>séchage</td>\n",
       "      <td>für bestimmte, sind die Ursache von Problemen</td>\n",
       "      <td>Verwendung von Fingernägeln zur Veredelung von...</td>\n",
       "      <td>Beendigung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Qu'est ce qui est utilisé pour la finition des...</td>\n",
       "      <td>doigts</td>\n",
       "      <td>Fingern</td>\n",
       "      <td>Fingernails</td>\n",
       "      <td>Finger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Quel est le nom du panneau central du retable ?</td>\n",
       "      <td>La Vierge aux rochers</td>\n",
       "      <td>La Vierge aux rochers</td>\n",
       "      <td>La Vierge zu den Felsen</td>\n",
       "      <td>Vier zu den Räubern</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Combien d'artistes ont été désignés dans le co...</td>\n",
       "      <td>trois</td>\n",
       "      <td>alle mit einer Rolle, die von den Historien de...</td>\n",
       "      <td>drei Künstler im Auftrag der Bestellung</td>\n",
       "      <td>drei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Qui a peint le panneau central du retable ?</td>\n",
       "      <td>Léonard de Vinci</td>\n",
       "      <td>eine Plakette zentraler Tisch</td>\n",
       "      <td>Léonard de Vinci</td>\n",
       "      <td>Léonard de Vinci</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Quelle oeuvre est conservée à l’Académie des b...</td>\n",
       "      <td>une Vierge à l'Enfant</td>\n",
       "      <td>Giovanni Ambrogio de Predis</td>\n",
       "      <td>zwei Tische</td>\n",
       "      <td>viele stilistische und technische Indizes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>De quand date le tableau qui se trouve à Zurich ?</td>\n",
       "      <td>fin des années 1480</td>\n",
       "      <td>Giovanni Ambrogio de Predis</td>\n",
       "      <td>Léonard de Vinci das zentrale Panel</td>\n",
       "      <td>zwei Gemälde der einzigen Hand von Giovanni Am...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>A quoi correspond le titre moderne des œuvres ?</td>\n",
       "      <td>description</td>\n",
       "      <td>identifient incorrectly the instrument joué by...</td>\n",
       "      <td>Ange musicien in rot</td>\n",
       "      <td>Musicien</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>De quel couleur est l'ange musicien jouant de ...</td>\n",
       "      <td>vert</td>\n",
       "      <td>rouge jouant of the luth and Ange musicien en ...</td>\n",
       "      <td>rot</td>\n",
       "      <td>rot, der Luth spielt, und Ange musicien umkehrt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>De quel instrument joue l'ange musicien rouge ?</td>\n",
       "      <td>luth</td>\n",
       "      <td>vièle</td>\n",
       "      <td>luth</td>\n",
       "      <td>Lira da Braccio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>A partir de quand sont créés les deux tableaux ?</td>\n",
       "      <td>1490</td>\n",
       "      <td>Juli dieses Jahres</td>\n",
       "      <td>1490</td>\n",
       "      <td>1490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Lors de quel mois Ambrogio rentre-t-il d'Innsb...</td>\n",
       "      <td>juillet</td>\n",
       "      <td>ne rentred d'Innsbruck</td>\n",
       "      <td>adolfia</td>\n",
       "      <td>1490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>De la figure de qui l'ange vert s’inspire-t-il ?</td>\n",
       "      <td>Jean l'évangéliste de La Cène</td>\n",
       "      <td>Francesco Napoletano</td>\n",
       "      <td>– Herr Präsident!</td>\n",
       "      <td>Der Engel des Musikers inspirierte die Figur v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Quel est le nom du retable décoré par les pann...</td>\n",
       "      <td>l'Immaculée Conception</td>\n",
       "      <td>bois Giacomo del Maino</td>\n",
       "      <td>die Tabelle des unberührten Empfangs</td>\n",
       "      <td>die Retabelle der unberührten Empfängnis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Dans quelle ville se situe l'église Saint-Fran...</td>\n",
       "      <td>Milan</td>\n",
       "      <td>.</td>\n",
       "      <td>Mailand</td>\n",
       "      <td>Mailand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Combien d'artistes sont mandatés pour la dorur...</td>\n",
       "      <td>Trois</td>\n",
       "      <td>sculptor on the bois Giacomo del Maino</td>\n",
       "      <td>Künstler werden dann beauftragt, Seeleute zu s...</td>\n",
       "      <td>zwei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Combien d'anges doit comporter chaque panneau ?</td>\n",
       "      <td>quatre</td>\n",
       "      <td>.</td>\n",
       "      <td>Die Künstler sind dann beauftragt, Seeleute zu...</td>\n",
       "      <td>Brüder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Qui est le commanditaire de l'oeuvre ?</td>\n",
       "      <td>confrérie de l'Immaculée Conception</td>\n",
       "      <td>the confrérie of the Immaculée Conception</td>\n",
       "      <td>Confrérie des beeindruckenden Empfangs</td>\n",
       "      <td>Confrère de l'Immaculée Conception</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Quand est signé le contrat de l'oeuvre ?</td>\n",
       "      <td>25 avril 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Quelle est la date de livraison convenue ?</td>\n",
       "      <td>8 décembre 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "      <td>25. April 1483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Comment sont les commanditaires vis à vis de l...</td>\n",
       "      <td>insatisfaits</td>\n",
       "      <td>Unsatisfaits</td>\n",
       "      <td>Unzufrieden</td>\n",
       "      <td>unzufrieden</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>A partir de quand Léonard de Vinci travaille-t...</td>\n",
       "      <td>début des années 1490</td>\n",
       "      <td>1490</td>\n",
       "      <td>1490</td>\n",
       "      <td>1490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>A partir de quand au plus tard les deux panne...</td>\n",
       "      <td>1503</td>\n",
       "      <td>wahrscheinlich, dass sie eine Art identique be...</td>\n",
       "      <td>sie hatten eine Art identisch gekannt</td>\n",
       "      <td>1490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             question  \\\n",
       "0         Que concerne principalement les documents ?   \n",
       "1            Par quoi sont décrit les deux tableaux ?   \n",
       "2   Quels types d'objets sont les deux tableaux au...   \n",
       "3   Sur quelle jambe les personnages se tiennent-t...   \n",
       "4               Quel pied avancent les personnages ?    \n",
       "5   De quelle couleur est l'ange qui joue de la li...   \n",
       "6                 D'où sont issus les deux panneaux ?   \n",
       "7   Lors de quel processus des craquelures apparai...   \n",
       "8   Qu'est ce qui est utilisé pour la finition des...   \n",
       "9     Quel est le nom du panneau central du retable ?   \n",
       "10  Combien d'artistes ont été désignés dans le co...   \n",
       "11        Qui a peint le panneau central du retable ?   \n",
       "12  Quelle oeuvre est conservée à l’Académie des b...   \n",
       "13  De quand date le tableau qui se trouve à Zurich ?   \n",
       "14    A quoi correspond le titre moderne des œuvres ?   \n",
       "15  De quel couleur est l'ange musicien jouant de ...   \n",
       "16    De quel instrument joue l'ange musicien rouge ?   \n",
       "17   A partir de quand sont créés les deux tableaux ?   \n",
       "18  Lors de quel mois Ambrogio rentre-t-il d'Innsb...   \n",
       "19   De la figure de qui l'ange vert s’inspire-t-il ?   \n",
       "20  Quel est le nom du retable décoré par les pann...   \n",
       "21  Dans quelle ville se situe l'église Saint-Fran...   \n",
       "22  Combien d'artistes sont mandatés pour la dorur...   \n",
       "23    Combien d'anges doit comporter chaque panneau ?   \n",
       "24             Qui est le commanditaire de l'oeuvre ?   \n",
       "25           Quand est signé le contrat de l'oeuvre ?   \n",
       "26         Quelle est la date de livraison convenue ?   \n",
       "27  Comment sont les commanditaires vis à vis de l...   \n",
       "28  A partir de quand Léonard de Vinci travaille-t...   \n",
       "29   A partir de quand au plus tard les deux panne...   \n",
       "\n",
       "                    theoretical_answers  \\\n",
       "0                 La Vierge aux rochers   \n",
       "1               documents contemporains   \n",
       "2                objets de spéculations   \n",
       "3                                droite   \n",
       "4                                gauche   \n",
       "5                                  vert   \n",
       "6           atelier de Léonard de Vinci   \n",
       "7                               séchage   \n",
       "8                                doigts   \n",
       "9                 La Vierge aux rochers   \n",
       "10                                trois   \n",
       "11                     Léonard de Vinci   \n",
       "12                une Vierge à l'Enfant   \n",
       "13                  fin des années 1480   \n",
       "14                          description   \n",
       "15                                 vert   \n",
       "16                                 luth   \n",
       "17                                 1490   \n",
       "18                              juillet   \n",
       "19        Jean l'évangéliste de La Cène   \n",
       "20               l'Immaculée Conception   \n",
       "21                                Milan   \n",
       "22                                Trois   \n",
       "23                               quatre   \n",
       "24  confrérie de l'Immaculée Conception   \n",
       "25                        25 avril 1483   \n",
       "26                      8 décembre 1483   \n",
       "27                         insatisfaits   \n",
       "28                début des années 1490   \n",
       "29                                 1503   \n",
       "\n",
       "                                           t5-small-1  \\\n",
       "0            Objekte der Spéculation für die Forscher   \n",
       "1   La Vierge zu den Rössern, aber sie bleiben Obj...   \n",
       "2            Objekte der Spéculation für die Forscher   \n",
       "3   the characters hold en support on their jambe ...   \n",
       "4              hair longs and boucles, they are vêtus   \n",
       "5   jambe droite et avancer leur pied gauche vers ...   \n",
       "6   die beiden panneaux stammen aus Léonard von Vi...   \n",
       "7       für bestimmte, sind die Ursache von Problemen   \n",
       "8                                             Fingern   \n",
       "9                               La Vierge aux rochers   \n",
       "10  alle mit einer Rolle, die von den Historien de...   \n",
       "11                      eine Plakette zentraler Tisch   \n",
       "12                        Giovanni Ambrogio de Predis   \n",
       "13                        Giovanni Ambrogio de Predis   \n",
       "14  identifient incorrectly the instrument joué by...   \n",
       "15  rouge jouant of the luth and Ange musicien en ...   \n",
       "16                                              vièle   \n",
       "17                                 Juli dieses Jahres   \n",
       "18                             ne rentred d'Innsbruck   \n",
       "19                               Francesco Napoletano   \n",
       "20                             bois Giacomo del Maino   \n",
       "21                                                  .   \n",
       "22             sculptor on the bois Giacomo del Maino   \n",
       "23                                                  .   \n",
       "24          the confrérie of the Immaculée Conception   \n",
       "25                                     25. April 1483   \n",
       "26                                     25. April 1483   \n",
       "27                                       Unsatisfaits   \n",
       "28                                               1490   \n",
       "29  wahrscheinlich, dass sie eine Art identique be...   \n",
       "\n",
       "                                           t5-small-3  \\\n",
       "0                             Objekte der Spéculation   \n",
       "1                                              certes   \n",
       "2                             Objekte der Spéculation   \n",
       "3                  Unterstützung auf dem rechten Bein   \n",
       "4                                        Rechtes Bein   \n",
       "5                                                weiß   \n",
       "6                             Léonard de Vinciatelier   \n",
       "7   Verwendung von Fingernägeln zur Veredelung von...   \n",
       "8                                         Fingernails   \n",
       "9                             La Vierge zu den Felsen   \n",
       "10            drei Künstler im Auftrag der Bestellung   \n",
       "11                                   Léonard de Vinci   \n",
       "12                                        zwei Tische   \n",
       "13                Léonard de Vinci das zentrale Panel   \n",
       "14                               Ange musicien in rot   \n",
       "15                                                rot   \n",
       "16                                               luth   \n",
       "17                                               1490   \n",
       "18                                            adolfia   \n",
       "19                                  – Herr Präsident!   \n",
       "20               die Tabelle des unberührten Empfangs   \n",
       "21                                            Mailand   \n",
       "22  Künstler werden dann beauftragt, Seeleute zu s...   \n",
       "23  Die Künstler sind dann beauftragt, Seeleute zu...   \n",
       "24             Confrérie des beeindruckenden Empfangs   \n",
       "25                                     25. April 1483   \n",
       "26                                     25. April 1483   \n",
       "27                                        Unzufrieden   \n",
       "28                                               1490   \n",
       "29              sie hatten eine Art identisch gekannt   \n",
       "\n",
       "                                              t5-base  \n",
       "0                                    Verdachtsobjekte  \n",
       "1                                              certes  \n",
       "2                                    Verdachtsobjekte  \n",
       "3                                           threw him  \n",
       "4                                               links  \n",
       "5                                                gris  \n",
       "6                      Werkstatt von Léonard de Vinci  \n",
       "7                                          Beendigung  \n",
       "8                                              Finger  \n",
       "9                                 Vier zu den Räubern  \n",
       "10                                               drei  \n",
       "11                                   Léonard de Vinci  \n",
       "12          viele stilistische und technische Indizes  \n",
       "13  zwei Gemälde der einzigen Hand von Giovanni Am...  \n",
       "14                                           Musicien  \n",
       "15    rot, der Luth spielt, und Ange musicien umkehrt  \n",
       "16                                    Lira da Braccio  \n",
       "17                                               1490  \n",
       "18                                               1490  \n",
       "19  Der Engel des Musikers inspirierte die Figur v...  \n",
       "20           die Retabelle der unberührten Empfängnis  \n",
       "21                                            Mailand  \n",
       "22                                               zwei  \n",
       "23                                             Brüder  \n",
       "24                 Confrère de l'Immaculée Conception  \n",
       "25                                     25. April 1483  \n",
       "26                                     25. April 1483  \n",
       "27                                        unzufrieden  \n",
       "28                                               1490  \n",
       "29                                               1490  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_qa[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "883ab359",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = {'t5-small-1': {'exact_match': 4.736511919698871, 'f1': 10.399724902918903, 'bleu':6.0798},\n",
    " 't5-small-3': {'exact_match': 8.939774153074028, 'f1': 15.967568119984543, 'bleu': 20.9125},\n",
    " 't5-base': {'exact_match': 10.319949811794228, 'f1': 17.164089432704756, 'bleu': 26.0477},\n",
    " 'helsinki': ({'exact_match': 29.76787, 'f1': 56.5588, 'bleu':56.3933})}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "d06d7972",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl0AAAHFCAYAAADIX0yYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAByNElEQVR4nO3dd1hT1/8H8HcIEFYIgkBAERBBREFxo7Vo3bOuOlulVq11Uq3bCo6Ku2itaIe4aq31p9Y9qkKHVHHgbK0DN4iigqKAwvn9wZdbI0MIIWh8v54nT5tzT8793JuY++Gcc09kQggBIiIiIipVRmUdABEREdGbgEkXERERkR4w6SIiIiLSAyZdRERERHrApIuIiIhID5h0EREREekBky4iIiIiPWDSRURERKQHTLqIiIiI9KBESdfKlSshk8kKfERFRekoTO2sW7cO4eHhZRpDfs6dO4fQ0FBcuXJFq9cHBQXByspKt0GVgitXrkAmk2HlypWF1ouKiirwM9S9e3ep3h9//IGBAweiTp06UCgUkMlkWp9DKlju+/b8w9raGjVr1kR4eDiysrI06jdt2hQ1atQotM3Q0NBCvyty38fcfc+fPz/fdubPn8/3/QWZmZkYMmQInJycIJfLUatWrVLdX0Hfqy9770pbft83uZ+75xV0vu7du4devXrBwcEBMpkMnTt31l/wxbRz506EhoYWuf6rei3MT1BQENzc3LR67etwnMa6aCQyMhLe3t55yn18fHTRvNbWrVuHM2fOIDg4uEzjeNG5c+cwbdo0NG3aVOsPlyGaNWsWmjVrplFmZ2cn/f/+/fvx66+/wt/fH9bW1mWe1Bu6ESNGoE+fPgCABw8eYOvWrfj0009x/fp1LFiwQKs2d+/eDZVKlafcycmpRLG+ySIiIrB8+XJ89dVXqFOnTqn/Qfaqfq/mZ+DAgWjTpo1GWUHna8aMGdi8eTNWrFgBDw8P2NralkXIRbJz5058/fXXRU68Xqf3rCReh+PUSdJVo0YN1K1bVxdN0RvM09MTDRs2LHD7559/jpCQEAA5PR6vY9KVlZWFZ8+eQaFQlHUoL1WpUiWN96NNmzY4c+YMfvzxR62Trjp16qB8+fK6CvGV9vjxY1hYWJT6fs6cOQNzc3MMHz5cZ20+efIE5ubmOmuvrFSsWBEVK1bUKCvofJ05cwYeHh7o27evzvb/Op7H1+k7Sh+EEEhPT9fZ+6iXOV3r16+HTCbDkiVLNMpDQkIgl8uxb98+qWzatGlo0KABbG1tYW1tjdq1a+P7779Hfr/LvW7dOgQEBMDKygpWVlaoVasWvv/+ewA5Qx47duzA1atXNYYxCuPm5oYOHTpg+/bt8Pf3h7m5OapVq4bt27cDyBlOrVatGiwtLVG/fn0cPXpU4/VHjx5Fr1694ObmBnNzc7i5uaF37964evWqVGflypV47733AADNmjWT4nq+S3z37t1o3rw5VCoVLCwsUK1aNYSFheWJ9+LFi2jXrh2srKzg4uKCMWPGICMjo9BjBICffvoJrVq1gpOTk3SMEyZMQFpamka93GHMouzn1q1b6NGjB5RKJVQqFXr27InExMSXxlIcRkYl/7hGRESgZs2asLKyglKphLe3NyZNmqRR5+bNmxg8eDBcXFxgamoKZ2dndO/eHbdv35bqXLt2De+//z4cHBygUChQrVo1LFiwANnZ2VKd3OGOuXPnYubMmXB3d4dCocDBgwcB5HxeOnXqBFtbW5iZmcHf3x8bNmwoNP6nT5/CwcEBH3zwQZ5tDx48gLm5OUaPHg0AyM7OxsyZM1G1alWYm5vDxsYGfn5+WLRokdbnT6VSwcTEROvX68vly5fRq1cvODs7Q6FQwNHREc2bN0dcXJxGvcK+Q3KtWLECNWvWhJmZGWxtbdGlSxf8/fffGnVy/62cPn0arVq1glKpRPPmzQHkDGfNnDkT3t7eUCgUsLe3x4cffog7d+5otHHgwAE0bdoUdnZ2MDc3R6VKldCtWzc8fvy4wOOUyWT47rvv8OTJkzzfJenp6Zg4cSLc3d1hamqKChUqYNiwYXjw4IFGG7nfe5s2bYK/vz/MzMwwbdq0fPdX1O/VhQsXwt3dHVZWVggICMBff/2Vp442n/9cRf2+eXF4saDzJZPJ8Ouvv+Lvv//OMz2mqO9fYecxMTERH3/8MSpWrAhTU1O4u7tj2rRpePbsmfT654dnCzt/QUFB+Prrr6XjeXGI/kWFvWeFfUelp6djzJgxqFWrFlQqFWxtbREQEIBffvklzz5kMhmGDx+ONWvWoFq1arCwsEDNmjWla2euO3fuSN+tueeycePG+PXXXwt6qwEAX3/9Nd5++204ODjA0tISvr6+mDt3Lp4+fVqk4wRyho+HDh2KChUqwNTUFJUrV8bkyZPzXMtyj2XZsmWoVq0aFAoFVq1aBaBo14+X0UlPV25m/GLgcrkcANCrVy9ER0djzJgxaNiwIerWrYsDBw5g5syZmDRpElq2bCm97sqVK/j4449RqVIlAMBff/2FESNG4ObNm5g6dapUb+rUqZgxYwa6du2KMWPGQKVS4cyZM1KCs3TpUgwePBiXLl3C5s2bi3wsJ0+exMSJEzF58mSoVCpMmzYNXbt2xcSJE7F//37MmjULMpkM48ePR4cOHRAfHy9lwFeuXEHVqlXRq1cv2NraIiEhAREREahXrx7OnTuH8uXLo3379pg1axYmTZqEr7/+GrVr1wYAeHh4AAC+//57DBo0CIGBgVi2bBkcHBzw77//4syZMxpxPn36FJ06dcJHH32EMWPG4LfffsOMGTOgUqk0zlN+Lly4gHbt2iE4OBiWlpb4559/MGfOHBw5cgQHDhwo9n6ePHmCFi1a4NatWwgLC4OXlxd27NiBnj17Fvm8AzlJwoufI2NjnXxEAeQk/0OHDsWIESMwf/58GBkZ4eLFizh37pxU5+bNm6hXrx6ePn2KSZMmwc/PD8nJydizZw/u378PR0dH3LlzB40aNUJmZiZmzJgBNzc3bN++HZ999hkuXbqEpUuXaux38eLF8PLywvz582FtbQ1PT08cPHgQbdq0QYMGDbBs2TKoVCqsX78ePXv2xOPHjxEUFJTvMZiYmOD999/HsmXL8PXXX8Pa2lra9uOPPyI9PR0ffvghAGDu3LkIDQ3FlClT8Pbbb+Pp06f4559/8lx0C/L8+5GSkoJffvkFu3fvxvjx44tx1jW97LtCV9q1a4esrCzMnTsXlSpVwt27d3Ho0CGNY3/ZdwgAhIWFYdKkSejduzfCwsKQnJyM0NBQBAQEIDY2Fp6enlLdzMxMdOrUCR9//DEmTJiAZ8+eITs7G++++y5+//13jBs3Do0aNcLVq1cREhKCpk2b4ujRozA3N8eVK1fQvn17NGnSBCtWrICNjQ1u3ryJ3bt3IzMzs8Aes5iYGMyYMQMHDx6U/u16eHhACIHOnTtj//79mDhxIpo0aYJTp04hJCQEMTExiImJ0ejJOH78OP7++29MmTIF7u7usLS0zHd/Rfle/frrr+Ht7S3Nrfn888/Rrl07xMfHS0PL2n7+gZJ93+R3vtzd3RETE4OhQ4ciJSUFP/zwA4Cc6TFFff8KO4+JiYmoX78+jIyMMHXqVHh4eCAmJgYzZ87ElStXEBkZWazz9/nnnyMtLQ0bN25ETEyM9LqChuiL8p7l9x2VkZGBe/fu4bPPPkOFChWQmZmJX3/9FV27dkVkZCT69eun0caOHTsQGxuL6dOnw8rKCnPnzkWXLl1w/vx5VK5cGQDwwQcf4Pjx4/jiiy/g5eWFBw8e4Pjx40hOTi70fbt06RL69Okj/QFx8uRJfPHFF/jnn3+wYsWKlx5neno6mjVrhkuXLmHatGnw8/PD77//jrCwMMTFxWHHjh0a9bds2YLff/8dU6dOhVqthoODQ5GuH0UiSiAyMlIAyPchl8s16qanpwt/f3/h7u4uzp07JxwdHUVgYKB49uxZge1nZWWJp0+fiunTpws7OzuRnZ0thBDi8uXLQi6Xi759+xYaX/v27YWrq2uRj8fV1VWYm5uLGzduSGVxcXECgHBychJpaWlS+ZYtWwQAsXXr1gLbe/bsmXj06JGwtLQUixYtksp//vlnAUAcPHhQo/7Dhw+FtbW1eOutt6RjzU///v0FALFhwwaN8nbt2omqVasW9XCFEEJkZ2eLp0+fiujoaAFAnDx5stj7iYiIEADEL7/8olFv0KBBAoCIjIwsNIaDBw8W+Dm6cOFCvq+ZN2+eACDi4+OLfKzDhw8XNjY2hdYZMGCAMDExEefOnSuwzoQJEwQAcfjwYY3yTz75RMhkMnH+/HkhhBDx8fECgPDw8BCZmZkadb29vYW/v794+vSpRnmHDh2Ek5OTyMrKKnD/p06dEgDEN998o1Fev359UadOHY22atWqVejx5ic37vweQUFBef7NBgYGiurVqxfaZkhISIFtenh45Nn3vHnz8m2nKO/73bt3BQARHh5eYJ2ifIfcv39fmJubi3bt2mmUX7t2TSgUCtGnTx+pLPffyooVKzTq/vjjjwKA+L//+z+N8tjYWAFALF26VAghxMaNGwUAERcXV2A8Benfv7+wtLTUKNu9e7cAIObOnatR/tNPP+X57Li6ugq5XC59bl+moO/V3PfO19dX4zNy5MgRAUD8+OOPUllJPv/F+b7J/dw9L7/zJUT+n+Oivn9CFHweP/74Y2FlZSWuXr2qUT5//nwBQJw9e1YIUbzzN2zYsDzHVZiXvWf5fUe96NmzZ+Lp06fio48+Ev7+/hrbAAhHR0eRmpoqlSUmJgojIyMRFhYmlVlZWYng4OBC99O/f/9Cr9u5ecHq1auFXC4X9+7de+lxLlu2LN9r2Zw5cwQAsXfvXo1jUalUGu0KUbTrR1HoZHhx9erViI2N1XgcPnxYo45CocCGDRuQnJyM2rVrQwiBH3/8Mc9fuAcOHECLFi2gUqkgl8thYmKCqVOnIjk5GUlJSQCAffv2ISsrC8OGDdNF+Bpq1aqFChUqSM+rVasGIKfr8vm/NnPLn/+r+NGjRxg/fjyqVKkCY2NjGBsbw8rKCmlpaXmGI/Jz6NAhpKamYujQoS8dCpXJZOjYsaNGmZ+fn0Y8Bbl8+TL69OkDtVotnePAwEAAyBNnUfZz8OBBKJVKdOrUSaNe7iTsopozZ06ez5GLi0ux2gD+603JfeQO+dWvXx8PHjxA79698csvv+Du3bt5Xrtr1y40a9ZMen/zc+DAAfj4+KB+/foa5UFBQRBC5Okt7NSpk8aQ3MWLF/HPP/9Ic0eej7Vdu3ZISEjA+fPnC9y/r68v6tSpo/EX8t9//40jR45gwIABUln9+vVx8uRJDB06FHv27EFqamqBbeZn1KhR0vtw8OBBzJo1Cxs2bEDv3r2L1c7zfv311zzv8ZYtW7RuLz+2trbw8PDAvHnzsHDhQpw4cUJj2Bco2ndITEwMnjx5kqfXxcXFBe+88w7279+f5zXdunXTeL59+3bY2NigY8eOGu9zrVq1oFarpSGsWrVqwdTUFIMHD8aqVatw+fJl7Q7+f3I/gy/G/t5778HS0jJP7H5+fvDy8irRPnO1b99e43vdz88PwH/flSX9/Ovq+6Yoivr+5crvPG7fvh3NmjWDs7OzRhtt27YFAERHR2vUf9n5Kw0vfkfl+vnnn9G4cWNYWVnB2NgYJiYm+P777/O9njVr1gxKpVJ67ujoCAcHB42469evj5UrV2LmzJn466+/NIYHC3PixAl06tQJdnZ20jWrX79+yMrKwr///vvS1x84cACWlpYad8MD//37ePHfwzvvvINy5cpplBXl+lEUOkm6qlWrhrp162o86tSpk6delSpV0KRJE6Snp6Nv3755ukOPHDmCVq1aAQC+/fZb/Pnnn4iNjcXkyZMB5HQrA5DG0l+cIKkLL96xYmpqWmh5enq6VNanTx8sWbIEAwcOxJ49e3DkyBHExsbC3t5eir0wxTkuCwsLmJmZaZQpFAqNePLz6NEjNGnSBIcPH8bMmTMRFRWF2NhYbNq0CQDyxFmU/SQnJ8PR0THPvtRq9UuP43mVK1fO8znSZjKnh4cHTExMpMf06dMB5HRtr1ixAlevXkW3bt3g4OCABg0aaMwpvHPnzkvPf3Jycr5d+c7OztL2571YN3du2GeffaYRp4mJCYYOHQoAL/0HPWDAAMTExOCff/4BkHMHsUKh0EiIJk6ciPnz5+Ovv/5C27ZtYWdnh+bNm+eZi1iQihUrSu9D06ZNMXHiRHz++ef4+eefsWfPniK18aKaNWvmeY+fX24idzj5xWUpcuUOTRY2r0wmk2H//v1o3bo15s6di9q1a8Pe3h4jR47Ew4cPARTt31ru+1jQe/3i+2xhYaEx3AvkvNcPHjyAqalpnvc6MTFRep89PDzw66+/wsHBAcOGDYOHhwc8PDy0nn+XnJwMY2Nj2Nvba5TLZDKo1eqXfkZL4vk7jgFI/4Zzv1tK+vnX1fdNURT1/cuV33m8ffs2tm3bluf11atXB5D3WF92/kpDfnFv2rQJPXr0QIUKFbB27VrExMQgNjYWAwYMyPc682LcQE7sz8f9008/oX///vjuu+8QEBAAW1tb9OvXr9D5v9euXUOTJk1w8+ZNLFq0CL///jtiY2OleW1FOS/JyclQq9V5OjMcHBxgbGxcpH8PRbl+FIXuJswUwXfffYcdO3agfv36WLJkCXr27IkGDRpI29evXw8TExNs375d40L/4l/CuV8kN27c0KonpDSkpKRg+/btCAkJwYQJE6Ty3HHxonj+uErLgQMHcOvWLURFRUm9WwCKPM8nP3Z2djhy5Eiecl1PpC+qbdu2aUyOzE2GAODDDz/Ehx9+iLS0NPz2228ICQlBhw4d8O+//8LV1RX29vYvPf92dnZISEjIU37r1i0AyHN33ov/0HO3T5w4EV27ds13H1WrVi00ht69e2P06NFYuXIlvvjiC6xZswadO3fW+OvM2NgYo0ePxujRo/HgwQP8+uuvmDRpElq3bo3r169rdWdd7l/dJ0+eROvWrYv9+pcpX7485HI5bt68me/2mzdvQi6X5/sF/zxXV1dpQvy///6LDRs2IDQ0FJmZmVi2bFmRvkNy91HQe/2y9zn3eOzs7LB79+589/F8z0CTJk3QpEkTZGVl4ejRo/jqq68QHBwMR0dH9OrVq9DjzS/2Z8+e4c6dOxqJlxACiYmJqFev3ktjLy0l/fzr8/umOO8fUPBnwM/PD1988UW+bTz//VRW8ot77dq1cHd3x08//aSxvSg3axWkfPnyCA8PR3h4OK5du4atW7diwoQJSEpKKvAcb9myBWlpadi0aRNcXV2l8hdviimMnZ0dDh8+DCGExrEkJSXh2bNnRfq3DLz8+lEUeluR/vTp0xg5ciT69euH33//HX5+fujZsyfu378v1ZHJZDA2NtboWn3y5AnWrFmj0VarVq0gl8sRERFR6D5fzLJLk0wmgxAiT8/Md999l+ev9oL+cmnUqBFUKhWWLVuW792auorz+RhyLV++XOs2mzVrhocPH2Lr1q0a5evWrdO6zZLw9fXV6EnJ70vN0tISbdu2xeTJk5GZmYmzZ88CANq2bYuDBw8WOrzRvHlznDt3DsePH9coX716NWQyWZ61xl5UtWpVeHp64uTJk3l6fXIfL36Zv6hcuXLo3LkzVq9eje3btyMxMVFjaPFFNjY26N69O4YNG4Z79+5pvbho7hedg4ODVq9/GTMzMzRu3Bhbt27N89d0eno6tm7dirfeeitP72thvLy8MGXKFPj6+krvWVG+QwICAmBubo61a9dqlN+4cQMHDhyQ7k4sTIcOHZCcnIysrKx83+f8kgu5XI4GDRpIf8m/+DkritzYXoz9//7v/5CWllak2AtS0u/Vkn7+9fl9o837l18buctR5NeGNklXcXu/tHnPZDIZTE1NNRKQxMTEfO9e1EalSpUwfPhwtGzZstDPeH7XLCEEvv322zx1CzrO5s2b49GjR3k6cFavXi1tL46Crh9FoZOerjNnzuS5IwnI6TK3t7dHWloaevToAXd3dyxduhSmpqbYsGEDateujQ8//FA6Ee3bt8fChQvRp08fDB48GMnJyZg/f36eBMHNzQ2TJk3CjBkz8OTJE/Tu3RsqlQrnzp3D3bt3pdt0fX19sWnTJkRERKBOnTowMjIqtfXErK2t8fbbb2PevHkoX7483NzcEB0dje+//x42NjYadXOHU7755hsolUqYmZnB3d0ddnZ2WLBgAQYOHIgWLVpg0KBBcHR0xMWLF3Hy5Mk8S25oo1GjRihXrhyGDBmCkJAQmJiY4IcffsDJkye1brNfv3748ssv0a9fP3zxxRfw9PTEzp07tR6CKsidO3ek+Q+nT58GkDMHy97eHvb29ho9d/kZNGgQzM3N0bhxYzg5OSExMRFhYWFQqVTSX/7Tp0/Hrl278Pbbb2PSpEnw9fXFgwcPsHv3bowePRre3t749NNPsXr1arRv3x7Tp0+Hq6srduzYgaVLl+KTTz4p0tyY5cuXo23btmjdujWCgoJQoUIF3Lt3D3///TeOHz+On3/++aVtDBgwAD/99BOGDx+OihUrokWLFhrbO3bsKK2hZ29vj6tXryI8PByurq4ad90V5Nq1a9Kt6mlpaYiJiUFYWBhcXV3z9FCkpqZi48aNedp48X05duxYvouj+vj4SENzs2fPRrNmzRAQEIDg4GBUqlQJ165dQ3h4OG7fvo3169cXGvepU6cwfPhwvPfee/D09ISpqSkOHDiAU6dOSb3QRfkOsbGxweeff45JkyahX79+6N27N5KTkzFt2jSYmZlJa8YVplevXvjhhx/Qrl07jBo1CvXr14eJiQlu3LiBgwcP4t1330WXLl2wbNkyHDhwAO3bt0elSpWQnp4u3ZX14vtaFC1btkTr1q0xfvx4pKamonHjxtLdi/7+/vkuOVJUuvheLcnnX1/fN0DR37/CTJ8+Hfv27UOjRo0wcuRIVK1aFenp6bhy5Qp27tyJZcuWFXuqjK+vL4CcebBt27aFXC6Hn5+fNO0lv/rFfc9yl78YOnQounfvjuvXr2PGjBlwcnLChQsXihUvkDMa1KxZM/Tp0wfe3t5QKpWIjY3F7t27C+zxBHI+y6ampujduzfGjRuH9PR0REREaHTYvOw4+/Xrh6+//hr9+/fHlStX4Ovriz/++AOzZs1Cu3btivRvrCjXjyIpySz8wu5eBCC+/fZbIYQQ77//vrCwsJDu0siVexffl19+KZWtWLFCVK1aVSgUClG5cmURFhYmvv/++3zvWFq9erWoV6+eMDMzE1ZWVsLf31/jzpV79+6J7t27CxsbGyGTyV56t4erq6to3759nnIAYtiwYRpl+d1ldePGDdGtWzdRrlw5oVQqRZs2bcSZM2eEq6ur6N+/v8brw8PDhbu7u5DL5XnuuNm5c6cIDAwUlpaWwsLCQvj4+Ig5c+ZI2wu6+ya/O3Xyc+jQIREQECAsLCyEvb29GDhwoDh+/HieOIqzn9xjt7KyEkqlUnTr1k0cOnSoWHcv/vzzz0Wql98jMDDwpce9atUq0axZM+Ho6ChMTU2Fs7Oz6NGjhzh16pRGvevXr4sBAwYItVotTExMpHq3b9+W6ly9elX06dNH2NnZCRMTE1G1alUxb948jbuuXnYn3smTJ0WPHj2Eg4ODMDExEWq1Wrzzzjti2bJlLz0WIXLu4nFxcREAxOTJk/NsX7BggWjUqJEoX768MDU1FZUqVRIfffSRuHLlSqHt5nf3opmZmfDy8hLBwcEiISFBo35gYOBL35fC7l4EIPbt26fR5tGjR0WXLl1E+fLlhVwuF+XLlxddunQRx44de+l5uX37tggKChLe3t7C0tJSWFlZCT8/P/Hll1/mufPyZd8hQgjx3XffCT8/P2FqaipUKpV4991383yXFfRvRQghnj59KubPny9q1qwp7cfb21t8/PHH0t25MTExokuXLsLV1VUoFAphZ2cnAgMDC707+mX7fvLkiRg/frxwdXUVJiYmwsnJSXzyySfi/v37GvUK+t4rSEHfq4V93gGIkJAQjbKSfP6L+n1T0rsXhSja+ydE4efxzp07YuTIkcLd3V2YmJgIW1tbUadOHTF58mTx6NEjIUTxzl9GRoYYOHCgsLe3l96Dwu7o1eY9E0KI2bNnCzc3N6FQKES1atXEt99+m+85ze8amXtOcq996enpYsiQIcLPz09YW1sLc3NzUbVqVRESEqKxMkB+dy9u27ZNOv8VKlQQY8eOFbt27cqzCkBh1/zk5GQxZMgQ4eTkJIyNjYWrq6uYOHGiSE9PL9KxFPX68TKy/+2EiIiIiEqR3uZ0EREREb3JmHQRERER6QGTLiIiIiI9YNJFREREpAdMuoiIiIj0gEkXERERkR7o9WeAykJ2djZu3boFpVKp15+6ICIiIu0JIfDw4UM4OzvDyMgw+ogMPum6devWK/P7jERERFQ8169fL/aq/a8qg0+6cn/D6/r169LPjBAREdGrLTU1FS4uLi/9LdrXicEnXblDitbW1ky6iIiIXjOGNDXIMAZJiYiIXgFNmzZFcHCw1q8PDQ1FrVq1dF43l5ubG8LDwwvcXtL4qXAG39NFRERkiD777DOMGDFCp21u2rQJJiYmOm2T/sOki4iI6DVkZWUFKysrnbZpa2ur0/ZIE5Ou/8nKysLTp0/LOgx6BZmYmEAul5d1GET0msjOzsa4cePw3XffwdTUFEOGDEFoaCgAICUlBWPHjsWWLVuQnp6OunXr4ssvv0TNmjXzbSsqKgrjxo3D2bNnYWJigurVq2PdunVwdXVFaGgotmzZgri4OABAUFAQHjx4gLfeegsLFixAZmYmevXqhfDw8AJ7ryIjIxEcHIyNGzeiZcuWaNq0KWrVqlXoECRp741PuoQQSExMxIMHD8o6FHqF2djYQK1WG9SETiIqHatWrcLo0aNx+PBhxMTEICgoCI0bN0aLFi3Qvn172NraYufOnVCpVFi+fDmaN2+Of//9N08v07Nnz9C5c2cMGjQIP/74IzIzM3HkyJFCv4cOHjwIJycnHDx4EBcvXkTPnj1Rq1YtDBo0KE/d+fPnIywsDHv27EHDhg11fh4orzc+6cpNuBwcHGBhYcGLKmkQQuDx48dISkoCADg5OZVxRET0qvPz80NISAgAwNPTE0uWLMH+/fshl8tx+vRpJCUlQaFQAMhJfLZs2YKNGzdi8ODBGu2kpqYiJSUFHTp0gIeHBwCgWrVqhe67XLlyWLJkCeRyOby9vdG+fXvs378/T9I1ceJErFq1ClFRUfD19dXVodNLvNFJV1ZWlpRw2dnZlXU49IoyNzcHACQlJcHBwYFDjURUKD8/P43nTk5OSEpKwrFjx/Do0aM815snT57g0qVLedqxtbVFUFAQWrdujZYtW6JFixbo0aNHoX/8Va9eXeM7ysnJCadPn9aos2DBAqSlpeHo0aOoXLmyNodIWnqjl4zIncNlYWFRxpHQqy73M8J5f0T0Mi/On5LJZMjOzkZ2djacnJwQFxen8Th//jzGjh2bb1uRkZGIiYlBo0aN8NNPP8HLywt//fVXsff9vCZNmiArKwsbNmzQ8ghJW290T1cuDinSy/AzQkQlVbt2bSQmJsLY2Bhubm5Ffp2/vz/8/f0xceJEBAQEYN26dSWag1W/fn2MGDECrVu3hlwuLzDhI91j0kVERKQHLVq0QEBAADp37ow5c+agatWquHXrFnbu3InOnTujbt26GvXj4+PxzTffoFOnTnB2dsb58+fx77//ol+/fiWOJSAgALt27UKbNm1gbGyMTz/9tPiNZGcBVw8Bj24DVo6AayPAiNMvCsOki944UVFRaNasGe7fvw8bG5uyDoeI3hAymQw7d+7E5MmTMWDAANy5cwdqtRpvv/02HB0d89S3sLDAP//8g1WrViE5ORlOTk4YPnw4Pv74Y53E07hxY+zYsQPt2rWDXC7HyJEji/7ic1uB3eOB1Fv/lVk7A23mAD6ddBKfIZIJIURZB1GaUlNToVKpkJKSkue3F9PT0xEfHw93d3eYmZmVUYSvJ5lMhs2bN6Nz585lGsfKlSsRHBxcrCU/tEm6+FkhIvqfc1uBDf0AvJg+/G8aRo/VOkm8Crt+v67e6In0upKVLRBzKRm/xN1EzKVkZGUbdB5LRERvquysnB6uPAkX/ivbPSGnHuXBpKuEdp9JwFtzDqD3t39h1Po49P72L7w15wB2n0ko1f0KITB37lxUrlwZ5ubmqFmzJjZu3AghBFq0aIE2bdogtxPzwYMHqFSpEiZPngwgZ6mMjz76CO7u7jA3N0fVqlWxaNGiPPtYsWIFqlevDoVCIXVrA5AmgHbp0gUymaxIE0Jzf5h1xYoVqFSpEqysrPDJJ58gKysLc+fOhVqthoODA7744guN1y1cuBC+vr6wtLSEi4sLhg4dikePHgHI6bH68MMPkZKSAplMBplMJq36nJGRgXHjxsHFxQUKhQKenp74/vvvNdo+duwY6tatCwsLCzRq1Ajnz58v8vknInojXT2kOaSYhwBSb+bUozw4p6sEdp9JwCdrj+fJ9xNT0vHJ2uOIeL822tQoncU0p0yZgk2bNiEiIgKenp747bff8P7778Pe3h6rVq2Cr68vFi9ejFGjRmHIkCFwdHSUEpLs7GxUrFgRGzZsQPny5XHo0CEMHjwYTk5O6NGjBwAgIiICo0ePxuzZs9G2bVukpKTgzz//BADExsbCwcEBkZGRaNOmTZHXrbp06RJ27dqF3bt349KlS+jevTvi4+Ph5eWF6OhoHDp0CAMGDEDz5s2lO3OMjIywePFiuLm5IT4+HkOHDsW4ceOwdOlSNGrUCOHh4Zg6daqUMOX+Dlm/fv0QExODxYsXo2bNmoiPj8fdu3c14pk8eTIWLFgAe3t7DBkyBAMGDJCOkYiI8vHotm7rvWGYdGkpK1tg2rZzBXawygBM23YOLX3UkBvpdrmBtLQ0LFy4EAcOHEBAQAAAoHLlyvjjjz+wfPlyrFu3DsuXL8cHH3yA27dvY9u2bThx4oS0fouJiQmmTZsmtefu7o5Dhw5hw4YNUtI1c+ZMjBkzBqNGjZLq1atXDwBgb28P4L+fximq7OxsrFixAkqlEj4+PmjWrBnOnz+PnTt3wsjICFWrVsWcOXMQFRUlJV3BwcEacc6YMQOffPIJli5dClNTU6hUKshkMo04/v33X2zYsAH79u1DixYtpPPzoi+++AKBgYEAgAkTJqB9+/ZIT0/nnC0iooJY5Z3wX6J6bxgmXVo6En8PCSnpBW4XABJS0nEk/h4CPHS72v25c+eQnp6Oli1bapRnZmbC398fAPDee+9h8+bNCAsLQ0REBLy8vDTqLlu2DN999x2uXr2KJ0+eIDMzE7Vq1QKQs/L6rVu30Lx5c53G7ebmBqVSKT13dHSEXC6HkZGRRlnuT+4AOb8jNmvWLJw7dw6pqal49uwZ0tPTkZaWBktLy3z3ExcXB7lcLiVUBXl+1ejcFZ6TkpJQqVIlrY6PiMjguTbKuUsxNQH5z+uS5Wx3baTvyF4LTLq0lPSw4IRLm3rFkbu68I4dO1ChQgWNbbm/5/X48WMcO3YMcrkcFy5c0KizYcMGfPrpp1iwYAECAgKgVCoxb948HD58GMB/P3uja/mtlFzY6slXr15Fu3btMGTIEMyYMQO2trb4448/8NFHHxW6MnxR439+37mLn764cjMRET3HSJ6zLMSGfsgZ03k+8frfqE6b2VyvqwCcSK8lB2XRhqCKWq84fHx8oFAocO3aNVSpUkXj4eLiAgAYM2YMjIyMsGvXLixevBgHDhyQXv/777+jUaNGGDp0KPz9/VGlShWN3/1SKpVwc3PD/v37C4zBxMQEWVmle3fK0aNH8ezZMyxYsAANGzaEl5cXbt3SnMBpamqaJw5fX19kZ2cjOjq6VOMjInoj+XTKWRbC+oU5y9bOOlsuwlCxp0tL9d1t4aQyQ2JKekEdrFCrzFDf3Vbn+1Yqlfjss8/w6aefIjs7G2+99RZSU1Nx6NAhWFlZoXz58lixYgViYmJQu3ZtTJgwAf3798epU6dQrlw5VKlSBatXr8aePXvg7u6ONWvWIDY2Fu7u7tI+QkNDMWTIEDg4OKBt27Z4+PAh/vzzT4wYMQIApKSscePGUCgUKFeunM6P08PDA8+ePcNXX32Fjh074s8//8SyZcs06ri5ueHRo0fYv38/atasCQsLC7i5uaF///4YMGCANJH+6tWrSEpKkuasERFRCfh0Arzbc0X6YmJPl5bkRjKEdPQBIHWoSnKfh3T00fkk+lwzZszA1KlTERYWhmrVqqF169bYtm0b3Nzc8NFHHyE0NBS1a9fOiSMkBM7OzhgyZAgAYMiQIejatSt69uyJBg0aIDk5GUOHDtVov3///ggPD8fSpUtRvXp1dOjQQWOYcsGCBdi3bx9cXFykeWS6VqtWLSxcuBBz5sxBjRo18MMPPyAsLEyjTqNGjTBkyBD07NkT9vb2mDt3LoCcuy+7d++OoUOHwtvbG4MGDUJaWlqpxElE9EYykgPuTQDf7jn/ZcL1UlyRvoSrjO8+k4Bp285pTKp3UpkhpKNPqS0XQfrHFemJiPTLEFek5/BiCbWp4YSWPmocib+HpIfpcFDmDCmWVg8XERERvZ6YdOmA3Eim82UhXjfVq1fH1atX8922fPly9O3bV88RERERvVqYdJFO7Ny5s8BlHBwduUgeERERky7SCVdX17IOgYiI6JXGuxeJiIiI9IBJFxEREZEeMOkiIiIi0gMmXURERER6wKSLiIiISA+YdL2mhBAYPHgwbG1tIZPJEBcXV9YhERERUSGYdL2mdu/ejZUrV2L79u1ISEhAamoqOnbsCGdnZ8hkMmzZsqWsQyQiIqLnMOnShewsIP534PTGnP9mZ5X6Li9dugQnJyc0atQIarUaaWlpqFmzJpYsWVLq+yYiIqLi4+KoJXVuK7B7PJB6678ya2egzRzAp1Op7DIoKAirVq0CAMhkMri6uuLKlSto27ZtqeyPiIiISo49XSVxbiuwoZ9mwgUAqQk55ee2lspuFy1ahOnTp6NixYpISEhAbGxsqeyHiIiIdIdJl7ays3J6uCDy2fi/st0TSmWoUaVSQalUQi6XQ61Ww97eXuf7ICIiIt1i0qWtq4fy9nBpEEDqzZx6RERE9MZj0qWtR7d1W4+IiIgMGpMubVk56rYeERERGTTevagt10Y5dymmJiD/eV2ynO2ujfQSzqNHj3Dx4kXpeXx8POLi4mBra4tKlSrpJQYiIiIqGHu6tGUkz1kWAgAge2Hj/563mZ1TTw+OHj0Kf39/+Pv7AwBGjx4Nf39/TJ06VS/7JyIiosLJhBD5ddMYjNTUVKhUKqSkpMDa2lpjW3p6OuLj4+Hu7g4zMzPtdpDvOl0VchKuUlqni/RPJ58VIiIqssKu368rDi+WlE8nwLt9zl2Kj27nzOFybaS3Hi4iIiJ6PTDp0gUjOeDepKyjICIiolcY53QRERER6QGTLiIiIiI9YNIFwMDvJSAd4GeEiIhK6o1OukxMTAAAjx8/LuNI6FWX+xnJ/cwQEREV1xs9kV4ul8PGxgZJSUkAAAsLC8hkL665RW8yIQQeP36MpKQk2NjYQC7nXalERKSdNzrpAgC1Wg0AUuJFlB8bGxvps0JERKSNNz7pkslkcHJygoODA54+fVrW4dAryMTEhD1cRERUYmWadIWGhmLatGkaZY6OjkhMTASQM7Qzbdo0fPPNN7h//z4aNGiAr7/+GtWrV9d5LHK5nBdWIiIiKjVlPpG+evXqSEhIkB6nT5+Wts2dOxcLFy7EkiVLEBsbC7VajZYtW+Lhw4dlGDERERFR8ZV50mVsbAy1Wi097O3tAeT0coWHh2Py5Mno2rUratSogVWrVuHx48dYt25dGUdNREREVDxlnnRduHABzs7OcHd3R69evXD58mUAQHx8PBITE9GqVSuprkKhQGBgIA4dOlRgexkZGUhNTdV4EBEREZW1Mk26GjRogNWrV2PPnj349ttvkZiYiEaNGiE5OVma1+Xo6KjxmufnfOUnLCwMKpVKeri4uJTqMRAREREVRZkmXW3btkW3bt3g6+uLFi1aYMeOHQCAVatWSXVeXDdLCFHoWloTJ05ESkqK9Lh+/XrpBE9ERERUDGU+vPg8S0tL+Pr64sKFC9KaSC/2aiUlJeXp/XqeQqGAtbW1xoOIiIiorL1SSVdGRgb+/vtvODk5wd3dHWq1Gvv27ZO2Z2ZmIjo6Go0aNSrDKImIiIiKr0zX6frss8/QsWNHVKpUCUlJSZg5cyZSU1PRv39/yGQyBAcHY9asWfD09ISnpydmzZoFCwsL9OnTpyzDJiIiIiq2Mk26bty4gd69e+Pu3buwt7dHw4YN8ddff8HV1RUAMG7cODx58gRDhw6VFkfdu3cvlEplWYZNREREVGwyIYQo6yBKU2pqKlQqFVJSUji/i4iI6DVhiNfvV2pOFxEREZGhYtJFREREpAdMuoiIiIj0gEkXERERkR4w6SIiIiLSAyZdRERERHrApIuIiIhID5h0EREREekBky4iIiIiPWDSRURERKQHTLqIiIiI9IBJFxEREZEeMOkiIiIi0gMmXURERER6wKSLiIiISA+YdBERERHpAZMuIiIiIj1g0kVERESkB0y6iIiIiPSASRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRERER6QGTLiIiIiI9YNJFREREpAdMuoiIiIj0gEkXERERkR4w6SIiIiLSAyZdRERERHrApIuIiIhID5h0EREREekBky4iIiIiPWDSRURERKQHTLqIiIiI9IBJFxEREZEeMOkiIiIi0gMmXURERER6wKSLiIiISA+YdBERERHpAZMuIiIiIj1g0kVERESkB0y6iIiIiPSASRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRERER6QGTLiIiIiI9YNJFREREpAdMuoiIiIj0gEkXERERkR4w6SIiIiLSAyZdRERERHrwyiRdYWFhkMlkCA4OlsqEEAgNDYWzszPMzc3RtGlTnD17tuyCJCIiItLSK5F0xcbG4ptvvoGfn59G+dy5c7Fw4UIsWbIEsbGxUKvVaNmyJR4+fFhGkRIRERFpp8yTrkePHqFv37749ttvUa5cOalcCIHw8HBMnjwZXbt2RY0aNbBq1So8fvwY69atK8OIiYiIiIqvzJOuYcOGoX379mjRooVGeXx8PBITE9GqVSupTKFQIDAwEIcOHSqwvYyMDKSmpmo8iIiIiMqacVnufP369Th+/DhiY2PzbEtMTAQAODo6apQ7Ojri6tWrBbYZFhaGadOm6TZQIiIiohIqs56u69evY9SoUVi7di3MzMwKrCeTyTSeCyHylD1v4sSJSElJkR7Xr1/XWcxERERE2iqznq5jx44hKSkJderUkcqysrLw22+/YcmSJTh//jyAnB4vJycnqU5SUlKe3q/nKRQKKBSK0guciIiISAtl1tPVvHlznD59GnFxcdKjbt266Nu3L+Li4lC5cmWo1Wrs27dPek1mZiaio6PRqFGjsgqbiIiISCtl1tOlVCpRo0YNjTJLS0vY2dlJ5cHBwZg1axY8PT3h6emJWbNmwcLCAn369CmLkImIiIi0VqYT6V9m3LhxePLkCYYOHYr79++jQYMG2Lt3L5RKZVmHRkRERFQsMiGEKOsgSlNqaipUKhVSUlJgbW1d1uEQERFRERji9bvM1+kiIiIiehPoNOm6f/8+Vq9ercsmiYiIiAyCTpOua9eu4cMPP9Rlk0REREQGoVgT6V/2kzr8IWoiIiKi/BUr6bKxsSl0NfiXrRZPRERE9KYqVtKlVCoxefJkNGjQIN/tFy5cwMcff6yTwIiIiIgMSbGSrtq1awMAAgMD891uY2MDA1+BgoiIiEgrxZpI36dPn0J/nFqtViMkJKTEQREREREZGi6OSkRERK8cQ7x+c3FUIiIiIj0oVtJVqVIlJCcnS8+XLFny0mUkiIiIiKiYSdeNGzeQlZUlPZ80aRLu3r2r86CIiIiIDE2JhhcNfDoYERERkc5wThcRERGRHhRrnS4A+O6772BlZQUAePbsGVauXIny5ctr1Bk5cqRuoiMiIiIyEMVaMsLNze2lP/Mjk8lw+fLlEgemK4Z4yykREZGhM8Trd7F6uq5cuVJKYRAREREZtmKvSL9hwwY8fPiwtOIhIiIiMkjFSrq8vLwwZ84c2Nvbo1WrVvj6669x/fr10oqNiIiIyGBo9TNAN27cwNatW/HLL78gOjoaPj4+6NSpE9599134+/uXRpxaM8QxYSIiIkNniNfvEv/24sOHD7Fr1y788ssv2LVrF5RKJTp27IhPPvkE1atX11WcWjPEN42IiMjQGeL1u8TrdCmVSvTo0QM//PAD7ty5gxUrVkAulyMmJkYX8REREREZhBL3dL3qDDFTJiIiMnSGeP3Wqqfr9u3b+OCDD+Ds7AxjY2PI5XKNBxERERFpKvaK9AAQFBSEa9eu4fPPP4eTk9NLF0wlIiIietNplXT98ccf+P3331GrVi0dh0NERERkmLQaXnRxcYGBTwUjIiIi0imtkq7w8HBMmDCBPwtEREREVERFHl4sV66cxtyttLQ0eHh4wMLCAiYmJhp17927p7sIiYiIiAxAkZOu8PDwUgyDiIiIyLAVOenq379/acZBREREZNC0mtO1c+dO7NmzJ0/53r17sWvXrhIHRURERGRotEq6JkyYgKysrDzl2dnZmDBhQomDIiIiIjI0WiVdFy5cgI+PT55yb29vXLx4scRBERERERkarZIulUqFy5cv5ym/ePEiLC0tSxwUERERkaHRKunq1KkTgoODcenSJans4sWLGDNmDDp16qSz4IiIiIgMhVZJ17x582BpaQlvb2+4u7vD3d0d1apVg52dHebPn6/rGImIiIhee1r99qJKpcKhQ4ewb98+nDx5Eubm5vDz88Pbb7+t6/iIiIiIDIJMaPEjiqtXr0bPnj2hUCg0yjMzM7F+/Xr069dPZwGWVGpqKlQqFVJSUmBtbV3W4RAREVERGOL1W6ukSy6XIyEhAQ4ODhrlycnJcHBwyHc5ibJiiG8aERGRoTPE67dWc7qEEBq/w5jrxo0bUKlUJQ6KiIiIyNAUa06Xv78/ZDIZZDIZmjdvDmPj/16elZWF+Ph4tGnTRudBEhEREb3uipV0de7cGQAQFxeH1q1bw8rKStpmamoKNzc3dOvWTacBEhERERmCYiVdISEhAAA3Nzf07NkTZmZmpRIUERERkaHRasmI/v376zoOIiIiIoOmVdKVlZWFL7/8Ehs2bMC1a9eQmZmpsf3evXs6CY6IiIjIUGh19+K0adOwcOFC9OjRAykpKRg9ejS6du0KIyMjhIaG6jhEIiKi10PTpk0RHBys9etDQ0NRq1YtndfN5ebmhvDw8AK3lzR+KpxWPV0//PADvv32W7Rv3x7Tpk1D79694eHhAT8/P/z1118YOXKkruMkIiKi53z22WcYMWKETtvctGkTTExMdNom/Uernq7ExET4+voCAKysrJCSkgIA6NChA3bs2KG76IiIiChfVlZWsLOz02mbtra2UCqVOm2T/qNV0lWxYkUkJCQAAKpUqYK9e/cCAGJjY/P8NBAREdGbJDs7G+PGjYOtrS3UarXGtJuUlBQMHjwYDg4OsLa2xjvvvIOTJ08W2FZUVBTq168PS0tL2NjYoHHjxrh69SqAvMOLQUFB6Ny5M+bPnw8nJyfY2dlh2LBhePr0aYHtR0ZGQqVSYd++fQA4vFjatEq6unTpgv379wMARo0ahc8//xyenp7o168fBgwYoNMAiYiIXierVq2CpaUlDh8+jLlz52L69OnYt28fhBBo3749EhMTsXPnThw7dgy1a9dG8+bN870B7dmzZ+jcuTMCAwNx6tQpxMTEYPDgwfn+IkyugwcP4tKlSzh48CBWrVqFlStXYuXKlfnWnT9/Pj777DPs2bMHLVu21NXhUyG0mtM1e/Zs6f+7d++OihUr4tChQ6hSpQo6deqks+CIiIheN35+ftK6lp6enliyZAn2798PuVyO06dPIykpSRoVmj9/PrZs2YKNGzdi8ODBGu2kpqYiJSUFHTp0gIeHBwCgWrVqhe67XLlyWLJkCeRyOby9vdG+fXvs378fgwYN0qg3ceJErFq1ClFRUdJ0ISp9WiVdL2rYsCEaNmyoi6aIiIhea35+fhrPnZyckJSUhGPHjuHRo0d55mE9efIEly5dytOOra0tgoKC0Lp1a7Rs2RItWrRAjx494OTkVOC+q1evDrlcrrHv06dPa9RZsGAB0tLScPToUVSuXFmbQyQtaZ103bx5E3/++SeSkpKQnZ2tsY13LxIR0Zvqxbv/ZDIZsrOzkZ2dDScnJ0RFReV5jY2NTb5tRUZGYuTIkdi9ezd++uknTJkyBfv27Suwo6OgfT+vSZMm2LFjBzZs2IAJEyYU/cCoxLRKuiIjIzFkyBCYmprCzs5OY3xZJpMVOemKiIhAREQErly5AiAnQ586dSratm0LABBCYNq0afjmm29w//59NGjQAF9//TWqV6+uTdhERERlpnbt2khMTISxsTHc3NyK/Dp/f3/4+/tj4sSJCAgIwLp160o0ulS/fn2MGDECrVu3hlwux9ixY7Vui4pHq4n0U6dOxdSpU5GSkoIrV64gPj5eely+fLnI7VSsWBGzZ8/G0aNHcfToUbzzzjt49913cfbsWQDA3LlzsXDhQixZsgSxsbFQq9Vo2bIlHj58qE3YREREZaZFixYICAhA586dsWfPHly5cgWHDh3ClClTcPTo0Tz14+PjMXHiRMTExODq1avYu3cv/v3335fO6yqKgIAA7Nq1C9OnT8eXX36pVRtZ2QIxl5LxS9xNxFxKRla2KHFchk6rnq7Hjx+jV69eMDLSKmeTdOzYUeP5F198gYiICPz111/w8fFBeHg4Jk+ejK5duwLIuSPE0dER69atw8cff1yifRMREemTTCbDzp07MXnyZAwYMAB37tyBWq3G22+/DUdHxzz1LSws8M8//2DVqlVITk6Gk5MThg8frrPrX+PGjbFjxw60a9cOcrm8WFODdp9JwLRt55CQki6VOanMENLRB21qFDzn7E0nE0IUOzXNXX9El2PBWVlZ+Pnnn9G/f3+cOHECZmZm8PDwwPHjx+Hv7y/Ve/fdd2FjY4NVq1bl205GRgYyMjKk56mpqXBxcUFKSgqsra11Fi8REdGbaPeZBHyy9jheTB5yJxpFvF9bJ4lXamoqVCqVQV2/terpCgsLQ4cOHbB79274+vrmmbi3cOHCIrd1+vRpBAQEID09HVZWVti8eTN8fHxw6NAhAMiT/Ts6OkoLwxUU27Rp04pxNERERFQUWdkC07ady5NwAYBATuI1bds5tPRRQ25U8Hpibyqtkq5Zs2Zhz549qFq1KgDkmUhfHFWrVkVcXBwePHiA//u//0P//v0RHR1dYHtCiEL3MXHiRIwePVp6ntvTRURERCVzJP6expDiiwSAhJR0HIm/hwAP3f5EkSHQKulauHAhVqxYgaCgoBIHYGpqiipVqgAA6tati9jYWCxatAjjx48HkPM7j8+vSZKUlJTv2HcuhULBnyIiIiIqBUkPC064tKn3ptFqJrxCoUDjxo11HQuAnJ6sjIwMuLu7Q61WS78HBQCZmZmIjo5Go0aNSmXfREREVDAHpZlO671ptEq6Ro0aha+++qrEO580aRJ+//13XLlyBadPn8bkyZMRFRWFvn37QiaTITg4GLNmzcLmzZtx5swZBAUFwcLCAn369CnxvomIiKh46rvbwkllhoIm+ciQcxdjfXdbfYb12tBqePHIkSM4cOAAtm/fjurVq+eZSL9p06YitXP79m188MEHSEhIgEqlgp+fH3bv3i398Oa4cePw5MkTDB06VFocde/evVAqldqETURERCUgN5IhpKMPPll7HDJAY0J9biIW0tGHk+gLoNWSER9++GGh2yMjI7UOSNcM8ZZTIiKisqSPdboM8fqtVdL1OjHEN42IiKisZWULHIm/h6SH6XBQ5gwp6rKHyxCv31r/4DURERG9ueRGMi4LUUwl+x0fIiIiIioSJl1EREREesCki4iIiEgPtEq6Vq9erfGj0rkyMzOxevXqEgdFREREZGi0untRLpcjISEBDg4OGuXJyclwcHBAVlaWzgIsKUO8+4GIiMjQGeL1W6ueroJ+dPrGjRtQqVQlDoqIiIjI0BRryQh/f3/IZDLIZDI0b94cxsb/vTwrKwvx8fFo06aNzoMkIiIiet0VK+nq3LkzACAuLg6tW7eGlZWVtM3U1BRubm7o1q2bTgMkIiIiMgTFSrpCQkIAAG5ubujVqxcUCkWpBEVERERkaLSa0+Xj44O4uLg85YcPH8bRo0dLGhMRERGRwdEq6Ro2bBiuX7+ep/zmzZsYNmxYiYMiIiIiMjRaJV3nzp1D7dq185T7+/vj3LlzJQ6KiIiIyNBolXQpFArcvn07T3lCQoLGHY1ERERElEOrpKtly5aYOHEiUlJSpLIHDx5g0qRJaNmypc6CIyIiIjIUWnVLLViwAG+//TZcXV3h7+8PIGcZCUdHR6xZs0anARIREREZAq2SrgoVKuDUqVP44YcfcPLkSZibm+PDDz9E7969YWJiousYiYiIiF57Wk/AsrS0xODBg3UZCxEREZHBKtGs93PnzuHatWvIzMzUKO/UqVOJgiIiIiIyNFolXZcvX0aXLl1w+vRpyGQyCCEAQPoR7KysLN1FSERERGQAtLp7cdSoUXB3d8ft27dhYWGBs2fP4rfffkPdunURFRWl4xCJiIiIXn9a9XTFxMTgwIEDsLe3h5GREYyMjPDWW28hLCwMI0eOxIkTJ3QdJxEREdFrTauerqysLFhZWQEAypcvj1u3bgEAXF1dcf78ed1FR0RERGQgtOrpqlGjBk6dOoXKlSujQYMGmDt3LkxNTfHNN9+gcuXKuo6RiIiI6LWnVdI1ZcoUpKWlAQBmzpyJDh06oEmTJrCzs8NPP/2k0wCJiIiIDIFM5N56WEL37t1DuXLlpDsYXxWpqalQqVRISUmBtbV1WYdDRERERWCI12+t5nTl92PXtra2kMlkOHXqVImDIiIiIjI0WiVdvr6+2Lp1a57y+fPno0GDBiUOioiIiMjQaJV0jR8/Hj179sSQIUPw5MkT3Lx5E++88w7mzZvHOV1ERERE+dAq6RozZgz++usv/Pnnn/Dz84Ofnx/Mzc1x6tQp/gQQERERUT60SroAoHLlyqhevTquXLmC1NRU9OjRA46OjrqMjYiIiMhgaJV05fZwXbx4EadOnUJERARGjBiBHj164P79+7qOkYiIiOi1p1XS9c4776Bnz56IiYlBtWrVMHDgQJw4cQI3btyAr6+vrmMkIiIieu1ptTjq3r17ERgYqFHm4eGBP/74A1988YVOAiMiIiIyJDpbHPVVZYiLqxERERk6Q7x+F2t4sV27dkhJSZGef/HFF3jw4IH0PDk5GT4+PjoLjoiIiMhQFCvp2rNnDzIyMqTnc+bMwb1796Tnz549w/nz53UXHREREZGBKFbS9eJIpIGPTBIRERHpjNbrdBERERFR0RUr6ZLJZJDJZHnKiIiIiKhwxVoyQgiBoKAgKBQKAEB6ejqGDBkCS0tLANCY70VERERE/ylW0tW/f3+N5++//36eOv369StZREREREQGqFhJV2RkZGnFQURERGTQOJGeiIiISA+YdBERERHpAZMuIiIiIj1g0kVERESkB0y6iIiIiPSASRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRERER6UGZJl1hYWGoV68elEolHBwc0LlzZ5w/f16jjhACoaGhcHZ2hrm5OZo2bYqzZ8+WUcRERERE2inTpCs6OhrDhg3DX3/9hX379uHZs2do1aoV0tLSpDpz587FwoULsWTJEsTGxkKtVqNly5Z4+PBhGUZOREREVDwyIYQo6yBy3blzBw4ODoiOjsbbb78NIQScnZ0RHByM8ePHAwAyMjLg6OiIOXPm4OOPP35pm6mpqVCpVEhJSYG1tXVpHwIRERHpgCFev1+pOV0pKSkAAFtbWwBAfHw8EhMT0apVK6mOQqFAYGAgDh06lG8bGRkZSE1N1XgQERERlbVXJukSQmD06NF46623UKNGDQBAYmIiAMDR0VGjrqOjo7TtRWFhYVCpVNLDxcWldAMnIiIiKoJXJukaPnw4Tp06hR9//DHPNplMpvFcCJGnLNfEiRORkpIiPa5fv14q8RIREREVh3FZBwAAI0aMwNatW/Hbb7+hYsWKUrlarQaQ0+Pl5OQklSclJeXp/cqlUCigUChKN2AiIiKiYirTni4hBIYPH45NmzbhwIEDcHd319ju7u4OtVqNffv2SWWZmZmIjo5Go0aN9B0uERERkdbKtKdr2LBhWLduHX755RcolUppnpZKpYK5uTlkMhmCg4Mxa9YseHp6wtPTE7NmzYKFhQX69OlTlqETERERFUuZJl0REREAgKZNm2qUR0ZGIigoCAAwbtw4PHnyBEOHDsX9+/fRoEED7N27F0qlUs/REhEREWnvlVqnqzQY4jofREREhs4Qr9+vzN2LRERERIaMSRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRaSkqKgoymQwPHjwAAKxcuRI2NjZlGhMREb26mHRRmWratCmCg4M1ymQyWZ7HsmXLyiZAHdu0aRPq1q0LGxsbWFpaolatWlizZk3ZBpWdBcT/DpzemPPf7KyyjYeIyEC9Er+9SPSiyMhItGnTRnquUqnKMBrdsbW1xeTJk+Ht7Q1TU1Ns374dH374IRwcHNC6dWv9B3RuK7B7PJB6678ya2egzRzAp5P+4yEiMmDs6aIyExQUhOjoaCxatEjq0bpy5QoAwMbGBmq1WnqYm5sX2tbJkyfRrFkzKJVKWFtbo06dOjh69CiA/4b9tm/fjqpVq8LCwgLdu3dHWloaVq1aBTc3N5QrVw4jRoxAVtZ/vTxr165F3bp1oVQqoVar0adPHyQlJZXomJs2bYouXbqgWrVq8PDwwKhRo+Dn54c//vijRO1q5dxWYEM/zYQLAFITcsrPbdV/TEREBoxJF5WZRYsWISAgAIMGDUJCQgISEhLg4uICABg+fDjKly+PevXqYdmyZcjOzi60rb59+6JixYqIjY3FsWPHMGHCBJiYmEjbHz9+jMWLF2P9+vXYvXs3oqKi0LVrV+zcuRM7d+7EmjVr8M0332Djxo3SazIzMzFjxgycPHkSW7ZsQXx8vPTzVLoghMD+/ftx/vx5vP322zprt0iys3J6uJDfD1L8r2z3BA41EhHpEIcXqcyoVCqYmprCwsICarVaKp8xYwaaN28Oc3Nz7N+/H2PGjMHdu3cxZcqUAtu6du0axo4dC29vbwCAp6enxvanT58iIiICHh4eAIDu3btjzZo1uH37NqysrODj44NmzZrh4MGD6NmzJwBgwIAB0usrV66MxYsXo379+nj06BGsrKy0Pu6UlBRUqFABGRkZkMvlWLp0KVq2bKl1e1q5eihvD5cGAaTezKnn3kRvYRERGTL2dNErZ8qUKQgICECtWrUwZswYTJ8+HfPmzZO2W1lZSY8hQ4YAAEaPHo2BAweiRYsWmD17Ni5duqTRpoWFhZRwAYCjoyPc3Nw0kidHR0eN4cMTJ07g3XffhaurK5RKpfTD7NeuXXvpMVy7dk0jzlmzZknblEol4uLiEBsbiy+++AKjR49GVFRUsc5RiT26rdt6RET0Uuzpoldew4YNkZqaitu3b8PR0RFxcXHSttwfQQ0NDUWfPn2wY8cO7Nq1CyEhIVi/fj26dOkCABpDjUDOHZL5leUOY6alpaFVq1Zo1aoV1q5dC3t7e1y7dg2tW7dGZmbmS2N2dnbWiNPW1lb6fyMjI1SpUgUAUKtWLfz9998ICwuTkjq9sHLUbT0thIaGYsuWLRrniYjIkDHpojJlamqqMXk9PydOnICZmZm0BlZuwvIiLy8veHl54dNPP0Xv3r0RGRkpJV3F9c8//+Du3buYPXu2NM8sd2J+URgbGxcY54uEEMjIyNAqTq25Nsq5SzE1AfnP65LlbHdtlGdL06ZNUatWLYSHh/9XWybLUy8iIkLqiSQiIiZdVMbc3Nxw+PBhXLlyBVZWVvjzzz+RlJSEgIAAmJub4+DBg5g8eTIGDx4MhUKRbxtPnjzB2LFj0b17d7i7u+PGjRuIjY1Ft27dtI6rUqVKMDU1xVdffYUhQ4bgzJkzmDFjRuEvys7OmXh+emNOD5FrI8BIrlElLCwMdevWhYeHBzIzM7Fz506sXr0aERERWseqFSN5zrIQG/oBkEEz8fpfAtVmdp74C2Ooy3wQEekK53RRmfrss88gl8vh4+MDe3t7JCQkYOnSpQgICICfnx8WLVqE6dOnY8GCBQW2IZfLkZycjH79+sHLyws9evRA27ZtMW3aNK3jsre3x8qVK/Hzzz/Dx8cHs2fPxvz58wt+wbmtwN7JQOYj4P8+AlZ1AMJr5Fl2IS0tDUOHDkX16tXRqFEjbNy4EWvXrsXAgQO1jlVrPp2AHqsBayfNcmvnnPJ81unS5TIfuZYvXw4XFxdYWFjgvffek1b4B4DY2Fi0bNkS5cuXh0qlQmBgII4fP67x+tDQUFSqVAkKhQLOzs4YOXKktC0zMxPjxo1DhQoVYGlpiQYNGuh//hwR0f/IhBD5jS0YjNTUVKhUKqSkpEjzf4h0Kne9qzzDdP/rMSoggXllZGfl3KX46HaBPXS5UlJS0LZtW9SoUQPTp08HkJOgGhsbo0KFCkhPT4e7uzs++ugjDB48GEZGBf9dFxoaivnz56NBgwZYsGABUlNT8dFHH6F+/fr44YcfAAAHDhzArVu3UKdOHQDAggULsH37dly4cAFKpRIbN27ERx99hPXr16N69epITEzEyZMnMWjQIAA5S4lcuXIFs2fPhrOzMzZv3owpU6bg9OnTee5wJaJXiyFevzm8SFQSL13vSpaz3pV3+2IN1emVkbzIy0LocpkPAEhPT8eqVatQsWJFAMBXX32F9u3bY8GCBVCr1XjnnXc06i9fvhzlypVDdHQ0OnTogGvXrkGtVqNFixYwMTFBpUqVUL9+fQDApUuX8OOPP+LGjRtwdnYGkNOzunv3bkRGRmrcUUpEpA8cXiQqieKsd2XAtFnmA8iZO5ebcAFAQEAAsrOzcf78eQBAUlIShgwZAi8vL6hUKqhUKjx69EhatuO9997DkydPULlyZQwaNAibN2/Gs2fPAADHjx+HEAJeXl4a+4+Ojs6zpAgRkT6wp4uoJLjeVb6KssxHfnLvgsz9b1BQEO7cuYPw8HC4urpCoVAgICBAWrbDxcUF58+fx759+/Drr79i6NChmDdvHqKjo5GdnQ25XI5jx45BLtfsZSzJ4rZERNpi0kVUEq/Aelf6pstlPq5du4Zbt25Jw38xMTEwMjKCl5cXAOD333/H0qVL0a5dOwDA9evXcffuXY02zM3N0alTJ3Tq1AnDhg2Dt7c3Tp8+DX9/f2RlZSEpKQlNmnBVfSIqe0y6iEqiBOtdva50scxHLjMzM/Tv3x/z589HamoqRo4ciR49ekjzxapUqYI1a9agbt26SE1NxdixYzXuily5ciWysrLQoEEDWFhYYM2aNTA3N4erqyvs7OzQt29f9OvXDwsWLIC/vz/u3r2LAwcOwNfXV0rkiIj0hXO6iEoid70rANLdihLt1rt61elimY9cVapUQdeuXdGuXTu0atUKNWrUwNKlS6XtK1aswP379+Hv748PPvgAI0eOhIODg7TdxsYG3377LRo3bgw/Pz/s378f27Ztg52dHYCctcP69euHMWPGoGrVqujUqRMOHz4sLXhLRKRPXDKCSBfObc25i/H5SfXWFXISrld5uQgioleUIV6/ObxIpAs+nXKWhSjieldERPTmYdJFpCvFWO+KiIjePJzTRURERKQHTLqIiIiI9IBJFxEREZEeMOkiIiIi0gMmXURERER6wKSLiIiISA+YdBERERHpAZMuIiIiIj1g0kVERESkB0y6iIiIiPSASRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRERER6QGTrjdcVFQUZDIZHjx4AABYuXIlbGxsyjQmIiIiQ8SkSweaNm2K4OBgjTKZTJbnsWzZsrIJUMfOnj2Lbt26wc3NDTKZDOHh4WUdEhER0SvPuKwDMGSRkZFo06aN9FylUpVhNLrz+PFjVK5cGe+99x4+/fTTsg6HiIjotcCerhIKCgpCdHQ0Fi1aJPVoXblyBQBgY2MDtVotPczNzQtt6+TJk2jWrBmUSiWsra1Rp04dHD16FMB/w37bt29H1apVYWFhge7duyMtLQ2rVq2Cm5sbypUrhxEjRiArK0tqc+3atahbty6USiXUajX69OmDpKSkEh1zvXr1MG/ePPTq1QsKhaJEbREREb0pmHSV0KJFixAQEIBBgwYhISEBCQkJcHFxAQAMHz4c5cuXR7169bBs2TJkZ2cX2lbfvn1RsWJFxMbG4tixY5gwYQJMTEyk7Y8fP8bixYuxfv167N69G1FRUejatSt27tyJnTt3Ys2aNfjmm2+wceNG6TWZmZmYMWMGTp48iS1btiA+Ph5BQUGlci6IiIioYBxeLCGVSgVTU1NYWFhArVZL5TNmzEDz5s1hbm6O/fv3Y8yYMbh79y6mTJlSYFvXrl3D2LFj4e3tDQDw9PTU2P706VNERETAw8MDANC9e3esWbMGt2/fhpWVFXx8fNCsWTMcPHgQPXv2BAAMGDBAen3lypWxePFi1K9fH48ePYKVlZXOzgMREREVjj1dpWTKlCkICAhArVq1MGbMGEyfPh3z5s2TtltZWUmPIUOGAABGjx6NgQMHokWLFpg9ezYuXbqk0aaFhYWUcAGAo6Mj3NzcNJInR0dHjeHDEydO4N1334WrqyuUSiWaNm0KICfBe5lr165pxDlr1iytzgURERGxp0tvGjZsiNTUVNy+fRuOjo6Ii4uTtllbWwMAQkND0adPH+zYsQO7du1CSEgI1q9fjy5dugCAxlAjkHOHZH5lucOYaWlpaNWqFVq1aoW1a9fC3t4e165dQ+vWrZGZmfnSmJ2dnTXitLW11ebQiYiICEy6dMLU1FRj8np+Tpw4ATMzM2kNrCpVquRbz8vLC15eXvj000/Ru3dvREZGSklXcf3zzz+4e/cuZs+eLc0zy52YXxTGxsYFxklERETFw6RLW9lZwNVDwKPbcLMzw+HDf+HKlSuwsrLCn3/+iaSkJAQEBMDc3BwHDx7E5MmTMXjw4ALv9nvy5AnGjh2L7t27w93dHTdu3EBsbCy6deumdYiVKlWCqakpvvrqKwwZMgRnzpzBjBkztG4vV2ZmJs6dOyf9/82bNxEXFwcrKysmaURERAXgnC5tnNsKhNcAVnUA/u8jfFb+AORJZ+BTrSrs7e2RkJCApUuXIiAgAH5+fli0aBGmT5+OBQsWFNikXC5HcnIy+vXrBy8vL/To0QNt27bFtGnTtA7T3t4eK1euxM8//wwfHx/Mnj0b8+fP17q9XLdu3YK/vz/8/f2RkJCA+fPnw9/fHwMHDixx268TQ1nNPytbIOZSMn6Ju4mYS8nIyhZlHRIRkUGSCSEM+hs2NTUVKpUKKSkp0typEjm3FdjQD8CLp02W858eqwGfTiXfzxuiadOmqFWrlsaq9jKZLE+9iIgI6YaDV0VUVBSaNWuG+/fvw8bGBitXrkRwcLCUhOVn06ZNmDVrFi5evIinT5/C09MTY8aMwQcffKC/wJ+z+0wCpm07h4SUdKnMSWWGkI4+aFPDqUxiIiICSuH6/Qrg8GJxZGcBu8cjb8KF/5XJgN0TAO/2gJFcz8EZFkNdzd/W1haTJ0+Gt7c3TE1NsX37dnz44YdwcHBA69at9RrL7jMJ+GTt8Tyf5sSUdHyy9jgi3q/NxIuISIfKdHjxt99+Q8eOHeHs7AyZTIYtW7ZobBdCIDQ0FM7OzjA3N0fTpk1x9uzZsgkWyJnDlXqrkAoCSL2ZU49e6k1czb9p06bo0qULqlWrBg8PD4waNQp+fn74448/StRucWVlC0zbdq7APx8AYNq2cxxqJCLSoTJNutLS0lCzZk0sWbIk3+1z587FwoULsWTJEsTGxkKtVqNly5Z4+PChniP9n0e3dVvvDfemr+YvhMD+/ftx/vx5vP322zprtyiOxN/TGFJ8kQCQkJKOI/H39BcUEZGBK9PhxbZt26Jt27b5bhNCIDw8HJMnT0bXrl0BAKtWrYKjoyPWrVuHjz/+WJ+h5rBy1G29N9ybupp/SkoKKlSogIyMDMjlcixduhQtW7bUuj1tJD0sOOHSpp42QkNDsWXLFo214IiIDNkre/difHw8EhMT0apVK6lMoVAgMDAQhw4VPHyXkZGB1NRUjYfOuDYCrJ0hTZrPQwZYV8ipR1oz9NX8lUol4uLiEBsbiy+++AKjR49GVFRUsc5RSTkozbSu17RpUwQHB2uU5Q4PP/9YtmyZLkIlIjIYr+xE+sTERAA5F8LnOTo64urVqwW+LiwsrETLLBTKSA60mfO/uxdl0JxQ/79ErM1sTqLXMUNbzd/IyEhaz6xWrVr4+++/ERYWJiV1+lDf3RZOKjMkpqTnO69LBkCtMkN996L/CoGh3vxARKQrr2xPV64Xlw8QQuS7pECuiRMnIiUlRXpcv35dtwH5dMpZFsL6hbu6rJ25XIQWtF3NP/fh4OAg1ctdyX/v3r3o2rUrIiMjtY7r+dX8mzRpAm9v72JNos9dzT/3UdhPKAkhkJGRoXWs2pAbyRDS0QdA3n7b3OchHX0gN9LcqsubH3ItX74cLi4usLCwwHvvvaex5EZsbCxatmyJ8uXLQ6VSITAwEMePH9d4fWhoKCpVqgSFQgFnZ2eMHDlS2paZmYlx48ahQoUKsLS0RIMGDfTeq0hElOuV7enKneOTmJgIJ6f/EpykpKQ8vV/PUygUBa76rjM+nXKWhfjfivSwcswZUmQPV7G5ubnh8OHDBrGaf3a2QFa2wC9xN+GgzOklejFpCQsLQ926deHh4YHMzEzs3LkTq1evRkREhNaxaqtNDSdEvF87zzpd6kLW6Vq0aBH+/fdf1KhRA9OnTweQswgvkHPzw8CBA+Hu7o6PPvoIgwcPhpFR4X/XXbx4ERs2bMC2bduQmpqKjz76CMOGDcMPP/wAAHj48CH69++PxYsXAwAWLFiAdu3a4cKFC1Aqldi4cSO+/PJLrF+/HtWrV0diYiJOnjwptf/hhx/iypUrWL9+PZydnbF582a0adMGp0+fzjPvj4io1IlXBACxefNm6Xl2drZQq9Vizpw5UllGRoZQqVRi2bJlRW43JSVFABApKSm6DJd05Pz586Jhw4bC3NxcABARERGiVq1awsrKSlhYWIgaNWqI8PBw8fTp0wLbyMjIEL169RIuLi7C1NRUODs7i+HDh4snT54IIYSIjIwUKpVK4zUhISGiZs2aGmX9+/cX7777rvR83bp1ws3NTSgUChEQECC2bt0qAIgTJ04IIYQ4ePCgACDu378vdp2+JSp3GytkCkvhOn67cB2/XTSc9avYdfqWxj4mT54sqlSpIszMzES5cuVEQECAWL9+vdbnTxeeZWWLQxfvii0nbohDF++KZ1nZhdYPDAwUo0aN0iibMWOGOHTokDhx4oSYP3++sLCwEDNmzCi0nZCQECGXy8X169elsl27dgkjIyORkJCQf6zPngmlUim2bdsmhBBiwYIFwsvLS2RmZuape/HiRSGTycTNmzc1yps3by4mTpxYaGxEVPYM8fpdpivSP3r0CBcvXgQA+Pv7Y+HChWjWrBlsbW1RqVIlzJkzB2FhYYiMjISnpydmzZqFqKgonD9/Hkqlskj7MMQVbenVUtAio7l9XIa2yGh+vyLwogULFmD69OlISUkBAI0bFt5//30sW7YMoaGhWL16NS5fvixtS0lJgY2NDaKiohAYGIikpCRMnToVBw4cwO3bt5GVlYXHjx9jyZIlGDp0KK5fv47GjRtDCIE2bdqgXbt26NixI4yNjfHzzz+jR48esLS01IgtIyMDXbt2xU8//aTbE0NEOmWI1+8yHV48evQomjVrJj0fPXo0AKB///5YuXIlxo0bhydPnmDo0KG4f/8+GjRogL179xY54SIqbS9bZFSGnEVGW/qo8ww1GrKi3PyQn9z5mrn/DQoKwp07dxAeHg5XV1coFAoEBARINzO4uLjg/Pnz2LdvH3799VcMHToU8+bNQ3R0NLKzsyGXy3Hs2DHI5ZpD/yVZ8oOISFtlmnQ1bdoUhXW0yWQyhIaGIjQ0VH9BERVDcRYZDfCw019gpUjbmx/yc+3aNdy6dQvOzs4AgJiYGBgZGcHLywsA8Pvvv2Pp0qVo164dAOD69eu4e/euRhvm5ubo1KkTOnXqhGHDhsHb2xunT5+Gv78/srKykJSUhCZNmpTkkImIdOKVnUhP9Dp4FRYZ1Tdd3PyQy8zMDP3798f8+fORmpqKkSNHokePHtKNNFWqVMGaNWtQt25dpKamYuzYsRp3Ra5cuRJZWVlo0KABLCwssGbNGpibm8PV1RV2dnbo27cv+vXrhwULFsDf3x93797FgQMH4OvrKyVyRET68sovGUH0KivJIqOvq88++wxyuRw+Pj6wt7dHQkICli5dioCAAPj5+WHRokWYPn06FixY8NK2qlSpgq5du6Jdu3Zo1aoVatSogaVLl0rbV6xYgfv378Pf3x8ffPABRo4cqbFMiI2NDb799ls0btwYfn5+2L9/P7Zt2wY7u5xexcjISPTr1w9jxoxB1apV0alTJxw+fFj6uSkiIn0q04n0+mCIE/Ho1ZGVLfDWnAMvXWT0j/HvvFFzuoiISsoQr9/s6SIqAW0XGSUiojcPky6iEspdZFSt0hxCVKvMDG65CCIi0h4n0hPpQJsaTmjpo8aR+HtIephe4Ir0RET05mLSRaQjciOZwSwLQUREusfhRSIiIiI9YNJFREREpAdMuoiIiIj0gEkXERERkR4w6SIiIiLSAyZdRERERHrApOsNFxUVBZlMhgcPHgDI+QFhGxubMo2JiIjIEDHp0oGmTZsiODhYo0wmk+V5LFu2rGwC1LGzZ8+iW7ducHNzg0wmQ3h4eFmHRERE9Mrj4qilKDIyEm3atJGeq1SqMoxGdx4/fozKlSvjvffew6efflrW4RAREb0W2NNVQkFBQYiOjsaiRYukHq0rV64AAGxsbKBWq6WHubl5oW2dPHkSzZo1g1KphLW1NerUqYOjR48C+G/Yb/v27ahatSosLCzQvXt3pKWlYdWqVXBzc0O5cuUwYsQIZGVlSW2uXbsWdevWhVKphFqtRp8+fZCUlFSiY65Xrx7mzZuHXr16QaFQlKgtIiKiNwWTrhJatGgRAgICMGjQICQkJCAhIQEuLi4AgOHDh6N8+fKoV68eli1bhuzs7ELb6tu3LypWrIjY2FgcO3YMEyZMgImJibT98ePHWLx4MdavX4/du3cjKioKXbt2xc6dO7Fz506sWbMG33zzDTZu3Ci9JjMzEzNmzMDJkyexZcsWxMfHIygoqFTOBRERERWMw4slpFKpYGpqCgsLC6jVaql8xowZaN68OczNzbF//36MGTMGd+/exZQpUwps69q1axg7diy8vb0BAJ6enhrbnz59ioiICHh4eAAAunfvjjVr1uD27duwsrKCj48PmjVrhoMHD6Jnz54AgAEDBkivr1y5MhYvXoz69evj0aNHsLKy0tl5ICIiosKxp6uUTJkyBQEBAahVqxbGjBmD6dOnY968edJ2Kysr6TFkyBAAwOjRozFw4EC0aNECs2fPxqVLlzTatLCwkBIuAHB0dISbm5tG8uTo6KgxfHjixAm8++67cHV1hVKpRNOmTQHkJHgvc+3aNY04Z82apdW5ICIiIvZ06U3Dhg2RmpqK27dvw9HREXFxcdI2a2trAEBoaCj69OmDHTt2YNeuXQgJCcH69evRpUsXANAYagRy7pDMryx3GDMtLQ2tWrVCq1atsHbtWtjb2+PatWto3bo1MjMzXxqzs7OzRpy2trbaHDoRERGBSZdOmJqaakxez8+JEydgZmYmrYFVpUqVfOt5eXnBy8sLn376KXr37o3IyEgp6Squf/75B3fv3sXs2bOleWa5E/OLwtjYuMA4iYiIqHiYdGkpK1vgSPw9JD1Mh7mtGn8dPowrV67AysoKf/75J5KSkhAQEABzc3McPHgQkydPxuDBgwu82+/JkycYO3YsunfvDnd3d9y4cQOxsbHo1q2b1jFWqlQJpqam+OqrrzBkyBCcOXMGM2bM0Lq9XJmZmTh37pz0/zdv3kRcXBysrKyYpBERERWASZcWdp9JwLRt55CQkg4AeGr9FlITj8K7WjVkpKcjIiICy5cvx+jRo5GdnY3KlStj+vTpGDZsWIFtyuVyJCcno1+/frh9+zbKly+Prl27Ytq0aVrHaW9vj5UrV2LSpElYvHgxateujfnz56NTp05atwkAt27dgr+/v/R8/vz5mD9/PgIDAxEVFVWitomIiAyVTAghyjqI0pSamgqVSoWUlBRp7lRJ7D6TgE/WHseLJ032v/9GvF8bbWo4lXg/REREbzJdX79fBbx7sRiysgWmbTuXJ+ECIJVN23YOWdkGnccSERGRFph0FcOR+HvSkGJ+BICElHQcib+nv6CIiIjotcCkqxiSHhaccGlTj4iIiN4cTLqKwUFpptN6RERE9OZg0lUM9d1t4aQykybNv0gGwEllhvruXESUiIiINDHpKga5kQwhHX0AIE/ilfs8pKMP5EYFpWVERET0pmLSVUxtajgh4v3aUKs0hxDVKjMuF0FEREQF4uKoWmhTwwktfdTSivQOypwhRfZwERERUUGYdGlJbiRDgIddWYdBRERErwkOLxIRERHpAZMuIiIiIj1g0kVERESkB0y6iIiIiPSASRcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHQRERER6YHBr0gvhAAApKamlnEkREREVFS51+3c67ghMPik6+HDhwAAFxeXMo6EiIiIiuvhw4dQqVRlHYZOyIQhpZD5yM7Oxq1bt6BUKiGTGe4PUqempsLFxQXXr1+HtbV1WYdjkHiO9YPnWT94nksfz3HJCCHw8OFDODs7w8jIMGZDGXxPl5GRESpWrFjWYeiNtbU1/3GXMp5j/eB51g+e59LHc6w9Q+nhymUYqSMRERHRK45JFxEREZEeMOkyEAqFAiEhIVAoFGUdisHiOdYPnmf94HkufTzH9CKDn0hPRERE9CpgTxcRERGRHjDpIiIiItIDJl1EREREesCki4iIiEgPmHS9Rn777Td07NgRzs7OkMlk2LJli8Z2IQRCQ0Ph7OwMc3NzNG3aFGfPni2bYF9TYWFhqFevHpRKJRwcHNC5c2ecP39eow7Pc8lFRETAz89PWjQyICAAu3btkrbzHOteWFgYZDIZgoODpTKe55ILDQ2FTCbTeKjVamk7zzE9j0nXayQtLQ01a9bEkiVL8t0+d+5cLFy4EEuWLEFsbCzUajVatmwp/f4kvVx0dDSGDRuGv/76C/v27cOzZ8/QqlUrpKWlSXV4nkuuYsWKmD17No4ePYqjR4/inXfewbvvvitdjHiOdSs2NhbffPMN/Pz8NMp5nnWjevXqSEhIkB6nT5+WtvEckwZBryUAYvPmzdLz7OxsoVarxezZs6Wy9PR0oVKpxLJly8ogQsOQlJQkAIjo6GghBM9zaSpXrpz47rvveI517OHDh8LT01Ps27dPBAYGilGjRgkh+FnWlZCQEFGzZs18t/Ec04vY02Ug4uPjkZiYiFatWkllCoUCgYGBOHToUBlG9npLSUkBANja2gLgeS4NWVlZWL9+PdLS0hAQEMBzrGPDhg1D+/bt0aJFC41ynmfduXDhApydneHu7o5evXrh8uXLAHiOKS+D/8HrN0ViYiIAwNHRUaPc0dERV69eLYuQXntCCIwePRpvvfUWatSoAYDnWZdOnz6NgIAApKenw8rKCps3b4aPj490MeI5Lrn169fj+PHjiI2NzbONn2XdaNCgAVavXg0vLy/cvn0bM2fORKNGjXD27FmeY8qDSZeBkclkGs+FEHnKqGiGDx+OU6dO4Y8//sizjee55KpWrYq4uDg8ePAA//d//4f+/fsjOjpa2s5zXDLXr1/HqFGjsHfvXpiZmRVYj+e5ZNq2bSv9v6+vLwICAuDh4YFVq1ahYcOGAHiO6T8cXjQQuXfL5P5llSspKSnPX1n0ciNGjMDWrVtx8OBBVKxYUSrnedYdU1NTVKlSBXXr1kVYWBhq1qyJRYsW8RzryLFjx5CUlIQ6derA2NgYxsbGiI6OxuLFi2FsbCydS55n3bK0tISvry8uXLjAzzLlwaTLQLi7u0OtVmPfvn1SWWZmJqKjo9GoUaMyjOz1IoTA8OHDsWnTJhw4cADu7u4a23meS48QAhkZGTzHOtK8eXOcPn0acXFx0qNu3bro27cv4uLiULlyZZ7nUpCRkYG///4bTk5O/CxTHhxefI08evQIFy9elJ7Hx8cjLi4Otra2qFSpEoKDgzFr1ix4enrC09MTs2bNgoWFBfr06VOGUb9ehg0bhnXr1uGXX36BUqmU/kJVqVQwNzeX1jnieS6ZSZMmoW3btnBxccHDhw+xfv16REVFYffu3TzHOqJUKqW5iLksLS1hZ2cnlfM8l9xnn32Gjh07olKlSkhKSsLMmTORmpqK/v3787NMeZXdjZNUXAcPHhQA8jz69+8vhMi5PTkkJESo1WqhUCjE22+/LU6fPl22Qb9m8ju/AERkZKRUh+e55AYMGCBcXV2FqampsLe3F82bNxd79+6VtvMcl47nl4wQgudZF3r27CmcnJyEiYmJcHZ2Fl27dhVnz56VtvMc0/NkQghRRvkeERER0RuDc7qIiIiI9IBJFxEREZEeMOkiIiIi0gMmXURERER6wKSLiIiISA+YdBERERHpAZMuIiIiIj1g0kVERESkB0y6iKhIgoKCIJPJpIednR3atGmDU6dOadSTyWTYsmVLvm1ERUVptPH8I/cnl4KCgtC5c+c8r42Li4NMJsOVK1d0fGRERPrBpIuIiqxNmzZISEhAQkIC9u/fD2NjY3To0KHY7Zw/f15qJ/fh4OBQChGXnszMzLIOgYheM0y6iKjIFAoF1Go11Go1atWqhfHjx+P69eu4c+dOsdpxcHCQ2sl9GBmV/OsoNDQUlSpVgkKhgLOzM0aOHClty8jIwLhx4+Di4gKFQgFPT098//330vbo6GjUr18fCoUCTk5OmDBhAp49eyZtb9q0KYYPH47Ro0ejfPnyaNmyJQDg3LlzaNeuHaysrODo6IgPPvgAd+/eLfGxEJHhYdJFRFp59OgRfvjhB1SpUgV2dnZlHQ42btyIL7/8EsuXL8eFCxewZcsW+Pr6Stv79euH9evXY/Hixfj777+xbNkyWFlZAQBu3ryJdu3aoV69ejh58iQiIiLw/fffY+bMmRr7WLVqFYyNjfHnn39i+fLlSEhIQGBgIGrVqoWjR49i9+7duH37Nnr06KHXYyei14NxWQdARK+P7du3S4lKWloanJycsH379mL3UlWsWFHjeYUKFXD+/PkSxXbt2jWo1Wq0aNECJiYmqFSpEurXrw8A+Pfff7Fhwwbs27cPLVq0AABUrlxZeu3SpUvh4uKCJUuWQCaTwdvbG7du3cL48eMxdepU6fiqVKmCuXPnSq+bOnUqateujVmzZkllK1asgIuLC/799194eXmV6JiIyLCwp4uIiqxZs2aIi4tDXFwcDh8+jFatWqFt27a4evVqsdr5/fffpXbi4uKwZ8+eEsf23nvv4cmTJ6hcuTIGDRqEzZs3S8ODcXFxkMvlCAwMzPe1f//9NwICAiCTyaSyxo0b49GjR7hx44ZUVrduXY3XHTt2DAcPHoSVlZX08Pb2BgBcunSpxMdERIaFPV1EVGSWlpaoUqWK9LxOnTpQqVT49ttv8wzFFcbd3R02Njb5brO2ts43iXvw4AEAQKVS5fs6FxcXnD9/Hvv27cOvv/6KoUOHYt68eYiOjoa5uXmh8QghNBKu3DIAGuWWlpYadbKzs9GxY0fMmTMnT5tOTk6F7pOI3jzs6SIirclkMhgZGeHJkyc6a9Pb2xtnzpxBenq6RnlsbCzs7e1Rrly5Al9rbm6OTp06YfHixYiKikJMTAxOnz4NX19fZGdnIzo6Ot/X+fj44NChQ1KiBQCHDh2CUqlEhQoVCtxf7dq1cfbsWbi5uaFKlSoajxcTNCIiJl1EVGQZGRlITExEYmIi/v77b4wYMQKPHj1Cx44dNerFx8drDB/GxcXh0aNH0vakpCSpndzH06dPAQB9+/aFsbExPvjgAxw9ehSXLl3C2rVrERYWhrFjxxYY28qVK/H999/jzJkzuHz5MtasWQNzc3O4urrCzc0N/fv3x4ABA7BlyxbEx8cjKioKGzZsAAAMHToU169fx4gRI/DPP//gl19+QUhICEaPHl3ofLVhw4bh3r176N27N44cOYLLly9j7969GDBgALKyskpyqonIEAkioiLo37+/ACA9lEqlqFevnti4caNGvefrPP84ePCgOHjwYIHbY2JipDYuXLggunXrJipUqCAsLS2Fr6+vWLJkicjKyiowvs2bN4sGDRoIa2trYWlpKRo2bCh+/fVXafuTJ0/Ep59+KpycnISpqamoUqWKWLFihbQ9KipK1KtXT5iamgq1Wi3Gjx8vnj59Km0PDAwUo0aNyrPff//9V3Tp0kXY2NgIc3Nz4e3tLYKDg0V2drY2p5mIDJhMiOf604mIiIioVHB4kYiIiEgPmHQRERER6QGTLiIiIiI9YNJFREREpAdMuoiIiIj0gEkXERERkR4w6SIiIiLSAyZdRERERHrApIuIiIhID5h0EREREekBky4iIiIiPWDSRURERKQH/w953O3Eghzt7wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Sample data\n",
    "exact_match = [val['exact_match'] for key, val in metrics.items()]\n",
    "f1 = [val['f1'] for key, val in metrics.items()]\n",
    "bleus = [val['bleu'] for key, val in metrics.items()]\n",
    "names = [key for key, val in metrics.items()]\n",
    "v_shift = [0,-6, 1, -7, 1, -7, -8]\n",
    "v_position = ['bottom','bottom', 'top', 'bottom', 'bottom', 'top','bottom']\n",
    "position = ['bottom','bottom', 'top', 'bottom', 'bottom', 'top','bottom']\n",
    "\n",
    "# Create scatter plot\n",
    "plt.scatter(bleus, exact_match, label = 'exact_match')\n",
    "plt.scatter(bleus, f1, label = 'f1')\n",
    "val = 0.002\n",
    "# Add text annotations for names\n",
    "for i, name in enumerate(names):\n",
    "    plt.text(bleus[i] +v_shift[i], exact_match[i]+val if i %2 == 0 else exact_match[i]-val, name,\n",
    "             ha='left', va=position[i])\n",
    "    plt.text(bleus[i] +v_shift[i], f1[i]+val if i %2 == 0 else f1[i]-val, name,\n",
    "             ha='left', va=position[i])\n",
    "\n",
    "# Set plot title and labels\n",
    "plt.title(\"Exact match and F1-score vs BLEU scores for the different translators\")\n",
    "plt.xlabel(\"BLEU score\")\n",
    "plt.ylabel(\"Exact match/F1\")\n",
    "\n",
    "# Display the plot\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c595f032",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}