crawler/wiki-data-proc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "# from wikipedia_crawler import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load 309 doctrine list\n",
    "- from this page: https://en.wikipedia.org/wiki/Category:Legal_doctrines_and_principles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept</th>\n",
       "      <th>wiki_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abatement ab initio</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Abatement_ab_initio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abstention doctrine</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Abstention_doctrine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abstraction principle (law)</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Abstraction_prin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acquiescence</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Acquiescence</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Act of state doctrine</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Act_of_state_doc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>Wagoner Doctrine</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Wagoner_Doctrine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>Wahkohtowin</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Wahkohtowin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>Warranty tolling</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Warranty_tolling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>Workers' compensation employer defense</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Workers'_compens...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>Wrongdoing</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Wrongdoing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>309 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    concept  \\\n",
       "0                       Abatement ab initio   \n",
       "1                       Abstention doctrine   \n",
       "2               Abstraction principle (law)   \n",
       "3                              Acquiescence   \n",
       "4                     Act of state doctrine   \n",
       "..                                      ...   \n",
       "304                        Wagoner Doctrine   \n",
       "305                             Wahkohtowin   \n",
       "306                        Warranty tolling   \n",
       "307  Workers' compensation employer defense   \n",
       "308                              Wrongdoing   \n",
       "\n",
       "                                             wiki_link  \n",
       "0    https://en.wikipedia.org/wiki/Abatement_ab_initio  \n",
       "1    https://en.wikipedia.org/wiki/Abstention_doctrine  \n",
       "2    https://en.wikipedia.org/wiki/Abstraction_prin...  \n",
       "3           https://en.wikipedia.org/wiki/Acquiescence  \n",
       "4    https://en.wikipedia.org/wiki/Act_of_state_doc...  \n",
       "..                                                 ...  \n",
       "304     https://en.wikipedia.org/wiki/Wagoner_Doctrine  \n",
       "305          https://en.wikipedia.org/wiki/Wahkohtowin  \n",
       "306     https://en.wikipedia.org/wiki/Warranty_tolling  \n",
       "307  https://en.wikipedia.org/wiki/Workers'_compens...  \n",
       "308           https://en.wikipedia.org/wiki/Wrongdoing  \n",
       "\n",
       "[309 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./complex-law-doctrine-list.csv\")\n",
    "df['wiki_link'] = df.concept.apply(lambda x: \"https://en.wikipedia.org/wiki/\" + \"_\".join(x.split()))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawl wikipedia pages\n",
    "- after filtering, 294 doctrines remain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import time\n",
    "import argparse\n",
    "import re\n",
    "from urllib.parse import urlparse\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def scrap_texts(base_url, article, output_file, html_file):\n",
    "    \"\"\"Represents one request per article\"\"\"\n",
    "\n",
    "    full_url = base_url + article\n",
    "    try:\n",
    "        r = requests.get(full_url, headers={'User-Agent': USER_AGENT})\n",
    "    except requests.exceptions.ConnectionError:\n",
    "        print(\"Check your Internet connection\")\n",
    "        input(\"Press [ENTER] to continue to the next request.\")\n",
    "        return\n",
    "    if r.status_code not in (200, 404):\n",
    "        print(\"Failed to request page (code {})\".format(r.status_code))\n",
    "        input(\"Press [ENTER] to continue to the next request.\")\n",
    "        return\n",
    "    \n",
    "    soup = BeautifulSoup(r.text, 'html.parser')\n",
    "    content = soup.find('div', {'id':'mw-content-text'})\n",
    "\n",
    "    with open(html_file, 'w', encoding='utf-8') as file:\n",
    "        file.write(str(content))\n",
    "\n",
    "    # add new related articles to queue\n",
    "    # check if are actual articles URL\n",
    "    for a in content.find_all('a'):\n",
    "        href = a.get('href')\n",
    "        if not href:\n",
    "            continue\n",
    "        if href[0:6] != '/wiki/':  # allow only article pages\n",
    "            continue\n",
    "        elif ':' in href:  # ignore special articles e.g. 'Special:'\n",
    "            continue\n",
    "        elif href[-4:] in \".png .jpg .jpeg .svg\":  # ignore image files inside articles\n",
    "            continue\n",
    "        elif base_url + href in visited_urls:  # already visited\n",
    "            continue\n",
    "        if href in pending_urls:  # already added to queue\n",
    "            continue\n",
    "        pending_urls.append(href)\n",
    "\n",
    "    # skip if already added text from this article, as continuing session\n",
    "    if full_url in visited_urls:\n",
    "        return\n",
    "    visited_urls.add(full_url)\n",
    "\n",
    "    parenthesis_regex = re.compile('\\(.+?\\)')  # to remove parenthesis content\n",
    "    citations_regex = re.compile('\\[.+?\\]')  # to remove citations, e.g. [1]\n",
    "\n",
    "    # get plain text from each <p>\n",
    "    p_list = content.find_all('p')\n",
    "    with open(output_file, 'a', encoding='utf-8') as fout:\n",
    "        for p in p_list:\n",
    "            text = p.get_text().strip()\n",
    "            text = parenthesis_regex.sub('', text)\n",
    "            text = citations_regex.sub('', text)\n",
    "            if text:\n",
    "                fout.write(text + '\\n\\n')  # extra line between paragraphs\n",
    "\n",
    "response = requests.get('https://en.wikipedia.org/w/api.php', \n",
    "                        params={'action': 'query',\n",
    "                                'format': 'json',\n",
    "                                'titles': 'Abstention_doctrine',\n",
    "                                'prop': 'extracts',\n",
    "                                'exintro': True,\n",
    "                                'explaintext': True,})\n",
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"An abstention doctrine is any of several doctrines that a United States court may (or in some cases must) apply to refuse to hear a case if hearing the case would potentially intrude upon the powers of another court. Such doctrines are usually invoked where lawsuits involving the same issues are brought in two different court systems at the same time (such as federal and state courts).\\nThe United States has a federal court system with limitations on the cases that it can hear, while each state has its own individual court system. In some instances, the jurisdiction of these courts overlap, so a lawsuit between two parties may be brought in either or both courts. The latter circumstance can lead to confusion, waste resources, as well as cause the appearance that one court is disrespecting the other. Both federal and state courts have developed rules determining when one court will defer to another's jurisdiction over a particular case.\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# page_id = list(data[\"query\"][\"pages\"].keys())[0]\n",
    "# data[\"query\"][\"pages\"][page_id][\"extract\"]\n",
    "data = response.json()\n",
    "page_id = list(data[\"query\"][\"pages\"].keys())[0]\n",
    "data[\"query\"][\"pages\"][page_id][\"extract\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFAULT_OUTPUT = 'output.txt'\n",
    "DEFAULT_INTERVAL = 1.0  # interval between requests (seconds)\n",
    "DEFAULT_ARTICLES_LIMIT = 400  # total number articles to be extrated\n",
    "USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'\n",
    "\n",
    "visited_urls = set()  # all urls already visited, to not visit twice\n",
    "pending_urls = []  # queue\n",
    "\n",
    "for row in df.iloc[:].iterrows():\n",
    "    row = row[1]\n",
    "    concept, initial_url = row.concept, row.wiki_link\n",
    "    # interval = DEFAULT_INTERVAL\n",
    "    # articles_limit = DEFAULT_ARTICLES_LIMIT\n",
    "\n",
    "    # minutes_estimate = interval * articles_limit / 60\n",
    "    # print(\"This session will take {:.1f} minute(s) to download {} article(s):\".format(minutes_estimate, articles_limit))\n",
    "    # print(\"\\t(Press CTRL+C to pause)\\n\")\n",
    "    concept_string = \"_\".join(concept.split())\n",
    "    output_file = \"./crawled_data/intro_raw/{}.txt\".format(concept_string)\n",
    "    # html_file = \"./crawled_data/html/{}.html\".format(concept_string)\n",
    "    \n",
    "    # base_url = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(initial_url))\n",
    "    # initial_url = initial_url[len(base_url):]\n",
    "    # pending_urls.append(initial_url)\n",
    "\n",
    "    # next_url = pending_urls.pop(0)\n",
    "    # scrap_texts(base_url, next_url, output_file, html_file)\n",
    "    response = requests.get('https://en.wikipedia.org/w/api.php', \n",
    "                        params={'action': 'query',\n",
    "                                'format': 'json',\n",
    "                                'titles': concept_string,\n",
    "                                'prop': 'extracts',\n",
    "                                'exintro': True,\n",
    "                                'explaintext': True,})\n",
    "    data = response.json()\n",
    "    page_id = list(data[\"query\"][\"pages\"].keys())[0]\n",
    "    if (\"extract\" in data[\"query\"][\"pages\"][page_id]):\n",
    "        intro_text = data[\"query\"][\"pages\"][page_id][\"extract\"]\n",
    "        if len(intro_text.strip()) > 10: \n",
    "            with open(output_file, \"w\") as out:\n",
    "                out.write(intro_text)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter the raw texts --> 294 doctrines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(294, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = []\n",
    "for row in df.iterrows():\n",
    "    row = row[1]\n",
    "    concept, initial_url = row.concept, row.wiki_link\n",
    "    concept_string = \"_\".join(concept.split())\n",
    "    filename = \"./crawled_data/intro_raw/{}.txt\".format(concept_string)\n",
    "    if os.path.isfile(filename):\n",
    "        with open(filename) as f:\n",
    "            raw_text = f.read()\n",
    "            num_words = len(raw_text.split())\n",
    "            # if 150 < num_words < 500:\n",
    "            rows.append([concept_string, raw_text, num_words])\n",
    "df_rows = pd.DataFrame(rows, columns=['concept', 'intro_text', \"word_count\"])    \n",
    "df_rows.to_csv(\"../data/294-doctrines/legal_doctrines_294.tsv\", sep=\"\\t\", index=False)\n",
    "df_rows.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample 101 doctrines out of 294 doctrines\n",
    "- we pick doctrines whose definition length is between 100 and 200 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(101, 3)\n"
     ]
    }
   ],
   "source": [
    "df_medium_length = df_rows[df_rows.word_count <= 200]\n",
    "df_medium_length = df_medium_length[df_medium_length.word_count >= 100]\n",
    "print(df_medium_length.shape)\n",
    "\n",
    "df_medium_length.to_csv(\"../data/101-doctrines/legal_doctrines_101.tsv\", sep=\"\\t\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.02042484, 0.01429739, 0.01429739, 0.01021242, 0.01021242,\n",
       "        0.0122549 , 0.01633987, 0.0122549 , 0.01021242, 0.01021242,\n",
       "        0.00816993, 0.01838235, 0.00816993, 0.00204248, 0.00408497,\n",
       "        0.00612745, 0.01021242, 0.        , 0.00408497, 0.01633987]),\n",
       " array([100. , 104.8, 109.6, 114.4, 119.2, 124. , 128.8, 133.6, 138.4,\n",
       "        143.2, 148. , 152.8, 157.6, 162.4, 167.2, 172. , 176.8, 181.6,\n",
       "        186.4, 191.2, 196. ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAGdCAYAAAAYDtcjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3kklEQVR4nO3dfVRU173/8Q8PwvgQIEpkRFFYjQ0aiRhUHK+NvQ3LseHW0Ca3SFP1sqi2qRojiY1YBRN7iw/RaiINtY0PXY1Xy72GGrU0hNjYFoIRsEarVluVJDiotTKGRFA5vz/y83hOHdQxEqK8X2udZdj7e/bZZ0uYj3seCDAMwxAAAAAkSYHtPQEAAIDPE8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWwe09gc+TlpYW1dXV6Y477lBAQEB7TwcAAFwHwzB09uxZRUdHKzDw0+/7EI4s6urqFBMT097TAAAAN+C9995Tnz59PvU4hCOLO+64Q9InixsWFtbOswEAANfD6/UqJibGfBz/tAhHFpeeSgsLCyMcAQBwi7lZL4nhBdkAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsAhu7wl0JLGzt7bJuEcXprbJuAAAdETsHAEAAFgQjgAAACxuKBwVFBQoNjZWDodDycnJ2rlz51Xri4qKFB8fL4fDoYSEBG3bts3sO3/+vJ555hklJCSoa9euio6O1sSJE1VXV2cb4/Tp03rssccUFhamiIgIZWVl6cMPP7TV7NmzR1/60pfkcDgUExOjxYsX38jtAQCADszvcLRx40ZlZ2crLy9P1dXVGjx4sNxut06cOOGzvry8XBkZGcrKylJNTY3S0tKUlpamvXv3SpI++ugjVVdXa968eaqurtamTZt08OBBjRs3zjbOY489pn379qm0tFRbtmzRjh07NGXKFLPf6/VqzJgx6tevn6qqqrRkyRLNnz9fq1at8vcWAQBABxZgGIbhzwnJyckaNmyYVq5cKUlqaWlRTEyMpk+frtmzZ19Rn56ersbGRm3ZssVsGzFihBITE1VYWOjzGu+8846GDx+uY8eOqW/fvtq/f78GDhyod955R0OHDpUklZSU6KGHHtL777+v6OhovfTSS/rhD38oj8ejkJAQSdLs2bNVXFysAwcOXNe9eb1ehYeHq6GhQWFhYf4sy3XhBdkAANx8N/vx26+do+bmZlVVVSklJeXyAIGBSklJUUVFhc9zKioqbPWS5Ha7W62XpIaGBgUEBCgiIsIcIyIiwgxGkpSSkqLAwEBVVlaaNQ888IAZjC5d5+DBg/rnP//pz20CAIAOzK9wdOrUKV28eFFRUVG29qioKHk8Hp/neDwev+rPnTunZ555RhkZGWb683g86tmzp60uODhY3bt3N8dp7TqX+nxpamqS1+u1HQAAoGP7XL1b7fz58/rmN78pwzD00ksvtfn18vPzFR4ebh4xMTFtfk0AAPD55lc4ioyMVFBQkOrr623t9fX1cjqdPs9xOp3XVX8pGB07dkylpaW25wydTucVL/i+cOGCTp8+bY7T2nUu9fmSk5OjhoYG83jvvfdau3UAANBB+BWOQkJClJSUpLKyMrOtpaVFZWVlcrlcPs9xuVy2ekkqLS211V8KRocOHdIbb7yhHj16XDHGmTNnVFVVZba9+eabamlpUXJyslmzY8cOnT9/3nade+65R3feeafPuYWGhiosLMx2AACAjs3vp9Wys7P185//XOvWrdP+/fv1+OOPq7GxUZmZmZKkiRMnKicnx6yfMWOGSkpKtHTpUh04cEDz58/Xrl27NG3aNEmfBKNHH31Uu3bt0iuvvKKLFy/K4/HI4/GoublZkjRgwACNHTtWkydP1s6dO/WnP/1J06ZN0/jx4xUdHS1J+ta3vqWQkBBlZWVp37592rhxo1asWKHs7OxPvUgAAKDj8Pt3q6Wnp+vkyZPKzc2Vx+NRYmKiSkpKzBc/19bWKjDwcuYaOXKk1q9fr7lz52rOnDnq37+/iouLNWjQIEnSBx98oM2bN0uSEhMTbdfavn27vvzlL0uSXnnlFU2bNk0PPvigAgMD9cgjj+iFF14wa8PDw/X6669r6tSpSkpKUmRkpHJzc22fhQQAAHAtfn/O0e2MzzkCAODW066fcwQAAHC7IxwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAi+D2ngAA3A5iZ29tk3GPLkxtk3EBtI6dIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADA4obCUUFBgWJjY+VwOJScnKydO3detb6oqEjx8fFyOBxKSEjQtm3bbP2bNm3SmDFj1KNHDwUEBGj37t22/qNHjyogIMDnUVRUZNb56t+wYcON3CIAAOig/A5HGzduVHZ2tvLy8lRdXa3BgwfL7XbrxIkTPuvLy8uVkZGhrKws1dTUKC0tTWlpadq7d69Z09jYqFGjRmnRokU+x4iJidHx48dtx7PPPqtu3brpq1/9qq12zZo1trq0tDR/bxEAAHRgwf6esGzZMk2ePFmZmZmSpMLCQm3dulWrV6/W7Nmzr6hfsWKFxo4dq1mzZkmSFixYoNLSUq1cuVKFhYWSpAkTJkj6ZIfIl6CgIDmdTlvbq6++qm9+85vq1q2brT0iIuKKWgAAgOvl185Rc3OzqqqqlJKScnmAwEClpKSooqLC5zkVFRW2eklyu92t1l+Pqqoq7d69W1lZWVf0TZ06VZGRkRo+fLhWr14twzBaHaepqUler9d2AACAjs2vnaNTp07p4sWLioqKsrVHRUXpwIEDPs/xeDw+6z0ej59Tvezll1/WgAEDNHLkSFv7c889p6985Svq0qWLXn/9dX3/+9/Xhx9+qCeeeMLnOPn5+Xr22WdveB4AAOD24/fTau3t448/1vr16zVv3rwr+qxtQ4YMUWNjo5YsWdJqOMrJyVF2drb5tdfrVUxMzM2fNAAAuGX49bRaZGSkgoKCVF9fb2uvr69v9XU+TqfTr/pr+d///V999NFHmjhx4jVrk5OT9f7776upqclnf2hoqMLCwmwHAADo2PwKRyEhIUpKSlJZWZnZ1tLSorKyMrlcLp/nuFwuW70klZaWtlp/LS+//LLGjRunu+6665q1u3fv1p133qnQ0NAbuhYAAOh4/H5aLTs7W5MmTdLQoUM1fPhwLV++XI2Njea71yZOnKjevXsrPz9fkjRjxgyNHj1aS5cuVWpqqjZs2KBdu3Zp1apV5pinT59WbW2t6urqJEkHDx6U9Mmuk3WH6fDhw9qxY8cVn5MkSa+99prq6+s1YsQIORwOlZaW6sc//rGefvppf28RAAB0YH6Ho/T0dJ08eVK5ubnyeDxKTExUSUmJ+aLr2tpaBQZe3pAaOXKk1q9fr7lz52rOnDnq37+/iouLNWjQILNm8+bNZriSpPHjx0uS8vLyNH/+fLN99erV6tOnj8aMGXPFvDp16qSCggLNnDlThmHo7rvvNj92AAAA4HoFGFd7r3sH4/V6FR4eroaGhjZ5/VHs7K03fUxJOrowtU3GBXD9+P8baD83+/Gb360GAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABbB7T0BAPxGdwCfLx39ZxI7RwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAACLGwpHBQUFio2NlcPhUHJysnbu3HnV+qKiIsXHx8vhcCghIUHbtm2z9W/atEljxoxRjx49FBAQoN27d18xxpe//GUFBATYju9973u2mtraWqWmpqpLly7q2bOnZs2apQsXLtzILQIAgA7K73C0ceNGZWdnKy8vT9XV1Ro8eLDcbrdOnDjhs768vFwZGRnKyspSTU2N0tLSlJaWpr1795o1jY2NGjVqlBYtWnTVa0+ePFnHjx83j8WLF5t9Fy9eVGpqqpqbm1VeXq5169Zp7dq1ys3N9fcWAQBAB+Z3OFq2bJkmT56szMxMDRw4UIWFherSpYtWr17ts37FihUaO3asZs2apQEDBmjBggW6//77tXLlSrNmwoQJys3NVUpKylWv3aVLFzmdTvMICwsz+15//XX95S9/0a9+9SslJibqq1/9qhYsWKCCggI1Nzf7e5sAAKCD8iscNTc3q6qqyhZiAgMDlZKSooqKCp/nVFRUXBF63G53q/VX88orrygyMlKDBg1STk6OPvroI9t1EhISFBUVZbuO1+vVvn37fI7X1NQkr9drOwAAQMcW7E/xqVOndPHiRVsAkaSoqCgdOHDA5zkej8dnvcfj8Wui3/rWt9SvXz9FR0drz549euaZZ3Tw4EFt2rTpqte51OdLfn6+nn32Wb/mAQAAbm9+haP2NGXKFPO/ExIS1KtXLz344IP629/+pi984Qs3NGZOTo6ys7PNr71er2JiYj71XAEAwK3Lr6fVIiMjFRQUpPr6elt7fX29nE6nz3OcTqdf9dcrOTlZknT48OGrXudSny+hoaEKCwuzHQAAoGPzKxyFhIQoKSlJZWVlZltLS4vKysrkcrl8nuNyuWz1klRaWtpq/fW69Hb/Xr16mdd59913be+aKy0tVVhYmAYOHPiprgUAADoOv59Wy87O1qRJkzR06FANHz5cy5cvV2NjozIzMyVJEydOVO/evZWfny9JmjFjhkaPHq2lS5cqNTVVGzZs0K5du7Rq1SpzzNOnT6u2tlZ1dXWSpIMHD0qS+a60v/3tb1q/fr0eeugh9ejRQ3v27NHMmTP1wAMP6L777pMkjRkzRgMHDtSECRO0ePFieTwezZ07V1OnTlVoaOinWyUAANBh+B2O0tPTdfLkSeXm5srj8SgxMVElJSXmi59ra2sVGHh5Q2rkyJFav3695s6dqzlz5qh///4qLi7WoEGDzJrNmzeb4UqSxo8fL0nKy8vT/PnzFRISojfeeMMMYjExMXrkkUc0d+5c85ygoCBt2bJFjz/+uFwul7p27apJkybpueee839VAABAhxVgGIbR3pP4vPB6vQoPD1dDQ0ObvP4odvbWmz6mJB1dmNom4+Kzw/fGrY+/Q9xObrXv55v9+M3vVgMAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAItb5teHoHVt9a6CtsQ7cAAAn1fsHAEAAFgQjgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACxuKBwVFBQoNjZWDodDycnJ2rlz51Xri4qKFB8fL4fDoYSEBG3bts3Wv2nTJo0ZM0Y9evRQQECAdu/ebes/ffq0pk+frnvuuUedO3dW37599cQTT6ihocFWFxAQcMWxYcOGG7lFAADQQfkdjjZu3Kjs7Gzl5eWpurpagwcPltvt1okTJ3zWl5eXKyMjQ1lZWaqpqVFaWprS0tK0d+9es6axsVGjRo3SokWLfI5RV1enuro6Pf/889q7d6/Wrl2rkpISZWVlXVG7Zs0aHT9+3DzS0tL8vUUAANCBBft7wrJlyzR58mRlZmZKkgoLC7V161atXr1as2fPvqJ+xYoVGjt2rGbNmiVJWrBggUpLS7Vy5UoVFhZKkiZMmCBJOnr0qM9rDho0SP/3f/9nfv2FL3xB//3f/61vf/vbunDhgoKDL99GRESEnE6nv7cFAAAgyc+do+bmZlVVVSklJeXyAIGBSklJUUVFhc9zKioqbPWS5Ha7W62/Xg0NDQoLC7MFI0maOnWqIiMjNXz4cK1evVqGYbQ6RlNTk7xer+0AAAAdm187R6dOndLFixcVFRVla4+KitKBAwd8nuPxeHzWezweP6dqn8eCBQs0ZcoUW/tzzz2nr3zlK+rSpYtef/11ff/739eHH36oJ554wuc4+fn5evbZZ294HgAA4Pbj99Nq7c3r9So1NVUDBw7U/PnzbX3z5s0z/3vIkCFqbGzUkiVLWg1HOTk5ys7Oto0dExPTJvMGAAC3Br+eVouMjFRQUJDq6+tt7fX19a2+zsfpdPpVfzVnz57V2LFjdccdd+jVV19Vp06drlqfnJys999/X01NTT77Q0NDFRYWZjsAAEDH5lc4CgkJUVJSksrKysy2lpYWlZWVyeVy+TzH5XLZ6iWptLS01frWeL1ejRkzRiEhIdq8ebMcDsc1z9m9e7fuvPNOhYaG+nUtAADQcfn9tFp2drYmTZqkoUOHavjw4Vq+fLkaGxvNd69NnDhRvXv3Vn5+viRpxowZGj16tJYuXarU1FRt2LBBu3bt0qpVq8wxT58+rdraWtXV1UmSDh48KOmTXSen02kGo48++ki/+tWvbC+evuuuuxQUFKTXXntN9fX1GjFihBwOh0pLS/XjH/9YTz/99KdbIeD/i529tb2n4Le2nPPRhaltNjYAtCe/w1F6erpOnjyp3NxceTweJSYmqqSkxHzRdW1trQIDL29IjRw5UuvXr9fcuXM1Z84c9e/fX8XFxRo0aJBZs3nzZjNcSdL48eMlSXl5eZo/f76qq6tVWVkpSbr77rtt8zly5IhiY2PVqVMnFRQUaObMmTIMQ3fffbf5sQMAAADXK8C42nvdOxiv16vw8HDzYwJutltx56Gt3Iq7Dvz92d2Kf4dtqa2+P1hntIdb7fv5Zj9+87vVAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYBN/ISQUFBVqyZIk8Ho8GDx6sF198UcOHD2+1vqioSPPmzdPRo0fVv39/LVq0SA899JDZv2nTJhUWFqqqqkqnT59WTU2NEhMTbWOcO3dOTz31lDZs2KCmpia53W799Kc/VVRUlFlTW1urxx9/XNu3b1e3bt00adIk5efnKzj4hm4TbSh29tb2ngI+pVvx7/DowtT2ngI+p9ry+5nvu1uP3ztHGzduVHZ2tvLy8lRdXa3BgwfL7XbrxIkTPuvLy8uVkZGhrKws1dTUKC0tTWlpadq7d69Z09jYqFGjRmnRokWtXnfmzJl67bXXVFRUpLfeekt1dXX6xje+YfZfvHhRqampam5uVnl5udatW6e1a9cqNzfX31sEAAAdWIBhGIY/JyQnJ2vYsGFauXKlJKmlpUUxMTGaPn26Zs+efUV9enq6GhsbtWXLFrNtxIgRSkxMVGFhoa326NGjiouLu2LnqKGhQXfddZfWr1+vRx99VJJ04MABDRgwQBUVFRoxYoR++9vf6j/+4z9UV1dn7iYVFhbqmWee0cmTJxUSEnLNe/N6vQoPD1dDQ4PCwsL8WZbrciv+Sxu4nbTlv+Db6v9vdh0+G+wc2d1q3883+/Hbr52j5uZmVVVVKSUl5fIAgYFKSUlRRUWFz3MqKips9ZLkdrtbrfelqqpK58+ft40THx+vvn37muNUVFQoISHB9jSb2+2W1+vVvn37fI7b1NQkr9drOwAAQMfmVzg6deqULl68aAsgkhQVFSWPx+PzHI/H41d9a2OEhIQoIiKi1XFau86lPl/y8/MVHh5uHjExMdc9JwAAcHvq0O9Wy8nJUUNDg3m899577T0lAADQzvx6G1dkZKSCgoJUX19va6+vr5fT6fR5jtPp9Ku+tTGam5t15swZ2+6RdRyn06mdO3decZ1Lfb6EhoYqNDT0uucBAABuf37tHIWEhCgpKUllZWVmW0tLi8rKyuRyuXye43K5bPWSVFpa2mq9L0lJSerUqZNtnIMHD6q2ttYcx+Vy6d1337W9a660tFRhYWEaOHDgdV8LAAB0bH5/AFB2drYmTZqkoUOHavjw4Vq+fLkaGxuVmZkpSZo4caJ69+6t/Px8SdKMGTM0evRoLV26VKmpqdqwYYN27dqlVatWmWOePn1atbW1qqurk/RJ8JE+2fFxOp0KDw9XVlaWsrOz1b17d4WFhWn69OlyuVwaMWKEJGnMmDEaOHCgJkyYoMWLF8vj8Wju3LmaOnUqu0MAAOC6+R2O0tPTdfLkSeXm5srj8SgxMVElJSXmi59ra2sVGHh5Q2rkyJFav3695s6dqzlz5qh///4qLi7WoEGDzJrNmzeb4UqSxo8fL0nKy8vT/PnzJUk/+clPFBgYqEceecT2IZCXBAUFacuWLXr88cflcrnUtWtXTZo0Sc8995y/twgAADowvz/n6HbG5xwBtzc+5wit4XOO7G617+d2/ZwjAACA2x3hCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAIvg9p4AAHxWYmdvbe8p+K0t53x0YWqbjQ3cytg5AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADA4obCUUFBgWJjY+VwOJScnKydO3detb6oqEjx8fFyOBxKSEjQtm3bbP2GYSg3N1e9evVS586dlZKSokOHDpn9v//97xUQEODzeOeddyRJR48e9dn/9ttv38gtAgCADsrvcLRx40ZlZ2crLy9P1dXVGjx4sNxut06cOOGzvry8XBkZGcrKylJNTY3S0tKUlpamvXv3mjWLFy/WCy+8oMLCQlVWVqpr165yu906d+6cJGnkyJE6fvy47fjOd76juLg4DR061Ha9N954w1aXlJTk7y0CAIAOzO9wtGzZMk2ePFmZmZkaOHCgCgsL1aVLF61evdpn/YoVKzR27FjNmjVLAwYM0IIFC3T//fdr5cqVkj7ZNVq+fLnmzp2rhx9+WPfdd59++ctfqq6uTsXFxZKkkJAQOZ1O8+jRo4d+85vfKDMzUwEBAbbr9ejRw1bbqVMnf28RAAB0YH6Fo+bmZlVVVSklJeXyAIGBSklJUUVFhc9zKioqbPWS5Ha7zfojR47I4/HYasLDw5WcnNzqmJs3b9Y//vEPZWZmXtE3btw49ezZU6NGjdLmzZuvej9NTU3yer22AwAAdGx+haNTp07p4sWLioqKsrVHRUXJ4/H4PMfj8Vy1/tKf/oz58ssvy+12q0+fPmZbt27dtHTpUhUVFWnr1q0aNWqU0tLSrhqQ8vPzFR4ebh4xMTGt1gIAgI7hlvvFs++//75+97vf6de//rWtPTIyUtnZ2ebXw4YNU11dnZYsWaJx48b5HCsnJ8d2jtfrJSABANDB+bVzFBkZqaCgINXX19va6+vr5XQ6fZ7jdDqvWn/pz+sdc82aNerRo0ergccqOTlZhw8fbrU/NDRUYWFhtgMAAHRsfoWjkJAQJSUlqayszGxraWlRWVmZXC6Xz3NcLpetXpJKS0vN+ri4ODmdTluN1+tVZWXlFWMahqE1a9Zo4sSJ1/VC6927d6tXr17XfX8AAAB+P62WnZ2tSZMmaejQoRo+fLiWL1+uxsZG88XREydOVO/evZWfny9JmjFjhkaPHq2lS5cqNTVVGzZs0K5du7Rq1SpJUkBAgJ588kn96Ec/Uv/+/RUXF6d58+YpOjpaaWlptmu/+eabOnLkiL7zne9cMa9169YpJCREQ4YMkSRt2rRJq1ev1i9+8Qt/bxEAAHRgfoej9PR0nTx5Urm5ufJ4PEpMTFRJSYn5gura2loFBl7ekBo5cqTWr1+vuXPnas6cOerfv7+Ki4s1aNAgs+YHP/iBGhsbNWXKFJ05c0ajRo1SSUmJHA6H7dovv/yyRo4cqfj4eJ9zW7BggY4dO6bg4GDFx8dr48aNevTRR/29RQAA0IEFGIZhtPckPi+8Xq/Cw8PV0NDQJq8/ip299aaPCQA36ujC1PaewudGW/58vhXXua3Wo63W4mY/fvO71QAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsgtt7AgAAXK/Y2VvbewroANg5AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIDFDYWjgoICxcbGyuFwKDk5WTt37rxqfVFRkeLj4+VwOJSQkKBt27bZ+g3DUG5urnr16qXOnTsrJSVFhw4dstXExsYqICDAdixcuNBWs2fPHn3pS1+Sw+FQTEyMFi9efCO3BwAAOjC/w9HGjRuVnZ2tvLw8VVdXa/DgwXK73Tpx4oTP+vLycmVkZCgrK0s1NTVKS0tTWlqa9u7da9YsXrxYL7zwggoLC1VZWamuXbvK7Xbr3LlztrGee+45HT9+3DymT59u9nm9Xo0ZM0b9+vVTVVWVlixZovnz52vVqlX+3iIAAOjA/A5Hy5Yt0+TJk5WZmamBAweqsLBQXbp00erVq33Wr1ixQmPHjtWsWbM0YMAALViwQPfff79Wrlwp6ZNdo+XLl2vu3Ll6+OGHdd999+mXv/yl6urqVFxcbBvrjjvukNPpNI+uXbuafa+88oqam5u1evVq3XvvvRo/fryeeOIJLVu2zN9bBAAAHZhf4ai5uVlVVVVKSUm5PEBgoFJSUlRRUeHznIqKClu9JLndbrP+yJEj8ng8tprw8HAlJydfMebChQvVo0cPDRkyREuWLNGFCxds13nggQcUEhJiu87Bgwf1z3/+0+fcmpqa5PV6bQcAAOjYgv0pPnXqlC5evKioqChbe1RUlA4cOODzHI/H47Pe4/GY/ZfaWquRpCeeeEL333+/unfvrvLycuXk5Oj48ePmzpDH41FcXNwVY1zqu/POO6+YW35+vp599tlr3jcAAOg4/ApH7Sk7O9v87/vuu08hISH67ne/q/z8fIWGht7QmDk5ObZxvV6vYmJiPvVcAQDArcuvp9UiIyMVFBSk+vp6W3t9fb2cTqfPc5xO51XrL/3pz5iSlJycrAsXLujo0aNXvY71Gv8qNDRUYWFhtgMAAHRsfoWjkJAQJSUlqayszGxraWlRWVmZXC6Xz3NcLpetXpJKS0vN+ri4ODmdTluN1+tVZWVlq2NK0u7duxUYGKiePXua19mxY4fOnz9vu84999zj8yk1AAAAX/x+t1p2drZ+/vOfa926ddq/f78ef/xxNTY2KjMzU5I0ceJE5eTkmPUzZsxQSUmJli5dqgMHDmj+/PnatWuXpk2bJkkKCAjQk08+qR/96EfavHmz3n33XU2cOFHR0dFKS0uT9MmLrZcvX64///nP+vvf/65XXnlFM2fO1Le//W0z+HzrW99SSEiIsrKytG/fPm3cuFErVqywPW0GAABwLX6/5ig9PV0nT55Ubm6uPB6PEhMTVVJSYr74uba2VoGBlzPXyJEjtX79es2dO1dz5sxR//79VVxcrEGDBpk1P/jBD9TY2KgpU6bozJkzGjVqlEpKSuRwOCR98vTXhg0bNH/+fDU1NSkuLk4zZ860BZ/w8HC9/vrrmjp1qpKSkhQZGanc3FxNmTLlhhcHAAB0PAGGYRjtPYnPC6/Xq/DwcDU0NLTJ649iZ2+96WMCwI06ujC1vafgt1vx5yjrfFlbrcXNfvzmd6sBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFj4/etDAAC4mlvxU6xvRaxz22HnCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABY3FI4KCgoUGxsrh8Oh5ORk7dy586r1RUVFio+Pl8PhUEJCgrZt22brNwxDubm56tWrlzp37qyUlBQdOnTI7D969KiysrIUFxenzp076wtf+ILy8vLU3NxsqwkICLjiePvtt2/kFgEAQAfldzjauHGjsrOzlZeXp+rqag0ePFhut1snTpzwWV9eXq6MjAxlZWWppqZGaWlpSktL0969e82axYsX64UXXlBhYaEqKyvVtWtXud1unTt3TpJ04MABtbS06Gc/+5n27dunn/zkJyosLNScOXOuuN4bb7yh48ePm0dSUpK/twgAADqwAMMwDH9OSE5O1rBhw7Ry5UpJUktLi2JiYjR9+nTNnj37ivr09HQ1NjZqy5YtZtuIESOUmJiowsJCGYah6OhoPfXUU3r66aclSQ0NDYqKitLatWs1fvx4n/NYsmSJXnrpJf3973+X9MnOUVxcnGpqapSYmOjPLZm8Xq/Cw8PV0NCgsLCwGxrjamJnb73pYwLAjTq6MLVNxuVnnR3rfFlbrcXNfvz2a+eoublZVVVVSklJuTxAYKBSUlJUUVHh85yKigpbvSS53W6z/siRI/J4PLaa8PBwJScntzqm9EmA6t69+xXt48aNU8+ePTVq1Cht3rz5qvfT1NQkr9drOwAAQMfmVzg6deqULl68qKioKFt7VFSUPB6Pz3M8Hs9V6y/96c+Yhw8f1osvvqjvfve7Zlu3bt20dOlSFRUVaevWrRo1apTS0tKuGpDy8/MVHh5uHjExMa3WAgCAjiG4vSfgrw8++EBjx47Vf/7nf2ry5Mlme2RkpLKzs82vhw0bprq6Oi1ZskTjxo3zOVZOTo7tHK/XS0ACAKCD82vnKDIyUkFBQaqvr7e119fXy+l0+jzH6XRetf7Sn9czZl1dnf793/9dI0eO1KpVq6453+TkZB0+fLjV/tDQUIWFhdkOAADQsfkVjkJCQpSUlKSysjKzraWlRWVlZXK5XD7PcblctnpJKi0tNevj4uLkdDptNV6vV5WVlbYxP/jgA335y19WUlKS1qxZo8DAa0999+7d6tWrlz+3CAAAOji/n1bLzs7WpEmTNHToUA0fPlzLly9XY2OjMjMzJUkTJ05U7969lZ+fL0maMWOGRo8eraVLlyo1NVUbNmzQrl27zJ2fgIAAPfnkk/rRj36k/v37Ky4uTvPmzVN0dLTS0tIkXQ5G/fr10/PPP6+TJ0+a87m0u7Ru3TqFhIRoyJAhkqRNmzZp9erV+sUvfnHjqwMAADocv8NRenq6Tp48qdzcXHk8HiUmJqqkpMR8QXVtba1tV2fkyJFav3695s6dqzlz5qh///4qLi7WoEGDzJof/OAHamxs1JQpU3TmzBmNGjVKJSUlcjgckj7ZaTp8+LAOHz6sPn362OZj/SSCBQsW6NixYwoODlZ8fLw2btyoRx991N9bBAAAHZjfn3N0O+NzjgB0JHz+zmeDdb7stvycIwAAgNvdLfdWfgDAzXEr7jwAnwV2jgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYEE4AgAAsCAcAQAAWBCOAAAALAhHAAAAFoQjAAAAC8IRAACABeEIAADAgnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFgQjgAAACwIRwAAABaEIwAAAAvCEQAAgAXhCAAAwIJwBAAAYHFD4aigoECxsbFyOBxKTk7Wzp07r1pfVFSk+Ph4ORwOJSQkaNu2bbZ+wzCUm5urXr16qXPnzkpJSdGhQ4dsNadPn9Zjjz2msLAwRUREKCsrSx9++KGtZs+ePfrSl74kh8OhmJgYLV68+EZuDwAAdGB+h6ONGzcqOztbeXl5qq6u1uDBg+V2u3XixAmf9eXl5crIyFBWVpZqamqUlpamtLQ07d2716xZvHixXnjhBRUWFqqyslJdu3aV2+3WuXPnzJrHHntM+/btU2lpqbZs2aIdO3ZoypQpZr/X69WYMWPUr18/VVVVacmSJZo/f75WrVrl7y0CAIAOLMAwDMOfE5KTkzVs2DCtXLlSktTS0qKYmBhNnz5ds2fPvqI+PT1djY2N2rJli9k2YsQIJSYmqrCwUIZhKDo6Wk899ZSefvppSVJDQ4OioqK0du1ajR8/Xvv379fAgQP1zjvvaOjQoZKkkpISPfTQQ3r//fcVHR2tl156ST/84Q/l8XgUEhIiSZo9e7aKi4t14MCB67o3r9er8PBwNTQ0KCwszJ9luS6xs7fe9DEBAJ9vRxemtsm4t+JjSlutxc1+/A72p7i5uVlVVVXKyckx2wIDA5WSkqKKigqf51RUVCg7O9vW5na7VVxcLEk6cuSIPB6PUlJSzP7w8HAlJyeroqJC48ePV0VFhSIiIsxgJEkpKSkKDAxUZWWlvv71r6uiokIPPPCAGYwuXWfRokX65z//qTvvvPOKuTU1Nampqcn8uqGhQdIni9wWWpo+apNxAQCfXzymXNZWa3FpXD/3e1rlVzg6deqULl68qKioKFt7VFRUq7szHo/HZ73H4zH7L7VdraZnz572iQcHq3v37raauLi4K8a41OcrHOXn5+vZZ5+9oj0mJsbnvQAA4K/w5e09g8+Ptl6Ls2fPKjw8/FOP41c4ut3k5OTYdrVaWlp0+vRp9ejRQwEBATf1Wl6vVzExMXrvvffa5Ck7XIk1bx+se/tg3dsH694+/nXdDcPQ2bNnFR0dfVPG9yscRUZGKigoSPX19bb2+vp6OZ1On+c4nc6r1l/6s76+Xr169bLVJCYmmjX/+oLvCxcu6PTp07ZxfF3Heo1/FRoaqtDQUFtbRESEz9qbJSwsjP+BPmOseftg3dsH694+WPf2YV33m7FjdIlf71YLCQlRUlKSysrKzLaWlhaVlZXJ5XL5PMflctnqJam0tNSsj4uLk9PptNV4vV5VVlaaNS6XS2fOnFFVVZVZ8+abb6qlpUXJyclmzY4dO3T+/Hnbde655x6fT6kBAAD4ZPhpw4YNRmhoqLF27VrjL3/5izFlyhQjIiLC8Hg8hmEYxoQJE4zZs2eb9X/605+M4OBg4/nnnzf2799v5OXlGZ06dTLeffdds2bhwoVGRESE8Zvf/MbYs2eP8fDDDxtxcXHGxx9/bNaMHTvWGDJkiFFZWWn88Y9/NPr3729kZGSY/WfOnDGioqKMCRMmGHv37jU2bNhgdOnSxfjZz37m7y22iYaGBkOS0dDQ0N5T6TBY8/bBurcP1r19sO7to63X3e9wZBiG8eKLLxp9+/Y1QkJCjOHDhxtvv/222Td69Ghj0qRJtvpf//rXxhe/+EUjJCTEuPfee42tW7fa+ltaWox58+YZUVFRRmhoqPHggw8aBw8etNX84x//MDIyMoxu3boZYWFhRmZmpnH27FlbzZ///Gdj1KhRRmhoqNG7d29j4cKFN3J7beLcuXNGXl6ece7cufaeSofBmrcP1r19sO7tg3VvH2297n5/zhEAAMDtjN+tBgAAYEE4AgAAsCAcAQAAWBCOAAAALAhHn8KOHTv0ta99TdHR0QoICDB/X9wlhmEoNzdXvXr1UufOnZWSkqJDhw7Zak6fPq3HHntMYWFhioiIUFZWlj788MPP8C5uPVdb9/Pnz+uZZ55RQkKCunbtqujoaE2cOFF1dXW2MVh3/13r+93qe9/7ngICArR8+XJbO+vuv+tZ9/3792vcuHEKDw9X165dNWzYMNXW1pr9586d09SpU9WjRw9169ZNjzzyyBUfmgu7a637hx9+qGnTpqlPnz7q3LmzBg4cqMLCQlsN6+6f/Px8DRs2THfccYd69uyptLQ0HTx40FZzPWtaW1ur1NRUdenSRT179tSsWbN04cIFv+ZCOPoUGhsbNXjwYBUUFPjsX7x4sV544QUVFhaqsrJSXbt2ldvt1rlz58yaxx57TPv27VNpaam2bNmiHTt2aMqUKZ/VLdySrrbuH330kaqrqzVv3jxVV1dr06ZNOnjwoMaNG2erY939d63v90teffVVvf322z4/xp9199+11v1vf/ubRo0apfj4eP3+97/Xnj17NG/ePDkcDrNm5syZeu2111RUVKS33npLdXV1+sY3vvFZ3cIt6Vrrnp2drZKSEv3qV7/S/v379eSTT2ratGnavHmzWcO6++ett97S1KlT9fbbb6u0tFTnz5/XmDFj1NjYaNZca00vXryo1NRUNTc3q7y8XOvWrdPatWuVm5vr32Ta5AMCOiBJxquvvmp+3dLSYjidTmPJkiVm25kzZ4zQ0FDjf/7nfwzDMIy//OUvhiTjnXfeMWt++9vfGgEBAcYHH3zwmc39Vvav6+7Lzp07DUnGsWPHDMNg3W+G1tb9/fffN3r37m3s3bvX6Nevn/GTn/zE7GPdPz1f656enm58+9vfbvWcM2fOGJ06dTKKiorMtv379xuSjIqKiraa6m3F17rfe++9xnPPPWdru//++40f/vCHhmGw7jfDiRMnDEnGW2+9ZRjG9a3ptm3bjMDAQPODqQ3DMF566SUjLCzMaGpquu5rs3PURo4cOSKPx6OUlBSzLTw8XMnJyaqoqJAkVVRUKCIiQkOHDjVrUlJSFBgYqMrKys98zrerhoYGBQQEmL83j3VvGy0tLZowYYJmzZqle++994p+1v3ma2lp0datW/XFL35RbrdbPXv2VHJysu0poKqqKp0/f972syg+Pl59+/Y1fxbBfyNHjtTmzZv1wQcfyDAMbd++XX/96181ZswYSaz7zdDQ0CBJ6t69u6TrW9OKigolJCQoKirKrHG73fJ6vdq3b991X5tw1EY8Ho8k2f6CLn19qc/j8ahnz562/uDgYHXv3t2swadz7tw5PfPMM8rIyDB/OSHr3jYWLVqk4OBgPfHEEz77Wfeb78SJE/rwww+1cOFCjR07Vq+//rq+/vWv6xvf+IbeeustSZ+se0hIyBW/VNv6swj+e/HFFzVw4ED16dNHISEhGjt2rAoKCvTAAw9IYt0/rZaWFj355JP6t3/7Nw0aNEjS9a2px+Px+bh7qe96BX+KuQOfa+fPn9c3v/lNGYahl156qb2nc1urqqrSihUrVF1drYCAgPaeTofR0tIiSXr44Yc1c+ZMSVJiYqLKy8tVWFio0aNHt+f0bmsvvvii3n77bW3evFn9+vXTjh07NHXqVEVHR9t2NnBjpk6dqr179+qPf/xju1yfnaM24nQ6JemKV9HX19ebfU6nUydOnLD1X7hwQadPnzZrcGMuBaNjx46ptLTU3DWSWPe28Ic//EEnTpxQ3759FRwcrODgYB07dkxPPfWUYmNjJbHubSEyMlLBwcEaOHCgrX3AgAHmu9WcTqeam5t15swZW431ZxH88/HHH2vOnDlatmyZvva1r+m+++7TtGnTlJ6erueff14S6/5pTJs2TVu2bNH27dvVp08fs/161tTpdPp83L3Ud70IR20kLi5OTqdTZWVlZpvX61VlZaVcLpckyeVy6cyZM6qqqjJr3nzzTbW0tCg5Ofkzn/Pt4lIwOnTokN544w316NHD1s+633wTJkzQnj17tHv3bvOIjo7WrFmz9Lvf/U4S694WQkJCNGzYsCve7vzXv/5V/fr1kyQlJSWpU6dOtp9FBw8eVG1trfmzCP45f/68zp8/r8BA+0NoUFCQuZvHuvvPMAxNmzZNr776qt58803FxcXZ+q9nTV0ul959913bP8Qu/QP5X/8Rca3J4AadPXvWqKmpMWpqagxJxrJly4yamhrzXVELFy40IiIijN/85jfGnj17jIcfftiIi4szPv74Y3OMsWPHGkOGDDEqKyuNP/7xj0b//v2NjIyM9rqlW8LV1r25udkYN26c0adPH2P37t3G8ePHzcP6TgXW3X/X+n7/V//6bjXDYN1vxLXWfdOmTUanTp2MVatWGYcOHTJefPFFIygoyPjDH/5gjvG9733P6Nu3r/Hmm28au3btMlwul+Fyudrrlm4J11r30aNHG/fee6+xfft24+9//7uxZs0aw+FwGD/96U/NMVh3/zz++ONGeHi48fvf/972s/ujjz4ya661phcuXDAGDRpkjBkzxti9e7dRUlJi3HXXXUZOTo5fcyEcfQrbt283JF1xTJo0yTCMT97OP2/ePCMqKsoIDQ01HnzwQePgwYO2Mf7xj38YGRkZRrdu3YywsDAjMzPTOHv2bDvcza3jaut+5MgRn32SjO3bt5tjsO7+u9b3+7/yFY5Yd/9dz7q//PLLxt133204HA5j8ODBRnFxsW2Mjz/+2Pj+979v3HnnnUaXLl2Mr3/968bx48c/4zu5tVxr3Y8fP27813/9lxEdHW04HA7jnnvuMZYuXWq0tLSYY7Du/mntZ/eaNWvMmutZ06NHjxpf/epXjc6dOxuRkZHGU089ZZw/f96vuQT8/wkBAABAvOYIAADAhnAEAABgQTgCAACwIBwBAABYEI4AAAAsCEcAAAAWhCMAAAALwhEAAIAF4QgAAMCCcAQAAGBBOAIAALAgHAEAAFj8P2LkzmwwxhinAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(df_medium_length.word_count, density=True, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101 doctrine definition's median length -> 136.0\n",
      "101 doctrine definition's mean length -> 140.1089108910891\n"
     ]
    }
   ],
   "source": [
    "print(\"101 doctrine definition's mean length ->\", df_medium_length.word_count.mean())\n",
    "print(\"101 doctrine definition's median length ->\", df_medium_length.word_count.median())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample 20 doctrines from 101 for detailed evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>concept</th>\n",
       "      <th>intro_text</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Abstention_doctrine</td>\n",
       "      <td>An abstention doctrine is any of several doctr...</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>Constitutional_convention_(political_custom)</td>\n",
       "      <td>A constitutional convention is an informal and...</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>94</td>\n",
       "      <td>Unaccompanied_minor</td>\n",
       "      <td>An unaccompanied minor (sometimes \"unaccompani...</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>90</td>\n",
       "      <td>Substantial_certainty_doctrine</td>\n",
       "      <td>In law, the substantial certainty doctrine is ...</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50</td>\n",
       "      <td>Learned_intermediary</td>\n",
       "      <td>Learned intermediary is a defense doctrine use...</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>Doctrine_of_foreign_equivalents</td>\n",
       "      <td>This article discusses the trademark doctrine ...</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>78</td>\n",
       "      <td>Prosecutorial_discretion</td>\n",
       "      <td>In common law, the principle of prosecutorial ...</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>87</td>\n",
       "      <td>Safe_harbor_(law)</td>\n",
       "      <td>A safe harbor is a provision of a statute or a...</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>35</td>\n",
       "      <td>FTC_v._Dean_Foods_Co.</td>\n",
       "      <td>FTC v. Dean Foods Co., 384 U.S. 597 (1966), is...</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100</td>\n",
       "      <td>Volenti_non_fit_injuria</td>\n",
       "      <td>Volenti non fit iniuria (or injuria) (Latin: \"...</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>70</td>\n",
       "      <td>Plain_meaning_rule</td>\n",
       "      <td>The plain meaning rule, also known as the lite...</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>79</td>\n",
       "      <td>Purcell_principle</td>\n",
       "      <td>In United States law, the Purcell principle is...</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>73</td>\n",
       "      <td>Presumption_(Catholic_canon_law)</td>\n",
       "      <td>Presumption in the canon law of the Catholic C...</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>101</td>\n",
       "      <td>Wrongdoing</td>\n",
       "      <td>A wrong (from Old English wrang – 'crooked') i...</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>83</td>\n",
       "      <td>Remoteness_in_English_law</td>\n",
       "      <td>In English law, remoteness between a cause of ...</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>66</td>\n",
       "      <td>Pacta_sunt_servanda</td>\n",
       "      <td>Pacta sunt servanda, Latin for \"agreements mus...</td>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>55</td>\n",
       "      <td>Maxwellisation</td>\n",
       "      <td>Maxwellisation is the legal practice in Englis...</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>7</td>\n",
       "      <td>Brady_disclosure</td>\n",
       "      <td>In the legal system of the United States, a Br...</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>96</td>\n",
       "      <td>Undue_hardship</td>\n",
       "      <td>An undue hardship is an American legal term re...</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9</td>\n",
       "      <td>Commanding_precedent</td>\n",
       "      <td>In law, a commanding precedent is a precedent ...</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0                                       concept  \\\n",
       "0            0                           Abstention_doctrine   \n",
       "1           11  Constitutional_convention_(political_custom)   \n",
       "2           94                           Unaccompanied_minor   \n",
       "3           90                Substantial_certainty_doctrine   \n",
       "4           50                          Learned_intermediary   \n",
       "5           18               Doctrine_of_foreign_equivalents   \n",
       "6           78                      Prosecutorial_discretion   \n",
       "7           87                             Safe_harbor_(law)   \n",
       "8           35                         FTC_v._Dean_Foods_Co.   \n",
       "9          100                       Volenti_non_fit_injuria   \n",
       "10          70                            Plain_meaning_rule   \n",
       "11          79                             Purcell_principle   \n",
       "12          73              Presumption_(Catholic_canon_law)   \n",
       "13         101                                    Wrongdoing   \n",
       "14          83                     Remoteness_in_English_law   \n",
       "15          66                           Pacta_sunt_servanda   \n",
       "16          55                                Maxwellisation   \n",
       "17           7                              Brady_disclosure   \n",
       "18          96                                Undue_hardship   \n",
       "19           9                          Commanding_precedent   \n",
       "\n",
       "                                           intro_text  word_count  \n",
       "0   An abstention doctrine is any of several doctr...         159  \n",
       "1   A constitutional convention is an informal and...         179  \n",
       "2   An unaccompanied minor (sometimes \"unaccompani...         105  \n",
       "3   In law, the substantial certainty doctrine is ...         103  \n",
       "4   Learned intermediary is a defense doctrine use...         105  \n",
       "5   This article discusses the trademark doctrine ...         129  \n",
       "6   In common law, the principle of prosecutorial ...         104  \n",
       "7   A safe harbor is a provision of a statute or a...         153  \n",
       "8   FTC v. Dean Foods Co., 384 U.S. 597 (1966), is...         119  \n",
       "9   Volenti non fit iniuria (or injuria) (Latin: \"...         196  \n",
       "10  The plain meaning rule, also known as the lite...         153  \n",
       "11  In United States law, the Purcell principle is...         100  \n",
       "12  Presumption in the canon law of the Catholic C...         158  \n",
       "13  A wrong (from Old English wrang – 'crooked') i...         117  \n",
       "14  In English law, remoteness between a cause of ...         121  \n",
       "15  Pacta sunt servanda, Latin for \"agreements mus...         115  \n",
       "16  Maxwellisation is the legal practice in Englis...         127  \n",
       "17  In the legal system of the United States, a Br...         146  \n",
       "18  An undue hardship is an American legal term re...         122  \n",
       "19  In law, a commanding precedent is a precedent ...         136  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_sampled_20 = df_medium_length.sample(frac=0.2)\n",
    "# df_sampled_20.to_csv(\"../data/20-doctrines/legal_doctrines_20.tsv\", sep=\"\\t\")\n",
    "df_sampled_20 = pd.read_csv(\"../data/20-doctrines/legal_doctrines_20.tsv\", sep=\"\\t\")\n",
    "df_sampled_20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 doctrine definition's mean length -> 132.35\n",
      "20 doctrine definition's median length -> 124.5\n"
     ]
    }
   ],
   "source": [
    "print(\"20 doctrine definition's mean length ->\", df_sampled_20.word_count.mean())\n",
    "print(\"20 doctrine definition's median length ->\", df_sampled_20.word_count.median())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "audiencenlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}