diff --git a/fig/rnn_classifier.png b/fig/rnn_classifier.png new file mode 100644 index 0000000..0e4fd05 Binary files /dev/null and b/fig/rnn_classifier.png differ diff --git a/fig/tree_nn.png b/fig/tree_nn.png new file mode 100644 index 0000000..9b471d2 Binary files /dev/null and b/fig/tree_nn.png differ diff --git a/sgd_classifier.py b/sgd_classifier.py new file mode 100644 index 0000000..8aada14 --- /dev/null +++ b/sgd_classifier.py @@ -0,0 +1,178 @@ +import numpy as np +import random + +__author__ = "Christopher Potts" +__version__ = "CS224u, Stanford, Spring 2018 term" + + +class BasicSGDClassifier: + """Basic implementation hinge-loss stochastic sub-gradient descent + optimization, intended to illustrate the basic concepts of classifier + optimization in code.""" + def __init__(self, max_iter=10, eta=0.1): + """ + Parameters + ---------- + max_iter : int (default: 10) + Number of training epochs (full runs through shuffled data). + eta : float (default: 0.1) + Learning rate parameter. + + """ + self.max_iter = max_iter + self.eta = eta + self.params = ['max_iter', 'eta'] + + def fit(self, feat_matrix, labels): + """Core optimization function. + + Parameters + ---------- + feat_matrix : 2d matrix (np.array or any scipy.sparse type) + The design matrix, one row per example. Hence, the row + dimensionality is the example count and the column + dimensionality is number of features. + + labels : list + The labels for each example, hence assumed to have the + same length as, and be aligned with, `feat_matrix`. + + For attributes, we follow the `sklearn` style of using a + final `_` for attributes that are created by `fit` methods: + + Attributes + ---------- + self.classes_ : list + The set of class labels in sorted order. + + self.n_classes_ : int + Length of `self.classes_` + + self.coef_ : np.array of dimension (class count, feature count) + These are the weights, named as in `sklearn`. They are + organized so that each row represents the feature weights + for a given class, as is typical in `sklearn`. + + """ + # We'll deal with the labels via their indices into self.classes_: + self.classes_ = sorted(set(labels)) + self.n_classes_ = len(self.classes_) + # Useful dimensions to store: + examplecount, featcount = feat_matrix.shape + # The weight matrix -- classes by row: + self.coef_ = np.zeros((self.n_classes_, featcount)) + # Indices for shuffling the data at the start of each epoch: + indices = list(range(examplecount)) + for _ in range(self.max_iter): + random.shuffle(indices) + for i in indices: + # Training instance as a feature rep and a label index: + rep = feat_matrix[i] + label_index = self.classes_.index(labels[i]) + # Costs are 1.0 except for the true label: + costs = np.ones(self.n_classes_) + costs[label_index] = 0.0 + # Make a prediction: + predicted_index = self.predict_one(rep, costs=costs) + # Weight update if it's an incorrect prediction: + if predicted_index != label_index: + self.coef_[label_index] += self.eta * rep + + def predict_one(self, rep, costs=0.0): + """The core classification function. After using + `predict_one_proba`, the code just needs to figure out which + class is highest scoring and make a random choice from that + set (in case of ties). + + Parameters + ---------- + rep : np.array of dimension featcount or + `scipy.sparse` matrix of dimension (1 x `featcount`) + + costs : float or np.array of dimension self.classcount + Where this is 0.0, we're doing prediction. Where it + is an array, we expect a 0.0 at the coordinate + corresponding to the true label and a 1.0 in all + other positions. + + Returns + ------- + int + The index of the correct class. This is for the + sake of the `fit` method. `predict` returns the class + names themselves. + + """ + scores = rep.dot(self.coef_.T) + costs + # Manage the difference between scipy and numpy 1d matrices: + scores = scores.reshape(self.n_classes_) + # Set of highest scoring label indices (in case of ties): + candidates = np.argwhere(scores==np.max(scores)).flatten() + return random.choice(candidates) + + def predict(self, reps): + """Batch prediction function for experiments. + + Parameters + ---------- + reps : list or feature matrix + A featurized set of examples to make predictions about. + + Returns + ------- + list of str + A list of class names -- the predictions. Unlike `predict_one`, + it returns the class name rather than its index. + + """ + return [self.classes_[self.predict_one(rep)] for rep in reps] + + def get_params(self, deep=True): + """Gets the hyperparameters for the model, as given by the + `self.params` attribute. This is called `get_params` for + compatibility with sklearn. + + Returns + ------- + dict + Map from attribute names to their values. + + """ + return {p: getattr(self, p) for p in self.params} + + def set_params(self, **params): + for key, val in params.items(): + setattr(self, key, val) + return self + + +def simple_example(): + """Assess on the digits dataset and informally compare + against LogisticRegression. + """ + from sklearn.datasets import load_digits + from sklearn.model_selection import train_test_split + from sklearn.metrics import classification_report + from sklearn.linear_model import LogisticRegression + + digits = load_digits() + X = digits.data + y = digits.target + + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.33, random_state=42) + + models = [ + BasicSGDClassifier(max_iter=500), + LogisticRegression() + ] + + for mod in models: + print(mod) + mod.fit(X_train, y_train) + predictions = mod.predict(X_test) + print(classification_report(y_test, predictions)) + + +if __name__ == '__main__': + simple_example() diff --git a/sst_01_overview.ipynb b/sst_01_overview.ipynb new file mode 100644 index 0000000..56e2445 --- /dev/null +++ b/sst_01_overview.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Supervised sentiment: Overview of the Stanford Sentiment Treebank" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "__author__ = \"Christopher Potts\"\n", + "__version__ = \"CS224u, Stanford, Spring 2018 term\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## Contents\n", + "\n", + "0. [Overview of this unit](#Overview-of-this-unit)\n", + "0. [Paths through the material](#Paths-through-the-material)\n", + "0. [Overview of this notebook](#Overview-of-this-notebook)\n", + "0. [The complexity of sentiment analysis](#The-complexity-of-sentiment-analysis)\n", + "0. [Set-up](#Set-up)\n", + "0. [Data readers](#Data-readers)\n", + " 0. [Main readers](#Main-readers)\n", + " 0. [All-nodes readers](#All-nodes-readers)\n", + " 0. [Methodological notes](#Methodological-notes)\n", + "0. [Modeling the SST labels](#Modeling-the-SST-labels)\n", + " 0. [Train label distributions](#Train-label-distributions)\n", + " 0. [Dev label distributions](#Dev-label-distributions)\n", + "0. [Additional sentiment resources](#Additional-sentiment-resources)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Overview of this unit\n", + "\n", + "We have a few inter-related goals for this unit:\n", + "\n", + "* Provide a basic introduction to supervised learning in the context of a problem that has long been central to academic research and industry applications: __sentiment analysis__.\n", + "\n", + "* Explore and evaluate a diverse array of methods for modeling sentiment:\n", + " * Hand-built feature functions with (mostly linear) classifiers\n", + " * Dense feature representations derived from VSMs as we built them in the previous unit\n", + " * Recurrent neural networks (RNNs)\n", + " * Tree-structured neural networks\n", + " \n", + "* Begin discussing and implementing responsible methods for __hyperparameter optimization__ and __classifier assessment and comparison__.\n", + "\n", + "The unit is built around the [Stanford Sentiment Treebank (SST)](http://nlp.stanford.edu/sentiment/), a widely-used resource for evaluating supervised NLU models, and one that provides rich linguistic representations." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Paths through the material\n", + "\n", + "* If you're relatively new to supervised learning, we suggest studying the details of this notebook closely and following the links to [additional resources](#Additional-sentiment-resources). \n", + "\n", + "* If you're familiar with supervised learning, then you can focus right away on innovative feature representations and modeling. \n", + "\n", + "* As of this writing, the state-of-the-art for the SST seems to be around 88% accuracy for the binary problem and 48% accuracy for the five-class problem. Perhaps you can best these numbers!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Overview of this notebook\n", + "\n", + "This is the first notebook in this unit. It does two things:\n", + "\n", + "* Introduces sentiment analysis as a task.\n", + "* Introduces the SST and our tools for reading that corpus. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## The complexity of sentiment analysis\n", + "\n", + "Sentiment analysis seems simple at first but turns out to exhibit all of the complexity of full natural language understanding. To see this, consider how your intuitions about the sentiment of the following sentences can change depending on perspective, social relationships, tone of voice, and other aspects of the context of utterance:\n", + "\n", + "1. There was an earthquake in LA.\n", + "1. The team failed the physical challenge. (We win/lose!)\n", + "1. They said it would be great. They were right/wrong.\n", + "1. Many consider the masterpiece bewildering, boring, slow-moving or annoying.\n", + "1. The party fat-cats are sipping their expensive, imported wines.\n", + "1. Oh, you're terrible!\n", + "\n", + "SST mostly steers around these challenges by including only focused, evaluative texts (sentences from movie reviews), but you should have them in mind if you consider new domains and applications for the ideas." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Set-up\n", + "\n", + "* Make sure your environment includes all the requirements for [the cs224u repository](https://github.com/cgpotts/cs224u).\n", + "\n", + "* Download [the train/dev/test Stanford Sentiment Treebank distribution](http://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip), unzip it, and put the resulting folder in the same directory as this notebook. It will be called `trees`. (If you want to put it somewhere else, change sst_home below.)\n", + "\n", + "* Make sure you still have the `vsmdata` directory and its contents. ([Here's a link in case you need to redownload it.](http://web.stanford.edu/class/cs224u/data/vsmdata.zip)) In addition, you might want the [the Wikipedia 2014 + Gigaword 5 distribution of the pretrained GloVe vectors](http://nlp.stanford.edu/data/glove.6B.zip). This might already be in `vsmdata`, depending on what kind of work you did as part of the VSM unit." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from nltk.tree import Tree\n", + "import pandas as pd\n", + "import sst" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Data readers\n", + "\n", + "* The train/dev/test SST distribution contains files that are lists of trees where the part-of-speech tags have been replaced with sentiment scores `0...4`:\n", + " * `0` and `1` are negative labels.\n", + " * `2` is a neutral label.\n", + " * `3` and `4` are positive labels. \n", + "\n", + "* Our readers are iteratorrs that yield `(tree, label)` pairs, where `tree` is an [NLTK Tree](http://www.nltk.org/_modules/nltk/tree.html) instance and `score` is a string." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Main readers\n", + "\n", + "We'll mainly work with `sst.train_reader` and `sst.dev_reader`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tree, score = next(sst.train_reader())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, `score` is one of the labels. `tree` is an NLTK Tree instance. It should render pretty legibly in your browser:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAB2AAAALRCAIAAAACyCQVAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAHXRFWHRTb2Z0d2FyZQBHUEwgR2hvc3RzY3JpcHQgOS4xNnO9PXQAACAASURBVHic7N1PbCNpmh/oqNrEeCzN9m60LdkG3CuCDdiABBvGsnRcZB5ILFC1gC9FXXv6QgGGj9UWb9NzI921N2MAxqWrb4tkHbcKBsSD8raGKhaGdyXsAJ4AhRljDNGYgL0rYTA2nHv4slksScnUH4oRZDwPEgky+O9liBlM/vTy/T56+/ZtBAAAAABA9XxcdAEAAAAAABRDQAwAAAAAUFECYgAAAACAihIQAwAAAABUlIAYAAAAAKCiXhRdAACVk2VZlmVRFDUajSiK4jguuiIAAACoqI/evn1bdA0AVEiSJKPRqN1u53k+Go2yLPvuu++KLgoAAAAqSkAMwFK1Wq3j4+NwOs/zTz755E/+5E+KLQkAAAAqywxiAJYny7J6vT49G8fx69evC6wHAAAAKk5ADMDyhHS43++HGcTRb8cQAwAAAIUwYgKAZRuNRmH6cBzHnU5HRgwAAABFERADUJg8zw8ODl6/fh3HcdG1AAAAQBUZMQHA8gyHw+lwiSiK4jhuNBppmhZYEgAAAFSZgBiA5UnTdDgczm7JssyICQAAACjKi6ILAKBa8jw/PDwMMyVGo1Gn0zFfAgAAAIpiBjEAy5bneRgr0Ww2i64FAAAAKk1ADAAAAABQUWYQAwAAAABUlIAYAAAAAKCiLFIHwGLk19fpeHxz49VVenFxe2M2mbz4+OP/8l//a7yxUd/evnGF+tZWfWvrPhsBAACApzCDGKCisskkm0xubEzH4/z6+vbG2zcfnZ094kGbe3vT0yEg/sFDX1099A4btVq8uXl74x0Pvbt7Y0u8udnY2XnoIwIAAMA6ERADrIbR+fmNLXc250Z35bnZZJJdXj70Eevb27c7du/MXhs7O7dT2tuB7COkFxe3U+PbuyK661nnV1d3RtvzxZubt59jfWvr9hO8s6O5UavFGxsPfVAAAAAoioAYYPHubM69e+Pl5Z0du4/opZ1tzp26nXXGGxt3bKxML60fDQAAAMwSEANoU+VJqtncDQAAwHoQEAMr5qErod3YaNAt5Vf4eOjA+oEAAABVICAGnpeoCwrn1yoAAAC8j4AY+AFflgfuw2AWAACA9SAghhVmuS1gbTigAQAAFEJADMuj4Q5gmXwlAgAA4IMExHA3IzsBuMFQdQAAYP0IiFkTPrQDsHL8MhIAACicgJhi+NovADwH44wAAIAHERDzYRYOAoDK8t8AAABYbwLiNaR1CAAoM18kAgCA8hAQF8zwQQCAJ7IUAQAAPJqA+GF8/AAAWHt+hQ8AQHWseUDsC4wAAJSBIWAAAJRTiQJiS6AAAMA9+c8zAAALUWRAPDo/b/3qV/e/viYIAAB4Psv8+t3RZ5/1Pv/8gQUCALB4RQbE2WSSvHkTGaMGAADraM4CHs3dXdPVAADKoEQjJgAAAAAAWKaPiy4AAAAAAIBiCIgBAAAAACrqxfIfMk3Tfr+f53kURe12u9PpLL8GAACgQN1uN4qiXq9XdCEAAFW37A7iPM+73e5gMDg+Pj4+Ps7zfDgcLrkGAACgQEmSNBqNNE2LLgQAgKUHxFmWHR0dxXEczh4dHQmIAQCgOvI8T9O03W4XXQgAAFG0/BETjUZj9myapvV6fck1AAAARen3+6bMAQCUR5GL1OV5niTJ0dFRgTUAAABLk2VZdKtrBACAAhUWEOd5fnBw0Ol0puMmAACA9dbtdjWIAACUyrJHTARhqbper6d3AAAAKiJN0zzP+/1+OJtl2XA4NIwYAKBYBQTEWZb1+/1er6d3GAAAqqNer8+2D49GI+uRAAAU7qO3b98u8/HSNO12u69fv56mw/1+37fMAACgalqt1vHxcdFVAABU3bID4larlef5jd5h/y8EAIDqGA6HSZKkadrpdHq9XtHlAABU2rIDYgAAAAAASuLjogsAAAAAAKAYAmIAAAAAgIoSEAMAAAAAVJSAGAAAAACgol4U9cDpxcX/+i//5X/7u7/ba7fjjY2iygAAAIL8+jodj+dcIZtMsslk3hUuL/Pr6zlXSMfj/OoqiqLa3/yb4//wH6Ioqm9v17e25tykvrUVb27OuUJjZ2f+FZq7u3MuBQCouI/evn27/EdN3rzpDoe/8+LFX/7n//w3fu/3Xv+Tf9LY2Vl+GQAAUB5LyGdHZ2ePq22O5t7enEvjjY369vbt7X/1X/7L77y4o1tl/h7Ir67mX+ER4s3NRq025woSagBgvS07IM6vr7vDYXJy0t7fH/z+72eTyeFXX6Xjca/dPvr002VWAgAAsyqVz07Vt7bmN/A2arUV+sLfB39G6Xj8wR7nOZdKqAGA9bPUgDi9uLgdB9+IjFfof58AACyQfPZOq5XPIqG+k4QaAMpseQFxGCsRb27eOVBi/qUAADy3VQy2Ivks/JB/yHfyDxkA5lhGQHzPHuE7+4sBACpiFWMdjYfADb4KcCcJNQBl9uwB8YNiX+MmAICiyGfvJJ8FVouE+k4SagDmeN6A+HGDI4ybAIAKSi8u8qureVeQzwJQehLqx10BgAI9V0D8xF5g4yYAYMk+mM+Ozs8/cA8f/Dx8efmIwuaob29/8OOofBaASllCQp2Ox/P/z/AIjVrtA2/Zc38jG29sfPBXthJqgPd5loB4IfGucRMAVMpa5rPzP6pFH4pf481N3yUCgJUz/z8t+dVVenEx/wofHvokoQZYnMUHxIsdEGHcBADLIZ+9TT4LAKwiCfVtEmpgjkUGxM/U82vcBEAVPPH/8ZF8FgCAcqhgQh35ny2ssoUFxM8a4xo3AfDc/C/2Nv+LBQBgFem9uM3/7WGOxQTEyxkEYdwEsMYqmM/6HhwAAKwl09tuk1BTZk8NiJfc22vcBPBM5LO3yWcBAIBVJKG+TULNHE8KiAuJa42bgPWTX19/+N13bvyaXV7m19dzriCfBQAAYDnWMqGub2194CPwzs78K8yPsCnQ4wPiYgc+GDcBC7SEfHZ0dva42uZo7u3NuTTe2Khvb8+5wgfjV/ksAAAAq+iDCXU6Hn+wy2rOpfnV1fwrPEK8ufnBJioJ9TN5TEBckh5e4yZYG/LZOzVqNV8RAAAAgJXzwRxDQl0qDw6ISxXLliSqZm28L6iVzwIAAACsCgn1rA8G0w8OiEfn54e/+U2pBjskb970v/32uz/4A2EWTzQ6P2/96lf3vLJ8FgAAAIDbSpVQv/31r+df4UmL1MGame0gLm3bPwAAAADM8aCMS0AMAAAAAFBRHxddAAAAAAAAxRAQAwAAAABU1It7Xi9N036/n+d5FEXtdrvT6TxnVfdVzqpYOWmaJkmSZVkURY1Go9frFV0RAAAAADzSg1LTe80gzvP84ODg9evXcRxHUdTv9+v1ervdXlTFj1POqlhF3W736OgovJBGo9FoNJIRAwAAALCKHpqa3isgTtM0z/NmszndEh5jIRU/WjmrYg20Wq3j4+OiqwAAAACAB3toanqvERONRuPGY9Tr9UeXuCjlrIpVlyTJjZcWAAAAAKyKh6am951BPJXneZIkZfsCfjmrYrUcHh6madpoNAaDQdG1AAAAAMBT3Sc1/fih93hwcNDpdMIAi5IoZ1WsnMFgECZLJElSdC0AAAAA8CT3TE3vNYN4eo/dbrfT6ZTqC/jlrIqVZpg1AAAAACvt/qnpfUdMZFnW7/d7vV6punTLWRWrzjBrAAAAAFbXg1LTe42YSNP08PBw9h77/f6TalyEclbFyknTNMuy6dk8z2fPAgAAAMAKeWhqeq8RE61WK8/zG3lzmNZaoHJWxcrJ87zf709fS1mWDQYDPekAAAAArKKHpqYPmEEMayzP8zRN4zg2zBoAAACA6hAQAwAAAABU1L1mEAMAAAAAsH4ExAAAAAAAFSUghu+d/PEfZ5NJ0VUAAAAAwJK8eOgNkjdv6ltbzd3d56jm0cpZFaU1Oj+f/p2Ox+Hv/OpqeoV4c7NRq8UbG/Xt7Xhjo1GrxZubjZ2dguoFAAAAgAe7T2r64IB4eHraqNXKFsWWsyoKdzsIHp2dzV6hvr1d39qqb201arX61tb2j370e3/tr6XjcX59nV1e5tfXycnJ7eA4iqLwd3jJeeEBAAAAUEL3SU0fHBCXVoj/qKD8+jr0/6YXF9Fve4FvvB5CC3B9a+vos89CIhz+3HmHt//NpBcX+dVVNplkk0k4MTw9zS4v+9988/2t9vai3wbHjZ2daQ/yIp8qAAAAACzU+gTErL37B8HNvb3m3t4CU9r3DZcIkXH4E0oanZ3dWdJscDwnmwYAAACAZRIQUzqzqeu0bze7vJy9TmjXXXgQ/FDvi3pvZ9khO/7BbX873WIaGQuOAQAAAFgyATGFuT234X1BcHt/P1qpgb/xxkaoM1Q+68ZY5GwyGZ2c/OC21scDAAAAYFnWJCCub22F7/hTQiEInl35LXTXTq8wjURXLgh+hPc9u9t7yfp4AAAAADy3NQmI483NSEBctGlX7AeD4M6rV3pjbwj7wfp4AAAAACzTmgTELNONIQlRFL1vum6jVpuO2RUEP4718QAAAAB4PusTEBsxsVi3l1mL3h8EH332mWXWlsz6eAAAAAA83RoFxD9c3Ix7up0nhkm4s9eZNqI2ajWNqCVnfTwAAAAA7m99AmLmuz2R4H1BcHNvr7m3Z5Tt+rE+HgAAAAA3CIjXzWwQPF3c7EZ7dVjTLATBwj6sjwcAAABQWQLiVXW76/N9QXAYNSAI5qGsjwcAAACw9tYkIG7u7s52Na6T20FwmA4xvcK0Z7O9v29uLEtgfTwAAACAtbEmAfEauLGAWPTbMcHTK4RkLd7Y6Lx6JQimhKyPBwAAALByBMTLdjsIfl+LZaNW02LJerA+HgAAAEA5rVVAPDo/L0lydPu79iEFm71O6JGsb20dffaZIJhqsj4eAAAAQLHWKiAuUPLmTTaZzAmCm3t7zb09MRbcx0LWx6tvbXVevlxCtQAAAACr66O3b98+6AbpxUW8sVG2RtfQsVtg8Nr68stwQj8jLN/tnv2w/fiLL4osCwAAAKBQ98lyHxwQAwAAAACwHj4uugAAAAAAAIohIAYAAAAAqCgBMQAAAABARb2YnkrTNM/zGxfX6/UoipIkiaKo1+stoaDRaDT76KGAheh2u1mWvX79elF3mKZpv98PO63dbnc6nUXdM/B03W43WtaBCwAAAGBFfb9I3eHhYRzHURSlaVqv18Pp6LfxSqvVOj4+fu5qsixLkmQ4HLbb7bAlTdNer9doNBZy/wt8FnmeHxwcvH79Ouyofr9fr9enZQPFSpIkjuMkSZZw4AIAAABYXd93EMdxHLLgbrfbbDabzWb02xa8panX671eL4TCYUvIYUsY8WRZdnR0NI3Rj46ODg4OBMRQBnmep2k6GAzCtx8AAAAAeJ/vA+I7JyTMbkySZDQa5XkeYtxpNhpF0XA4HA6HeZ7HcVyv12eT0yeK4/hG+3CapkmSZFkWx3EItWcfK8uyfr+fZVn02wkVzWbzdgPywcFBqPbREyduV7XAaRjAU/T7fSNfAAAAAO7j+4D4znxzujFN03a7HeLUNE273e5gMAgXDYfD0Wg0TVrTND08PFzUqN/RaBTS3umdd7vd6WCHNE1n5zzkeR4eevbs7XQ4TA1e4OSKPM+TJDHqFMogHDEW9a8bAAAAYL19fM/rNRqNaUdeo9GYDW2Hw+E0LA6XNpvN4XD46JrSNG21Wq1W66c//els9BxFUZIk0/w3PFa73Z4+VpIks83LcRwfHR3dCL5DpjwYDBaYDh8cHHQ6nUU1TQNP0e12j46Oiq4CAAAAYDW8+PBVPmQ0GrVarRsbnzKNt9FohKHDw+EwTdPZi8JYiRsP1O/3w+k0TW8EQ7dT4CRJGo3GosLcPM+73e4Cm5GBp0jTNM/z6TEhy7LZRS8BAAAAuGEBAXGz2VzUQIkbQqwTEtiwJUyHmJWm6TTtrdfrWZbNnwU8GAxCh/JsJ/LjhHnHN4YgAwUKM9CnZ0ejkeHgAAAAAHPcd8TEHLM9vEGWZbMzKJ5457P31mw2u93u9NLQKjhtD+x0OrOXRr9d0e7GfTYajV6vF9ape3RhYdTybDp8YycAyxfHcXPG7VUuAQAAAKogy7Kf/vSntwc/3PbR27dvp2eSJBkOh1mWhUkOYXJCmLGbpmmn0wmdvK1WK6xZNx093O/30zQNnXrTJeAe0VebZdnh4WGapiHDDclOlmUHBwdHR0chCE6SZNoVmKbpjfEOs5feqOTg4GA0Gv3FX/xFuM+wd9rt9uMWl2u1Wnme33iOYTIGULjhcJgkyeyBCwAAAKA60jT95JNP4jgOcegcPwiIn2g0GkVRtMAJv++T53mYTdxsNoutBAAAAACghMJs3g+O31xkQAwAAAAAwApZwAxiAAAAAABWkYAYAAAAAKCiXhRdAMBiZJNJNpmE06Pz8yiKmru74Wx9a6u+tVVYZQAAAABlZQbxAnS//rq+tdV5+bLoQmA9pRcX+dVV9MMIOB2PpyfCpfcRb242arVwenpiGh/Hm5uNnZ1FlQ0AAABQfjqIF2AaVAEPEvp8oxDyXl9HUZRfXX3fBXx2duetboS84fT7Qt454XI6Hs8Jl5t7e+FEfWsr3tyMoije2Jg+7rQ3GQAAAGClCYiBBbs96iGKouzyMkTA2WSSXV7eecP69va7kHdjo769HdLYJ46JuGdH8Jyas8kkOz+/Z81ho9EWAAAAwKowYmIBWl9+2ajVep9/XnQh8LwePephnbpxF9L1HE4YbQEAAAAUTgcxsIxRD2tjGmfPz7WNtgAAAABWgoAY1lapRj1UjdEWAAAAwEowYmIBjJhgyYx6qCCjLQAAAIDnoIMYSsSoB97HaAsAAADgOQiI4dkZ9cDSGG0BAAAAPIgREwtgxERlGfXA2jPaAgAAANabDmK4g1EPEBhtAQAAAOtNQEyFGPUAz8RoCwAAAFhRRkwsgBEThTPqAdaM0RYAAACwHDqIKTWjHqCajLYAAACA5RAQUwCjHoCFMNoCAAAAnsiIiQUwYmLKqAdgpRltAQAAQNXoIOZejHoAqsBoCwAAAKpGQFxpRj0APILRFgAAAKwNIyYWYIVGTLS+/DIy6gGgZJ74LY3jL75YRpUAAACsIwFxtYQsO5w26gFgtbxvtIWAGAAAgEcTEAMAAAAAVNTHRRcAAAAAAEAxBMQAAAAAABUlIH6wNE3zPC+6CgAAAACApxIQP1i3203TtOgqHiZN04ODg1ar1Wq1kiQpuhwAFsCxHQAAgKcTED9Yu92u1+tFV/EAeZ53u93BYHB8fHx8fJzn+XA4LLooAJ7EsR0AAICF+Ojt27dF18DzCjMxms3mdMvBwcHr168LLAmAJ3JsBwAAYCFeFF3AKpkOl+j1eo1G48alaZp2u90oiuI4Di3GvV5v+UXedqPUNE1XqwMagNsc2wEAAFgIAfEDhMC32+3euUjd4eHh8fFxHMfRTFhcNnmeJ0lSkuQagIVwbAcAAODRzCBemHq9PhqNwulGozEYDIqt57Y8zw8ODjqdTkixAVgDju0AAAA8hYB4YQaDQZZl3W734ODg8PDwzi7jAoXljO4cjgHAinJsBwAA4ImMmFiMEAcfHR1Nz7Zare+++67Qor6XZVm/3+/1evrLANaGYzsAAABPp4N4MdI0TZJkerZUn9XTND08PJxNEPr9frElAfBEju0AAAAsxEdv374tuobVMBwOQwScZVkcx+ED+WAwCKvGj0ajJEmm27MsazabnU6n2JqDVquV5/mNzPr4+LioegB4Osd2AAAAFkJAvEh5nqdpGkVRs9ksuhYAAAAAgA8QEAMAAAAAVJQZxAAAAAAAFSUgBgAAAACoqBdFFwAAPFh+fZ2Ox6Pz8/zqKoqieHOzubvbqNXijY2iSwMAAGCVmEFcIa0vv4yi6PiLL4ouBIAHSy8ussvL9OIiHY/T8TjkwlEU/YO/+3ejKPq//uzPwtl4c7NRqzVqtcbOTn17u7GzU1jFAAAArAIdxABQOqFBOB2Ps8kkm0xGZ2dhe8h/O69e1be2Qgo8vUnIjrPJJB2Pk5OTaYLc3Nurb229u74WYwAAAH5IB3GFdL/+Oh2PdRADlNBsg3A2mWSXl2H7u1T34RMkZmdQhKw5bK9vb0/DZS3GAAAA6CAGgGWb3yDc3t+/3SD8UPHGRnN3t7m7O92ixRgAAIDbBMQA8OxCLDttE55tEK5vbR199llzdzektM9XQ2NnZzZxfpdNn59nl5chMg7bQ1tx+DuU93wlAQAAUDgBcbVMv2IMwLManZ+n4/G7OQ93NQjXt7Zm23uX73YNo/PzkBrfbjEOncWNWq3YmgEAAFg4AXG1TD/tA7BAZWgQfrob4e9si/Ho7Gx2irEWYwAAgLUhIAaAB7uzQTiKoubeXnt/fz2abe9sMZ4+6/4330y3azEGAABYXQJiAPiA2V7a0Cwctocu2qPPPqtIL+2NVe9m+6aHp6c3+qbr29sr0TcNAABQcR+9ffu26BpYkv6333aHw7e//nXRhQCU3ew03nQ8No33nuZMXn6XGhc9eRkAAIAbdBBXSKNWK7oEgDKa3yDcefWqIg3CT6fFGAAAYOXoIK6Q0fl561e/0kEMkF5cpOPxnQ3CIa8MHa/xxkaxda6Z0Fkc9nw2mdzZYtyo1Ro7O8XWCQAAUCk6iCsnv74WeQCV8m7iwfl5fnUVAsqwvb69Xd/aCg3CoVm40DLXX7yxcaPFODQXh7z+Rotxo1aLNzebu7uSegAAgGelg7hCQgfx8S9+Yf4jsN40CK8oLcYAAADLp4MYgNX2vgbhkCpqEF4h81uMk5OTadavxRgAAGBRBMQArJjZ0HC2QTiEhs29PaHh2mjs7DR2dtr7++Hs7V8G9L/5JpppMfbLAAAAgIcSEANQavPHDnRevTJ2oDrubDGejhOZbTE2TgQAAOCeBMQV0qjVii4B4MNmG4SzyeTGwmVHn32mQZip0GI8PXujxTg5OQnbw4KEWowBAABus0hdtbS+/LLXbvtgDJTW4W9+E0I965KxEHeuWNh59Wrws58VXRoAAEApCIgBKJHR+XkURWE4QNG1sIbCoJIoimbnVAAAAFSZgBgAAAAAoKI+LroAAAAAAACKISCurjRN8zwvugoAAAAAoDAC4hJJkuTw8DBN06ff1X2S3263u5DHutMCnwtQBWmaHhwctFqtVquVJEnR5bCe0jQ9PDwML7Nut1t0OQAAAKUgIC6RTqcTx/FCunoPDw8/eJ12u12v15/+WHda4HMB1l6e591udzAYHB8fHx8f53k+HA6LLoo1NBwOe71eeJk1m00ZMQAAQCQgXlf3SWY7nc7zBcQA95dl2dHRURzH4ezR0ZGAmOfQ6/WmL7Nms+lrLgAAAFEUvSi6gKobDofD4TDP8ziO2+32nEvr9fpsgBJFUZqmofspXBpFUa/Xy7IszHZotVrhanEcv379evZup8Mler1eo9GYvejg4KBer9fr9ZDO1Ov12Y/T8+uZ/1wA3ufGgShNU7++4rklSXLjhQcAAFBNAuIiDYfD0Wg0TW+TJBmNRs1m885Lw+TE2aj38PDw+Pg4RLTTsLherx8fH7darePj4/c9bq/Xi6Ko2+3ebjR+/fr1j3/84/AN3OndDgaDD9Yz/7kA3FOe50mShMMUPIfwa9RGoxHe3QAAACrOiIkiDYfD2U+nYW7v+y5tNBrNZnP2a9f1en00Gk0vXdQH3Uaj0el0pqezLLtPPfOfC8B95Hl+cHDgAMKzCtOuoyiyHCIAAECkg7hYtxt4Z7/uOhqNpmMipmZHNwwGgyRJut1ulmVxHE9T3Wcyv575zwXgg8JSdbdH38DCxXE8GAzCbyOKrgUAAKBgAuIi3W6Rm41Zm83mjdnBt695dHQ0Pdtqtb777rtnKPNe9cx/LgDzZVnW7/dnh57DczPqGgAAIDJiolj1en32+61pms5OkGi32/1+f/b6WZZNBz6kaTp72xuRSr1en81nF5LVzq9n/nMBmCPMNJ9Nh28cbeDp0jSdvmdFUZTn+exZAACAyvro7du3RddQaWGluNl4dzQaHR0dhdEN/X4/TdPQ4hRC3mmAMhqNkiSJ4ziczbKs2WxOvyobFpcLX9MON5wOCB4OhyHJDYMpws0Hg0HIlA8ODtI07XQ604XskiSZnp1TzwefC8D7tFqtG0ePKIrmrLQJj5Dneb/fn77SsiwbDAY61gEAAATExQt9uPV6/X3fdQ0r0TUajTvHOKRpGkVRs9l830V33vAp5tTzwecCAAUKb45xHBt1DQAAEAiIAQAAAAAqygxiAAAAAICKEhADAAAAAFSUgBgAAAAAoKJeFF0AAJWWTSbZZDI6P8+vrrLJ5P/7y7/8vd/93frWVry52dzdrW9t1be2iq6R9TE6Px+env6bP/3Tf/iTn7T395u7u0VXBAAAUDCL1AGwPDfi4NHZ2fSi5t5efWvr+q/+auN3fufOi0TGPFp+fZ2cnCRv3mSXl/Xt7b//t//2H//7fx9Od16+7Lx6FW9sFF0jAABAMQTEADyXOXFwo1arb23Vt7cbOzvh9J03T8fj9OIiu7wMp6cXiYy5p9AynJycRFHU3t8Pf8JFw9PT8CeKos6rVxqKAQCAahIQA7AY+fV1Oh6n4/H3J66uwkWzcXD4+3EPEcLi25FxvLnZqNUatVq8sTE9sZAnxYrKJpPh6em0Zbi9v995+fJ9v4dI3rwZnp5OG4rb+/t+5QAAAFSHgBiAx1hCHHwfImNumO0LvtEy/Ew3BAAAWGkCYgA+rCRx8H2IjKtpUY3A9289BgAAWA8CYgDuMDo/vzMOrm9v17e23oXCW1srMbN1dH4epiGn43E2mWSXl2H7jch4JZ4Ltz3TKOE5w4sBAADWiYAYgPdGqKsYB99H1Z7vWrrdMtx59WrhXeH59XVycqKhGAAAWGMCYoDKEY/eZp+skJALj87OooW2DM8321DcSekdzQAAIABJREFU3NsLSfFzPygAAMASCIgB1twHByyIPu80u99WfcjGeggtw8nJSX511ajVQjS85EHS+fV1iInT8Tje3Oy8eqWhGAAAWHUCYoC1Ml2ibf68XUu0PdScZfqmkXFJlulbP9OW4Xhzs72/33n1qvCdnF5cJCcnw9PT/OpKQzEAALDSBMQAK2waB2eXl6HXNWwXBy/BnMj4XX/x9rbI+ClmQ9iiWobnu9FQXJLwGgAA4EEExAArQxxcciLjhVjF1LX8WTYAAMD7CIgBSup9cXAURc29vTAAVxxcctOkOIwzDuuqBSLj29ZgbkMJp2EAAADMJyAGKIV3AeL5eX51FSLF6UUhDo43N5u7uyEXLq5Mnmr2B307Mm7UahX8Qa/fym9lWE8PAADgngTEAAWYkxKKg6umyi+G0fl5iIajKFrRluH5pg3FURSFmLi5u1t0UQAAAD8gIAZ4dlVOAHmEtX/B5NfXyclJ8uZNdnlZ394OufCKPpf7CA3Fw9PT8Hw7L192Xr3SUAwAAJSEgBhgwebPEJiOnQ2nC6yTFRKGUM8ZSL0qkfFsy3B7fz/8Kbqo5RmenoY/kYZiAACgNATEAE81Oj9Px+PpcmT51VXYbhUyns/71jCMNzenSxc2arWS5I9VaxmeT0MxAABQKgJigKf65A//MB2PxcEU63Zk3KjVvvuDPyi6riiKomwy+ek/+2cVbBmeb9pQ/Cf//J9XNi4HAAAKJyAGeKpsMhHuUEKlemXm19eaZO9kzwAAAMUSEAMAAAAAVNTHRRcAAAAAAEAxBMQAAAAAABX1ougCAFZVmqb9fj/P8yiK2u12p9MpuiKIopK9MktVTNmkaZokSZZlURQ1Go1er1d0RQAAQBWZQQzwGHmeHxwcvH79Oo7jKIr6/X69Xm+320XXRdWV6pVZqmJKqNvtHh0dhZ0zGo1Go5GMGAAAWD4BMcBjpGma53mz2ZxuCUFYgSVBVLJXZqmKKb9Wq3V8fFx0FQAAQOUYMQHwGI1GY/Zsmqb1er2oYmCqVK/MUhVTckmS3NhdAAAAyyEgBniqPM+TJPHdcMqmVK/MUhVTKoeHh2maNhqNwWBQdC0AAEAVfVx0AQCrLUxZ7XQ6YZAolESpXpmlKqZsBoNBmCyRJEnRtQAAAFVkBjHA4+V53u12O52O74ZTKqV6ZZaqmDIzoBkAACiEERMAj5RlWb/f7/V6miIplVK9MktVTMkZ0AwAABTCiAmAx0jT9PDwcDb26vf7xZYEUclemaUqpmzSNM2ybHo2z/PZswAAAEtjxATAY7RarTzPbzRFhkGiUKBSvTJLVUzZ5Hne7/en+yfLssFgoM8aAABYPgExAEAx8jxP0zSOYwOaAQCAogiIAQAAAAAqygxiAAAAAICKEhADAAAAAFSUgBjgSbLJ5OSP/7joKoBVlU0m2WRSdBUAAEB1vSi6AICym8Y3o/PzKIryq6t3Z8/OwhX+x1rt/xyPoyiqb2/Xt7aiKGrUalEU1be2wtnm7m4hlVNx3a+/bu7uluHll7x5E0VR5+XLogspkfz6enh6Ojo7G56eRlHU3t9v7u219/fjjY2iSwMAAKrFInUAH46Ap+LNzRD+zkbA//4//se//d/9dx+8rfiYJfvo5z8/+uyz3uefF11I1PryyyiKjr/4ouhCSiF582aaCzdqtf/lH/2jKIr+93/9r9PxOPptUixMBwAAlkYHMVAJj4uAG7VavLExPTu/s+/OnPfOx03HY/ExSxBvbhZdwjvxxkZ+fV10FQUbnp6Ozs+Hp6f51VWjVuu12+39/fDPPIqiX/7jf5xNJsPT0+Hp6eFXX3WHw/b+fnN3t72/X2zZAADA2tNBDKyJ/Po69N+l43GIor4/e3U1e80bXcD3j4AX7v6xtfiYR2h9+WWjVitDB3H366/T8biaHcQhFA65cH17u72/397fb+zszLlJenERbpJdXsabm+Em/rEDAADPRAcxsDIeFwE/qAt4yebnvLqPYXXdCHk7r159MBeeauzsNHZ2ep9/Hu4kOTlJTk7uGS4DAAA8lIAYKJH7R8BRFDX39qIoqm9tlTkCfgrxMevk9j/htZRNJsmbN7PNv83PP3/0mIhpUhzGUyQnJ/1vvglJcefly+l4CgAAgKcwYgJYthBoPigCDqNUQ6A5zTeZw/AKojItDdf9+uv+N9+8/fWviy7kuUzHB4cDWufVq2caHzxNiqMoatRqoafYIREAAHgKATGweCGUnGaUITHJJpPs8vLGNUXAhRAfV4SA+Lnl19dhBEQ4yrX395t7e+39/ef+HkN43NHZ2fD0NIqiRq0WRliszfcnAACAZRIQA48hAl5j4uO1ISB+JuXJZ4tKqAEAgHUiIAbudv8IuFGrxZub8cZGfXs7iqLGzk68uSkCXlfi4xUiIF640k54WNqMCwAAYP0IiKG60ouL/Ooqv7pKLy6ih0fA8eZmY2dn6VVTauLjUul+/XU6HpchIB6enh780R+tbkAccuHh6Wl+dVXyNeLuWCVPUgwAAMwlIIZ1diMCzi4v8+vr/OoqZMGzRMAsgfh4ycoTEI/Oz1u/+tXKBcTpxUUY4DDNhdv7+6tyVAzFT5PiMARjVYoHAACWSUAMq00EzNoQHy+cgPhxbkSrIRde3VdX6H2ebX+WFAMAALMExFB294+ApWasMfHxI3S//jo5OfmLf/Evii7kXUD83S9/WeZccnaM71oOZ5gdlFGqAcoAAECxBMRQvDuTLxEw3J/4+E7lWRouBMTHv/hFCffzjeXd2vv7zb29zsuXRdf1jJI3b0ZnZ8PT06hkS+0BAACFeFF0AVAJj0ivmnt7zb299U6vYFHm/0u58x9gOh5XPD6uuPz6enh6Os1J2/v7YUpvvLFRdGnPrvPyZefly+ke6A6H3eEwJOMV2QMAAMAsATEsxv0j4HhzM2RPjVqtUatJoOC5iY+ZdaN/ttduV7N/Nt7YCEnxtIf68KuvDr/6qgo91AAAwCwjJuC+HhcBR1EUb2xMz+rMgtWy0sMr+t9+2x0OyzBiIr24+OSXvyx2xIQJvB+09lOYAQCAOwmI4Q759XX/22+jKAojKdPxOL+6mr2CCBiIHhsfxxsbR59+uoTywuTfMgTEURR99POfFxUQH/7mNyEXrm9vh1y4zGvllUF6cRGS4uzyMiTFg5/9rOiiAACA5yIghrv9+J/+UxEw8BTvi4+jKDr+4oslFJBfX6fjcUkmYIzOz4s6fra+/DK0DMuFHyokxel4vJxXLAAAUAgBMQAAAABARX1cdAEAAAAAABRDQAwAAAAAUFECYgAAAACAihIQw/fSND04OGi1Wq1WK0mSossB1ko5jzBJkhweHqZp+qBb5Xl+e2OWZd1ut9vtPrqYO+/20cq5w1dImqaHh4dhBz7lxwoAAJScgBjeyfO82+0OBoPj4+Pj4+M8z4fDYdFFAWuitEeYTqcTx/FDk9nDw8PbG+v1eq/Xe2jW/MG7fZzS7vAVMhwOe71e2IHNZlNGDAAA60pADO9kWXZ0dBTHcTh7dHQkTQAWZc2OMItt9X2Ou12zHV6IXq833YHNZvMp0T8AAFBmL4ouAMqi0WjMnk3TtF6vF1UMsGYWfoTpdrtZluV5HsdxaN2dXnRwcFCv1+v1eohEw6XTpC+KouFwOBwOw23b7faDHjfLsjCSotVqhS1xHL9+/Xr2OkmSjEajPM9vP/T7yr7P3T6IQ/piJUlyY5cCAABr46O3b98WXQOUTvhu8o1cA2AhFnKECRlrON3v9+M47nQ600t//OMf93q9sCVN0yRJBoNBuGg4HI5Go+nZJEmSJOn1es1m8/6P3mq1jo+P77xozkN/sOw5d/sUDulPEYL7RqMx+3MEAADWiRETcFOe5wcHB2EuZ9G1AOtmUUeYMDh4NBplWdZoNLIsm7200WhMg9cblw6Hw9mkb+HHujkP/cGyn4ND+hOFOc5RFFnoDwAA1pURE/AD00Yz36UFFm5RR5g8zw8PD+M4DqFnaPC8/21vbFna4e4pZT/6ER3Sny6O48FgEHL2omsBAAAWT0AM38uyrN/v+xoy8BwWeIQ5PDw8Ojqahp6j0Wg0Gt3ztrcf/ZlWnLvtKWU/gkP6YhniDAAA68qICXgnTdPDw8PZKKHf7xdbErA2FnuEieN4tiU2LEZ3T/V6fXZWQJqmD7r59E5mY+V7RswfLPtxd3snh/QnStN0dgBInudLmAcCAAAUwiJ18E6r1ZpdPSl4juWSgApa7BEmLDQ3HdTQbDaTJGm3271eL4zcTdO00+n0er0oirrdbpIk07Nhy41iRqPR0dFRu92+ZwFpmna73ZD2hhh3MBjcfuhWq5WmabvdDlOP55Q9524ft4sc0p8oz/N+vz/dh1mWDQYDvdgAALCWBMQAsHryPE/T9EZP7v1lWZZlWb1ef/TcgFBAFEWNRuP+ueEHy37c3fJMnvgyAwAAVoKAGAAAAACgoswgBgAAAACoKAExAAAAAEBFvSi6ACidbDL5v//dv/uf/t7fizc2iq4FAIqUXlz8H//2327/6EfNvT1viwAAsJYExPC90fl5cnIyPD390V//6//Nxx+39/ePPv20vrVVdF3A+kjevMkmk97nnxddSFUkb94MT0+Pv/ii6EJWSXpxMTo7S8fj0fl5fnW19aMfTf7Tf4qiqFGrNff2Gjs7wmIAAFgnAmKIot8mCKOzs/r2dq/d/p//wT/43/7Vv0pOTpKTk86rV+39/ebubtE1Ausgm0xGZ2eRgHhZ3u1wPuRGKBxFUaNW67x6FeLgsBvT8Tg5OZleKiwGAID1ICCm0vLr6+Hpaf/bb7PLy0atNvj93++8fBku+kc/+cnRp5/2v/12eHqanJw09/Y6L1+29/eLLRhYA+l4XHQJFeJbIHPMD4VnY9/Gzk5jZyecziaT0fl5Oh4PT0/733wTbhX+NHd37XAAAFg5H719+7boGqAA+fV1/9tvQydUc2/v6NNP5/QIJ2/ehBC5vr199Omn0xAZ4KG6X3/d/+abt7/+ddGFVMXo/Lz1q1/Z4VN3hsKP7gWehsWj8/Ps8jKKovr2dnN3V1gMAAArREBM5WSTSYiGoyjqvHp1/ynDo/Pz/rffjs7O4s3NcENfqgUeSkC8ZALiaNGh8PsIiwEAYEUZMUGFTNegizc3jz77rPPy5YM+rzZ3d5u7u+nFRXJy0v/mm+TkxCp2ACshm0yqdqy+//iIRalvbXVevoxevoxmwuIwtjgSFgMAQInpIKYSZteg67x82Xn16omfjbPJJHnzJkyosIodcH/D09ODP/qjije0LlPoID7+xS+qcJReTqfwQ+XX16Ozs2ltURTFm5vvwuK9veloYwAAoCgCYtbZjTXoOq9eLXZ8cBhkPDw9zS4vrWIH3IeJB0u29gFxOUPh9xEWAwBACQmIWU8PWoPu6axiB9yTgHjJ1jIgXq1Q+H2ExQAAUBICYtbNo9egezqr2AEfFPLK7375S/nXcqxNQLweofD7TMPidDwenZ1FM2FxGFtcdIEAALDOLFLH+njiGnRPZxU74J5CwMcSrHS2uPyF5ooSb2y09/enY5pG5+dhjbvucBi2NPf2QlK80j9QAAAoJx3ErIOFr0H3dFaxA+60Ng2tK+Sjn/98hXb4encKP8I0LA6dxZGwGAAAFk1AzAp77jXons4qdsANAuLlK39ALBS+J2ExAAA8BwExK2nJa9A9nVXsgCC9uPjkl78seV65Zj76+c+PPvus9/nnRRfyA0LhJwpJ8ewODGGxHQgAAA8lIGbFFLgG3dNZxQ6IVqGhdc2UJyAWCj8TOxYAAJ7CInWsjMLXoHs6q9gBVE11FporUGNnp7GzE05Pd3j4mlEkLAYAgA8RELMCZteg67XbZViD7ikaOzuDn/3s6NNPwyp2ycmJVeygakJuxboSChdIWAwAAA9lxATlVf416J7OKnZQQeWZeFARPz06au/vP/cON+Wg5LLJ5N0Cd+fn2eVlFEWNWi38ae7u+ioPAABVJiCmpLLJ5JM//MNVWYPu6aar2DVqte/+4A+KLgd4RgLiJWt9+WWjVnu+HT48PT38zW+Ewivkdlhc394+/uILMTEAANUkIKa8ul9/vYqDhp9idH6eTSbr1ygNsMbSi4vh6alQeEVNw+LBz35WdC0AAFAMATEAAAAAQEV9XHQBAAAAAAAUQ0AMAAAAAFBRAmIAAAAAgIoSEFM6aZoeHBy0Wq1Wq5UkSdHlLEk1nzWsvTzPW63W4eHhIy7lQbIs63a73W53aY/ouL0G0jQ9PDwMP8RlvngAAKBULFJHueR5fnBw8Pr16ziOoyjq9/v1er3dbhdd1/Oq5rOGivjoo4+azebx8fEjLuWhWq3Wcnam4/Z66Ha7R0dH4Yc4Go1Go1Gv1yu6KAAAWDYdxJRLlmXTj2pRFB0dHQ2Hw2JLWoJqPmuoiEaj0Wg0HncppeW4vR56vd70h9hsNtM0LbYeAAAoxIuiC4AfuBGUpGlar9eLKmZpqvmsoSKm8dMjLuURkiQZjUZ5ntfr9dn4bzgcDofDPM/jOK7X67Px7iM4bq+fJEn8tgYAgGoSEFNeeZ4nSVK1L3tW81nDGms0Gs1m83GX8lBpmrbb7devX4fT3W53MBhEUTQcDkejUdge/Xby7PTsEzlur7rDw8M0TRuNRni1AABA1ZhBTEmF8Y69Xq9S7TzVfNYAi3JjBvH0bJgXPHvNJEniOH761GDH7fWQ53m32200Gp1Op+haAABg2XQQU0bhc1rVPm9X81kDLMFoNGq1Wjc2LiQddtxeD3EcDwaDg4MDATEAABUkIKZ0sizr9/uzgyOroJrPGmA5ms3mogZKTDlurx+DpAEAqKaPiy4AfiDMhZz9vN3v94staQmq+awBlqbdbt84rmZZlmXZo+/QcXsNpGk6+xrI8/wpLwkAAFhdZhBTLq1WKywxP7txdqDkWqrmswZYoDALOE3TTqcT1otrtVphzbqw8li/30/TNLSI5nkeRdFTmn8dt9dAnuf9fn/6c8yybDAY6AcHAKCCBMQAQFWMRqMoihqNhhyQIM/zNE3jODZIGgCAyhIQAwAAAABUlBnEAAAAAAAVJSAGAAAAAKioF0UXAHwvefPm//nzP/9bP/pR59WreGOj6HIAYJ2Nzs+zySSbTP74z//87/+dv9PY2alvbzd2doquCwAAlsoMYkqq+/XX6Xh8/MUXRReyDOnFxfD0NDk5ya+u/uFPfvJv/vRPoyjqvHrV3N1t7+8XXR0A71Wpd6tVl15cZJeXs3+H7fXt7b/3t/7Wv8qy/OoqbGnUavWtrfr2dnN3t761Vd/aKq5qAAB4djqIoTD59XXIhdPxON7cbO/vd169auzsZJNJ8uZNuKi+vd3e3++8fOnTKQDcX359nY7Ho/Pz/Ooqm0xGZ2dhe7y52ajV2vv78cZGo1Zr7u6+7yb9b77pf/NNuKi5txeS4kat1qjVfMsHAIB1ooOYklrvnqzR+XnIf6Moau7thQj40VcDoCjr/W61WqbzItLxOB2PZ9uBG7VavLn5iHbgd/d2cZFdXobTYXtImd81Gm9tzabMAACwcgTElNRafuSetgZnl5f3bw1+X6PxcmoGYI61fLdaCXPmRbzr832egcLpxUU6HoewOJtMZh93+ogGGQMAsFqMmIBlSN68GZ2dDU9PozBc+PPP7z9cON7Y6Lx82Xn5cjqqODk5CV+PtZYdAFXwiHkRz6Sxs3Mj/B2dn6fjcagwLCfw7ppP6FwGAIBl0kFMSa1HT9bs6nMLjHRvxs3WsgMoyHq8W5XQc8yLWI5Q9u0sO4qi5t5emF9skDEAAKWigxgW77mHQoSGYmvZAbAe5s+LCO+hqzK34fZU4tnnNTo7M8gYAICy0UFMSa1oT1Yhy8pZyw6gKCv6blWs+fMipj22axyYzvZHzw4yfhcWG2QMAMByCYgpqdX6yP241ecWy1p2AMu3Wu9WRVndeRHLERLz6SDjG1Mp6ltbdhEAAM9KQExJrcpH7hKOA36mwccA3LYq71bLNH9eRKNW0x77QbODjEN2HLbfaLI2yBgAgIUQEFNSJf/IvRIhbAnDa4A1U/J3qyUwL2I5Phi7T8P3YusEAGAVWaQOHmC1xjhYyw6AhZs/L+Los88MQ3gOjZ2dxs7O7C96Z38Q4TfW7645M8g4nC6oZAAAVoYOYkqqbD1Za7AQ3Bo8BYCyKdu71cKZF7EqpoOM342neM8gY1MpAAC4TUBMSZXkI3cZVp9brNVqggYouZK8Wy2KeRHrZDrIOLu8DI3GYbufJgAANwiIKanCP3Kv/QDflRijDFByhb9bPdH8eRGh59S8iLUx7QR/12isHxwAgCiKBMSUVlEfuSsYm659FA7wfFYrIDYvghvm/IYgDDL2GwIAgCqwSB1EUbUHL1jLDmAtzZ8X0d7fN2GAGz/9G6+Z/jff9L/55t019/ZCUjwdT1FEvQAAPAsdxJTU0nqyLN12gx0CcH+l6iA2L4KFe/dyurh43yDj8IryawYAgJUmIKakkjdvssmk9/nnz/oow9PTgz/6Iw2zt822VB999tlz/yAAVtRy3q3u46Of/zycMC+CZzUdYXxjkPHbX/+62MIAAHg0ATFVNzw9NXJ3jvTiIt7YEJ0DlFzy5o1GTgoRWtd93wgAYHUJiAEAAAAAKurjogsAAAAAAKAYAmIAAAAAgIoSEFO8PM+X/Ihpmh4cHLRarVarlSTJkh99VdhLADfked5qtQ4PD4su5B0HaoqVpunh4WF4BXa73aLLAQDgkcwgpngHBwevX79e2sPleR4eMY7jKIr6/X69Xm+320srYCXYSwB3+uijj5rN5vHxcdGFOFBTvG63e3R0FF6Bo9FoNBr1er2iiwIA4MEExBSv1Wot85N2mqZ5njebzemWJSfUK8FeArjTJ5980mw2y5CCOVBTNkv+Hx0AAIvyougCqLQsyw4PD9M0bbVaYUscxzc+3KZpmiRJlmVxHMdx3Ov1QqPKozUajRv3X6/Xn3KHa8leArjT+96D0jQNX7GP4zgcMJ87RHagplSSJLnxmgQAYFUIiClSvV4/Pj6e028SPm9Pvz8bhi1Ozz5dnudJkpShEazM7CWAqUajMdu0O3V4eHh8fDx9t1ryPFYHagoUftnfaDQGg0HRtQAA8BgWqaPUkiSZjYMbjUa73R4Ohwu58zC9sdPpLCpuXkv2EsCsXq93Z0Bcr9dHo1E4veSkzIGaYg0Gg/CbfislAgCsKAExpRbGSsxuabfbWZY9/Z7zPO92u71ez9ch57CXAO5pMBhkWdbtdg8ODg4PD/M8X87jOlBTBnEcDwaD6e9IAABYLUZMUGq3P2Cnafr0Dqksy/r9/tPHGa83ewngnsK71dHR0fRsq9X67rvvnvtxHagpFVOwAQBWlA5iilev12eD4NnTzWZzdoxjnuf9fr/dbj/l4dI0PTw8nP043e/3n3KHa8leAri/sJ7q9Oxy4loHaoqVpunsl7ryPF/Id7wAAFi+j96+fVt0DVRdWMwnfDc2pMOzoxuTJBmNRqEnJU3Tp3+LttVq5Xl+49P7+1bJqyx7CeD+RqNRkiTTsUhZljWbzU6n86wP6kBNscKv7acvwizLBoOBZnYAgFUkIKYU8jxP0zSKokajcfujxfTSO9cFAoAy8G5FBYWXfRzHpmADAKwuATEAAAAAQEWZQQwAAAAAUFECYgAAAACAihIQAwAAAABU1IuiC4AipRcX8cZGfWur6ELKKJtMRufno7Oz//cv//J/+Bt/o7m729zbizc2iq4LgB/Ir6/T8bhRqzlE89zCiy2KotH5eRRF2eVlfn2dTSb//cbGjzc3oyhq1GpRFNW3tupbW/HmZmNnp8hyAQC4H4vUUWmtL79s1Gq9zz8vupASySaT4enp8PQ0fAJs1Go/+fGP/82f/Vl2eRlFUXt/v7m3197fF0MAlMTo/Lz1q18d/+IXzd3domthHbwvBQ7/E5iqb2/Xt7bijY369vb1X/3Vxu/8zrtbnZ3duMPm3l4kOwYAKDEdxEAURVF6cTE6O5vmwu39/fBn2l6dXlwMT09HZ2eHX311+NVXISlu7u7qvwaAVRTy33Q8zq+v86urbDKZkwK39/ejKAq/hLhPu3p6cTG9z/Ao6XgsOwYAKCcBMVRaiH2Hp6fTBuHOq1d3Ngg3dnYaOzvR559PW4wPv/oqiv5/9u4mtpE0z+/8U4Vyj0ecaSPKI3kGMw0R0cCuQXnn4CgCC/iQOgQvysseMnjNqgsJJHysMuOwQPeR4arFYgFPAoxLZ94WCsG3FGwwDtJpDTOfyyxE2IArQKLHmLVoZGAGFtGYQUN7eLKiokmJ4kswXr8fFAqSmCQfhqgnIn78x/8RRr2+ECUDANJE4TBWWE6Bw7s79WFwRKvVVESrUmDj+Fir1VRcu/Xzrgh5yY4BAADyhhYTqLTKtpjwRiMVDQe3t1qtZjWbKufdqHGESorlZOKNRkIIo15X3Sc4eQOAlH3y1Ve0mKiyjVLgj/9PIgVO3HJ2LOhZAQAAsH9UEAMV4o1G/njsjUbh3Z3Khc0XL1S50Bb0w8Pe2ZkQIpzPVfcJ59075907/ehI1RRzqgYAQFI2SoHNkxPz5CSfKfAK1B0DAABkggpiVFoVKohVeqvqfMO7O5Xemo3GPgrNwvncv7mJMmj1XMbx8dYZNABgHZ989VXv+fNy784qYosS2sKlwImj7hgAAGBHVBAD5RRV9ar+D/rRkWouvNcTIbWIjdVsDl6+VNXK7tXVj9XKjQZJMQAAmwaaRr1OoLkCdccAAAA7IiAGSkX1BfbHY3XyY9TrfctSF5mmPJIoKfbHY1VT7F5dabWa2WioVsUb9TsGAKBYSIHzgOwYAABgHQTEQBmoXFi1khA/5MJWs5mHq01VO4v+ixdqWTz1X/fNG6vZJCkGABSaihdJgYuI7BgAACB3jg/GAAAgAElEQVRCD2JUWtF7EKvI1b+5UWekRYlcF+Js8+REdZ/IQ5wNAEX0817PajaLuzvLORUUhnd3cjoVP6TAcjIJ7+7i/8yo11Uv4KgjMHFh+dDvGAAAlBIBMSqtoAFxVIob3N6K4uTCy5YLn1VjCpJiANhIQXdnubJFCqwdHKgEcB/rvqJwyI4BAEBx0WICKAzVXNgbjX5c9u3Fi0Iv+6YfHvbOznpnZ9GSerbn2Z6nHx2ppJjzJQBAgjZNgY16nRQYa6JnBQAAKC4CYiDv4rmwSk5VQ4asx5Uk7eCg8+xZ59mzKCl2r66cd+9IigFgTQsRZ5WF8/nH4s3xWJACIwfIjgEAQM4REAN5FOWk3mgkhNCPjjqnp1XISaOkWPyQjKuk+GPFdOmScQBIikqXqmMhBQ5ub8P5PJjNVPOliH50pB8eagcHndNT8UP+SwqM/CA7BgAAeUBADORIMJv543GUCxv1eu/58yrkwg9StcODly9VUqzC4igpNk9OCtdzGQCwkU1TYPUhosp/VV1wBoMGEkJ2DAAAUkNADGRvea22vmWxVltEJcUitjqfe3Ulirw6HwAgQgoMbIrsGAAAJOuT+/v7rMcAZCbbZd/ldKqKhdWJsdVsGvU6ufA6VFLs39xEm848OTEbDTYdgGpqffedEGL49ddZD2SVqCNwOJ9HAdZjKbB+dCRIgYFELWfHQgiyYwAAIKggBtIXlcGqs2Kr2VT9hTn7XZ9xfGwcH4sXL6Li6+6bN0IIlbATsgOoGqNeV1lP5lT2FE+Bw7u7hbFptZrKnlQtsHF8rBaLY+oG9mofdcd8fgMAQDlQQYxKc6+v9cPDlBercS4vncvLqF6Yo+pEqKRYTibeaPT+l7+kzgVApfjjcTCbqRU+Mx9J69tvRSwF/vh/UmCgmFbXHd//6lcZjg0AACSFgBhIWzifEwrvD5sXADKkCodJgYEqkNMpH8kDAFAOBMQAAAAAAAAAUFGfZj0AAAAAAAAAAEA2CIgBAACKTUoZhmHWowAAAABQSATEKD/f923bllJmOwwpZbvdbrVarVbLdd1sB1NKbGEAlZX5bo4ZGKgg27Zt2856FAAAIAEExCg/0zRN08y2tCoMQ9u2B4PBcDgcDodhGHqel+F4yoctDKDKLMvSdT2rZ2cGBirIdV3DMDKvwAAAAIkgIAbSEARBr9fTNE192+v1OHlOFlsYQJV1Op0MA2JmYKBqwjCUUlqWlfVAAABAMj7LegBAeqIrcDVN6/V6hmGon3ue53leGIaapum6Hj/LTUr0XIqUMsMz+VJiCwMoLtd1VaKqdk+e5wVBcH5+rm6VUrquGwSBpmmapvX7/fhOKtq19fv9+EzYbrd1Xdd1XT2yrusLd1z9pBthBgaqxnGcTqeT9SgAAEBiCIhRFY7j9Hq9fr8vhAjDsN1uDwYDdebs+378PLzb7W53hrymMAxd11UjwT6whQEUiEpmh8Oh+ta27TAM43sl27bPz89Vtqta/UbfCiHUXKfuFX/Y8/Pzzz//vN/vq0dWjzMYDNZ50l0wAwOlFwSBWPpkCAAAFBotJlAVhmGYpqm+1jRtMBg4jiOE8DwvOmGO/tn+ro1V2XSn00m8SBkKWxhAsSzEqb1eT4Uv0a3xONgwDMuy1txJGYYRlfgZhrHwsCuedGvMwEAV2Lbd6/WyHgUAAEgSFcSoioUyB13X1cmw7/utVmvhH++ppZpaxmfhKmAkiC0MoOhUs6P4twthq2VZ6gPO/T3pdpiBgSqQUoZhGM1CQRB4nkczYgAAio6AGFWxsJKG6jgshDBNc68NJSJBEDiOs9ACEgliCwMoooUpKwiCeDHvQuMIIYSUcvdZbvWTboEZGKgItVxH9K3v+/QcBwCgBGgxgarwfT9+mm3btrrwdrkUa/fz5GWqtXH8zDnx+q+KYwsDKKhOp9PtdtUeShXhxitwTdO0bTv6VhXu7V6st/pJN8UMDFSHpmlmjKZpXDQAAEAJfHJ/f5/1GID9chzH8zy1RLuqcQiCwDTNqDOj4zjRkuvqbDnxGqhWqxXVLEei1YGwO7YwgOLyfd91XV3XwzDs9XoLDYJd141q9KSU8R4Onue5riuECIIgakYxGAw0TWu321LKTqcTrWLnum707ZNPuhFmYKCC1PwTn2cAAEBxERCjWlTfNMMwlvNf3/eFEA/eBABAarrdbnz1VCFEGIZSSiFEtNpqCk8KAAAAoCLoQYxqWXER3P7OugEAWJPv+8ufU6prulN+UgAAAAAVQUAMAACQpTAM2+22+towjHQu1s7kSQEAAADkEC0mAAAAAAAAAKCiqCAGUCrhfC6E0A4Osh4IAABAacnp9L+G4T/70z/VDw+zHgsAANgVATGQNvviovPsGQfTyfJGIzmd+jc3cjL56e///h/94R+ajYZRr1vNJmExAKTDH4+D2azz7FnWAwGwE388XvgiuL1Vn8HLySS8u1M/rP/RH03++39XX+tHR+rgVj881Go1IYR2cGDU60IIrVYzjo9TfQEAAGBDtJgA0vbJV18Nv/nGbDSyHkjh+eOxPx7LycS/uRFCaLWa2WiYJyc/+eyz/+e//Bd/PA5ub4UQRr1unpwYx8fmyQlhMQDsj31xISeT4ddfZz0QAA8L53M5mQghwrs7OZ2qH6qfCCHUAdUyo15Xma8KfIUQxvGxVqvN/+7vDn7yk2A2C2Yz9Zjqi2A2U8dgC7Ra7cdH+OGL6JCYY2MAADJEBTGAIonKhP3xWBWwWM1m37JUBBz9sy//xb8QQgSzmUqQ3asr9Y/NkxOjXjcbDU5CAABAmcjpVB3tbJ3Yqq8TTGxX5NHRB/zOu3cL93osjxZC6IeHXIQHAMA+UEEMpI0K4k1FOa83Gm2d8z6YLEfFxXscPQBUBhXEwJ6s2fMhrkA9H8r96gAAKAQqiAHkUTif+zc3qolE1Cmic3q6dacI4/g4OluIelPYnic8L+pNYTYalKUAAIDU7N7zQaWiha6xXadm+bH66GA2C344VlxARwsAANZHQAwgR+JrzQkh9KMjs9Ewzs6SXWsuXnocPWP3zZsfn5HV7QAAwG5y2POhuNYpCqajBQAAW6PFBJA2WkwseGytuZTreR+sWWZ1OwBYHy0mUB10RSgufncAACyjghhABtZcay5N2sGB1WxazaZgdTsAAKqKng+lR0cLAACWUUEMpK2yFcSJrDWXPla3A4A1UUGMnNu954P6gtQPIonPEj7+hM8SAAA5QAUxgD1KfK259LG6HQAA+bdj34CP7QLoG4C1aQcH0ScE6hK0Bz32zpSTCR0tAAD5QUAMIHnprDWXPla3AwAgZfR8QKHR0QIAUAi0mADSVtYWEzlZay59rG4HAAotJrApej4Aa6KjBQBgr6ggBrC9HK41lz5WtwMAYBk9H4AE0dECALBXVBADaSt6BXFB15pLH6vbAagaKogrgkpGoNCo3AcALKOCGMDTSrDWXPpY3Q4AUDi7J0fqa5IjILfWqVRY8TlQ1FDOefdu8ZH5HAgACouAGMCjyrrWXPpY3Q4AkDl6PgBYEx0tAKBqaDEBpC3nLSYqu9Zc+ljdDkDJ0GIiK/R8AJBbdLQAgEKgghgAa81lg9XtAABPoucDgEKjowUAFAIVxEDa8lNBHMxmzuUla83lzYN5fef0lN8LgJyjgngX9sWF2LDnA1dnA6iUHfvkaLWafnjYefYsrfECQJEQEANp88djo17PQwOBYDZrv35NT4M8izp+WM0mh7MAci6YzcL5nIxyO5989RXVcACwo9VXXeiHh3yKCQAPIiAGAAAAAAAAgIr6NOsBAAAAAAAAAACyQUAMAAAAAAAAABVFQAwAAAAAAAAAFfVZ1gMAyiMMQ03Tln8eBIHrukKIfr+f+qAeIKV0HCcMQyGEZVmdTifrEeFR/LIA5NNjuzxsSkrpum4QBEIIwzBycqgAAGXCETUAPIlF6oDEtNvt8/Pzx25ttVrD4TDN8TwoDEM1TnVi7ziOruuWZWU9LjyAXxaA3Fq9y8P6bNvu9Xpqnvd93/d9MmIASBBH1ACwDlpMAIlRH0rnXBAE0YmoEKLX63mel+2Q8Bh+WQByqxC7vELo9/vRPG+appQy2/EAQMlwRA0A66DFBJCAIAi63a6UstVqqZ9omrZcWuW6ru/7YRjquh4/IRRCeJ7neZ66YlfX9fhBTLIMw4h/K6XUdX0fT4Td8csCkEPr7PKitgmapmmatrDLw2Nc112Y+QEAO+KIGgDWQYsJIDGrm0h8/vnn/X5fdbxSZ86DwUDd5Hme7/vRt6pJVgqX7oZhaNs25+2FwC8LQK6s2OVJKW3bji7mXfgWD1KZu2EY0cEAACBxHFEDwGNoMQGkxDCMaD0EwzDUcjSK53nxE0LDMEzT3PelT6obV6fT4fAo//hlASgQ13XjcbBhGJZlcT3vaoPBQAXualVbAEDiOKIGgBVoMQFkz/f96ELdyF5XTog+POdS1vzjlwWgWFRbifhPLMtyHCer8RSFpmmDwUCFF1mPBQDKhiNqAFiNgBjInmmaaa4FHwSB4zhcWlUI/LIAFM7y+nVSSiaxNdEZEwASxxE1ADyJFhNAYnRdj58Vr7/C+3JpVRAE8R4UCZJSdrvd+OERVV25xS8LQG6t2OWZpmnbdvwmx3H2ellMcUkp47v7MAz3tPcHgMriiBoA1sEidUBi1Do86qoldaqsOgurdldSyk6n0+/3hRCtVktKaVlW1HrYcZxoRV113z19xN1qtcIwXHjkFWvrIUP8sgDk1mO7PMV1Xd/31U5NSsklvY9R6Xk01QdBMBgMKHADgARxRA0A6yAgBpIUhqGUUghhGMYWJ3i+7299XwAA0rR6lxfdappmBoMrFLWtNE0jRgcAAEAmCIgBAAAAAAAAoKLoQQwAAAAAAAAAFUVADAAAAAAAAAAV9VnWAwAAPEFOp/81DP/Zn/6pfniY9VgAAMnzRiOtVjPqde3gIOuxAECpyOk0vLsTQgSz2X/667/+Jz/9qVGvq5vMRiPLkQFAnhAQA9VlX1zoh4edZ8+yHggeFs7n3mjkXl3JyeSP/9E/+v/+5m+sZtM8OeFXBgAloCZ5/+bGG42iHxr1utVsWs0mnwgCwGr+eKy+kJNJOJ8LIcK7u2A2+3jrzc3yXeqHh5Mf/kGcfnSkZl3t4EA/OlI/jOJj/fCQORlA6bFIHVBdre++M+r1/osXWQ8Ei/zxWEXDQgjz5ESFBe7VlTcayclEq9WsZrNzemocH2c9UgDAZuR0qkJhOZkIIYx6Xc3z2sGB+qHKi/WjI6vZNBsNCtwAVEo4n6vpUcQi4OD2NoqAo1sXrB/yxp9iOVxe8RTqUg/1dfRF9MharcbBOYDiIiAGqqv13XdCiOHXX2c9EHwUzGbu9bU3GgW3tyoa6Dx7tlCwIKdTlR2Hd3f60VHn2TMKzQAg//zxWH34F9zeCiGsZlMVCy9P4KqyWCXF4d2d+lDQbDTMkxMaUAAormA2+zGBnU7VD6MoNpjN1PS4zKjXtVpNCKEfHqovtIODdNpExNtTRLXJvxMu3909eEfz5ER9kf6YAWA7BMRAdREQ54d7fR1dZdw5PTUbDavZXH0XbzRS/wkhaD0BADkUzuf+zY3KhbeLepdjZfPkxGw0+FwQQH6sTlEfbPWgRClq0atxH8y+o6rnFdk3rS0A5AcBMVBdBMSZi5cDq1KyzunpRgVi4XxO6wkAyJVgNvPH4+hjv0SaRSw3plBhMRM+gP1Z0eR30z4MxvFxVAVc2dCT1hYA8oyAGKguAuKsxFefSyrVpfUEAGRLzcP+zc1eM9wH02fj+PjJ604AQFnd5HedVg8Plrsa9TptcJJCawsA6SMgBqrLvriQkwkBcZqWV59LvC8ErScAIE3eaKSi4ZS7QDzYv0Kl0mQ0QDVFqeKDTX5JFcuH1hYAEkRADFQXAXFq1ll9Llm0ngCA/XlsHbmsynhVQr3OCngAiujHOt9NmvzSlwAKrS0ArIOAGKguAuIUbLH6XLJoPQEASQlmsygXFj+0d1AdHrIe2kdqhPFWxepqlfyMEEBk69guKv8UNPlFomhtAVQZATFQXfbFhXt19eHf/JusB1JCu68+lzhaTwDAdh5sLpzzD9tUjXO8VbFaJY9WxUAK4uHadk1+o5RN0OQX+UNrC6CUCIiB6rIvLpx37+5/9ausB1Ie+1h9Llm0ngCANXmjkT8e++NxcHur1Wpmo6GqcQsX0JTmhQCZ272+MqqppL4SpUdrC6BYCIiB6iIgTlAKq88li9YTALAsKrz1x2M1PZap8HahFFotpsfkD4hYnW8UYwma/AJpobUFkAcExEB1ERDvLv3V5xJH6wkAUK17/fFYxUClb91btdeLyooXMEYR8EZNfuMXwtPkF8gWrS2AvSIgBqrLuby0PY+AeDuZrz6XLFpPAKggVVGrPuQTlayoXaiYVvN/0fdoqIKtc6LlJr/xekOa/AIlQGsLYDsExEB1+eNx69tvCYg3ksPV55JF6wkApaeKZ73RKIpE1Xxepsl8C95opHYBKllT15SwWZCyBJv8xqsCudIcwDJaWwBxBMRAdREQry//q88ljtYTAMokKpVV05pqCmQcH1Mqu2yhVbFKz/mwEDtabvIbFfSJDZv8Rq0eKOgDkAJaW6AiCIiB6lIB8Ye/+AuKg1Yo3OpzyaL1BIBCk9OpCoWjrJNmu+tTrYrlZBJP1dWqfVkPDXnxYJPfKDfZtMkvuQmA4qK1BYqOgBioLhUQD7/5hjO9ZSVYfS5ZtJ4AUCD+eKw+3ou6JagyWGat7aj6a5UUx1sVmycnfMZcVg9WzEXpxtZNfjnmBABaWyCfCIiB6iIgfoxavk+UYvW5xMVbT/Qtq3d2lvWIAOBH/njcfv2aEHN/FsJ3dgTlIKdTdeTzWKsHsbLJLwVuAJC4HVtbGPV6/8WL1EaLEiAgBoBF/ngsJ5OSrT6XLNV6wqjX+XQBQK6E87lzeUkbhBSo9h3sCMpBBcTLTX5p9QAAOfdYawutViMgxkYIiAEAAAAAAACgoj7NegAAAAAAAAAAgGwQEAMAAAAAAABARREQAwAAAAAAAEBFfZb1AAAkQ0rpOE4YhkIIy7I6nU781iAIfN/3fb/T6ZimudEjh2GoaVqSY82r1dsQClsJQA4xNaVMSum6bhAEQgjDMPr9ftYjwjb4wwGAkmFix9aoIAbKIAxD27YHg8FwOBwOh2EYep4Xv9VxHE3TdF3f4sG73W5yI82v1dsQClsJQA4xNaXP87x+v682uGmatm1nPSJsjD8cACgZJnbs4pP7+/usxwBgV1LKMAzjpcHtdvv8/Hzhn9m2bZrmphXErVZrOBwmMMp8W3MbVhxbCUAOMTVlriKHCiXDHw4AlAwTO3ZBiwmgDAzDiH8rpdyoWFhKqWp/oipjda1oEATdbldK2Wq11L/UNK2sO5gdt2FFsJUA5BBTU7Zc1134FaAQ+MMBgJJhYscuCIiBsgnD0HXdjboBdrvd4XCoGg1HYbEQQtf14XBYwbKgLbZhBbGVAOQQU1Oa1KfIhmEMBoOsx4Kd8IcDACXDxI5N0YMYKJUwDNvtdqfT2WhZOV3Xfd9XX3Oat902rBq2EoAcYmpKmepyKIRwXTfrsWB7/OEAQMkwsWMLBMRAeaie9P1+f9MrPQeDQRAEtm232+1ut6vWPK2mrbdhpbCVAOQQU1MmNE0bDAbRx8woHP5wAKBkmNixHVpMACURBIHjOP1+f9MPCVUc3Ov1om9brdb79++TH2Lubb0NK4WtBCCHmJqyRYvDguIPBwBKhokdW6OCGCgDKWW3243vBhzHWf++8StDl3ckuq7Ha4rLWl+8yzasDrYSgBxiakqZlDIIgujbMAzj36Io+MMBgJJhYscuPrm/v896DAB21Wq1wjBcyHajleVUByIhRBAEmqZpmqbretRo2Pd913XVz9W/MU2z0+lEj6OWrVPXp6h0uJRNildvQyhsJQA5xNSUsjAMHceJtnkQBIPBgEqlwuEPBwBKhokduyAgBiCEEGEYSimFEKZprrjVMAzOAAEAgDo20DSNFocAAABFR0AMAAAAAAAAABVFD2IAAAAAAAAAqCgCYgAAAAAAAACoqM+yHgAA5It7fS2E6Dx7lvVAci2Yzf7v//gf/+gP/oANBQDVFMxm3mj03/72b//pn/yJ1WxqBwdZjwgAgEoL53M5mcjJJJjNhBBarWY2Gka9zj4a66AHMQB8JKfT7ps3cjIRQhj1+uDLL43j46wHlTvhfO5cXjrv3h385Cfzv/s7/eho8PKl2WhkPS4AEEKI1nffGfV6/8WLrAdSWioX9kYjtbv885/97C9//WshhNVsmicnJMVF5I/HrW+/vf/Vr7IeCABgM8FsJicTOZ0Gt7fq/+rn/8uf/ZkQ4v/9q79S3+pHR/rhoVGvG8fH+tERJ7l4EBXEACCEEM7lpXN5KYQYfvONEKL79u0Xv/xl7/lzUoY49/ra9rzw7q73/Hnv7ExOJs7lZevbb82Tk75lcagBIA/Cu7ush1BCcjpVubA6+bSazc7pqYqDo8i4++ZN980bo163mk2r2dQPD7MeNQAApSKnU1UgrCqF1TGPVqupnW+UAj/4792rq+gYyTw5UZXFRr1OrQ8UKogBVF04n3ffvPFGI6vZHHz5pSp9iupkKSVW/PHY9jw5mZgnJ4OXL+On/e71tXN5Gdzedk5Pe2dnJAIAMtT67jshxPDrr7MeSEnEc2F1peqKMmGVFPvjsX9zI4QgKS4KKogBIJ/iLSPUF+rnqgpY/d+o19ffzwazWTCb+ePxQsVx9IBmo6EfHrLjriYCYgCV5o/H7devhRC9s7Pe2dnyrd23b8O7uwdvrYhgNnMuL92rq9XdJOyLC/fqSgihYmIuMQaQCQLiRCzkwlazaTYaVrO55t3D+dwbjfybG280EiTFuUdADAA58VjLCKNeV0Gw+iLBUy1/PJaTiUqi1ee74oeS5B+fsfLFUhVBQAyguuyLiydrhMP53PY89+pquXK29FQZtYp914nIo7JrrVbrWxbr1wFIHwHxLvzx2B+Pt86Flz2YFJsnJ5xq5goBMQBkZUXLiEzy2SibVqNaSKhZ9a7cCIgBVFG0Ht2aXYa90aj79q1YLycth3jjiL5lrX8QEMxmtud5oxHr1wFIHwHxFlRfCG80Cu/uEsmFly0kxfrRkaopJinOAwJiAEhH4i0j9u1jZfF4HN7dLQyYVe/Kh4AYQOVE69Gdv3q1fnwZtSo2T07OX70q8aemcjq1Pc+/udll6Tl/PHYuL3d8EADYlH1xIScTAuJ1xHNhldgax8fJ5sLLVFIsJ5P485IUZ4uAGAD2JP2WEfv2YMmzYNW7UiAgBlAhD65Ht5GolHjw8uW+z6LTF/XT0I+O+i9e7P4CWb8OQMoIiJ+0nAtnlc/mZyQVR0AMAEnJW8uIfWPVuzIhIAZQFavXo1tfOJ+3X7/2b262TpnzKb7K3DptN7Z7ZNavA7BXBMQPynndbia1zIgQEAPAdgrXMiIFrHpXXATEACphnfXoNhL1qShBKbE3GtkXF3ut82X9OgDpULM9UZdSuM6/KXRDxjICYgBYU/laRuwbq94VCAExgJLbdD269QWzWfftW//mZtNl3PIjegnmyUnv7Gzf7aJYvw7AvhEQi6Vc2KjXrWbTPDnJbS68zB+PVVIc3N6SFO8bATEAPKZqLSP2jVXv8oyAGECZbbce3RZPodVqxUo8o5Je/eiod3aWZkkv69cB2J8qB8QP5sJWs1noK1vldOqNRiTFe0VADAAKLSPSx6p3+UFADKCcdl+Pbn3xIuVC9NhVoXZ4d5fhgFm/DsA+VDAgDmYz1ZlBdforRy68bCEpNhsN8+TEajbzv8/NPwJiAJVFy4i8YdW7DBEQAyihpNaj24hKJXLePMEfj7tv3wa3t1az2beszPesrF8HIFnu9XX3zZsqRF0qF1ZLz4ny5sLL4kmxEEJ1zyAp3gUBMYDqoGVE4bDqXWoIiAGUTeLr0a1vf/2Odxf1/zXq9b5l5SfCZv06AAkqfdS1kAtXOR5lUySl9H81ACqLlhHlw6p3+0NADKA8cpLPZphQP6gQCSzr1wFIRFmjLsLQFdg4OyrrXw2ACqJlRNWw6l2CCIgBlEQK69GtL+rkkHkpsXt9bXtetu2G18f6dQB2pKKu7//1vy5HNRDtFDZS2bYbOyIgBlBctIzAAla92xoBMYDCS3M9uvVFdbtZlRL747HteXIyMU9OBi9fFuj0mPXrAGxNRV3Db74p9KE/C7LtqCIL9yWFgBhAUdAyApti1bv1ERADKDy10Fma69GtT5US64eHw6+/TvmpP/+X/1Kr1YrbrkH9Ws1G4/zVq6zHAqAw5HRqe16hL0HwRqP269darWY1m2ajYTWbWY+owML53BuN/JsbbzQSQnz4i78gZF+m/mrSP1ABgI0Es9nP/9W/Ul/TMgJbe3DVu8GXX+azE2OaCIgBlEEwm+X2Q79wPhdCpH/UIqfT4uYjSjifh3d3uf3NAsCeqGtish5FqajzwIJ+YgoAUNzra1pGIFmqrJjCc0FADAAAAAAAAACV9WnWAwAAAAAAAAAAZIOAGAAAAAAAAAAq6rOsBwAAW5JSOo4ThqEQwrKsTqeT9Yh+RybDk1K6rhsEgRDCMIx+v5/CkyYu579ZAPln23YQBOfn51kPZAPlmMBzqGr7lDAMNU1b/nkQBK7rCiF4awEolqpN40gNb60F9CAGUEhhGLbb7fPzc3UW5DiOruuWZWU9ro+yGp5t271eTz2p7/u+7xfuPDDnv1kARdFqtYbDYdaj2EAJJvAcquA+Rb3ex24t3N8FgIqr4DSOdPDWWkZADKCQpJRhGJqmGf1k9RlRynIyvCKeB+Zk0wEouiJOgHFFH39OVHCfsvqdw/sKQLFUcBpHOnhrLaPFBIBCMgwj/q2UUri/4/IAACAASURBVNf1rAazLA/Dc113YRiFkIdNByBztm2rL9QkYBiG53lCiH6/H80Sqo+EuqBe1/XH6m3b7bb6N/GDfs/zPM+L7huV7uZEQSfwHKrUPiUIgm63K6VstVrqJwtve8V1Xd/3wzBUfzXxd/5jf1PtdlvXdV3X1Z/h8h0BYE8qNY0jTby1HnAPAAX34cOHTqfz4cOHrAfysPSH1+l0DMPodDqpPeOe5Pw3C2CvTNN8//79/f19r9dTE5qaE6J/EJ8c+v3+YDBYuLv6N5ZlqceJnJ+fxx/n/fv3lmXt50VsrDQTeA5VZJ+i3vmP0TQt+kt5//79wjttxd/U6jsCQAoqMo0jfby1lE+zDqgBYCeqeVCn08lnJUsmwxsMBuoCUrUcTUHl/DcLYN8Mw4iKO1RLOE3T1BpuiqZpYRj6vh8EgWEY8ZsUKWW73R4MBgtFIp7nDQaD+BOZpqlKIzNXjgk8h9inKOrjh+jrhb+aFX9Tq+8IAPvGNI494a0VocUEgAILw9C27fgVx7mS4fBUpY/a1aX81InI+W8WQObCMOx2u5qmqaN5KeXydKEaNSwf7vu+H12DH8nPsiRFn8BziH3KOtb5mwKATDCNY094a8UREAMoqiAIHMfJbRe8PAyvoH2U8rDpAORct9vt9XrR0bzv+77vL/ybwWCg+rFGS1QrpmnmfxGSgk7gOcQ+ZU3r/E0BQPqYxrEnvLUW0GICQCFJKbvdbnw2dxwn2yHFZTI8KWX8ks8wDIt4BWjOf7MAckLTtHitx2MNIgzD6Pf7ap266IeWZS1MLEEQZDthlmMCz6EK7lN0XY+/2+Nfr7bm3xQApKmC0zjSwVtr2Sf39/dZjwEANtZqtdQq2/Efqr6NeZDJ8MIwdBwnet4gCAaDQeE+Ds35bxZAOmzb9jzPsqx+v2/btu/75+fnuq5//vnn/X6/0+l4nuf7fnQtvGmaruuqfy+EaLfbvu9/+PBBCBEEgWooEd0qhHAcJ1quWiVo2daPlGMCz6EK7lOklLZtq6hXvbdVx23VY1FK2el01B9Cq9WSUlqWpf7BY39TvV5v4Y62bbuuG30LAPtTwWkc6eCttYyAGABKJQxDKeVCHRAAlM/u0526gv7BPsWZYAJHItQbSWz+3uYdCABAZREQAwAAAAAAAEBF0YMYAAAAAAAAACqKgBgAAAAAAAAAKoqAGAAAAAAAAAAq6rOsBwAAOwlms3A+N46Psx7Io+R0qh0c6IeHaT6pPx4b9bp2cJDmkyZLTqf/x7/7d3/4D/9h37IK/UIAYH3BbPa//9t/y9SXOH88/j///b+v/d7v9S0r5T0yAGA7cjoNbm/V/4PZ7O9/+9ujn/5UPzzUajWz0dBqtTyfA6IonMvL//D99//zn/xJ7+ys4odeLFIHoNjsiws5mQy//jrrgTyq9d13Rr3ef/EizSf95Kuvht98YzYaaT5pgtzra9vzfvLZZ7/5+7//x3/wB+evXnH8B6D05HTa+vbbv//tb//Hb35j1OuDL79k6ttdMJvZnueNRv/TH//xf/vbv/2b+bz3/DkngQCQN/54HMxmwWwmJ5NgNgtub9XPtVrNqNf1w8Pa7/3eX334EM7n/s1NdC91k350ZBwfq+A4o+GjePzxuPv2bXB7++c/+9lf/vrXWq3WOzvrnZ1lPa7MUEEMAMiRcD63Pc+9urKazcGXXwazWffNmy9++cu+ZVV5bw2g9Nzr6+6bN0a9Pvzmm2A2a79+3fr228HLl1azmfXQCsy+uHCvroQQaicSzufO5aXz7p17dVXxk0AAyFA4n8vJRE4m6ovlONhqNrWDA6Nef+yaSBUl++NxeHcXzGbu1VV4d6du0o+O9MNDdccVj4AqC2az7tu3/s2NOu4yGw31cbLtee719eDly2p+0kAFMYBio4L4QQWtIJbTaffNGzmZxOPghciYIzwA5WNfXDjv3nVOT6POEuF83n792r+56T1/nvIepByisqD4VlWimmL96KiyJ4EAkJoozBVCfMyFl8Jc/fBQPzzcZUJ+MnQ26nUhhNloqOfa9VWhmKKPirVarW9ZnWfP4rf647HteXIyMU9OBi9fVu19QkAMoNgIiB9UxIBYtZXQarUHG0qsvhUACiqcz7tv3nij0YNBsAqOzZOT81ev+HhsTVH+a9Trfct6bFcYnQRazSaNiQEgKXI6De/uotrebNtBRG0rVBdjOZlEN5knJ6qdsXF8rIa078Egc+qMMry7W91sas1/Vj4ExACKjYD4QcUKiNesEX6wvhgAiku1kghms+Ualggfj20k6imxZgcJ5/LSubys4EkgACRiYR25/Cew0YBX5Ne7lzMjbzYtDV5daFxWBMQAio2A+EEFCog3in1pNwGgNNSSdEKI4TffrD5tVvPk6hwZK3pKrFbNk0AA2MKT68ipuuBi9XBYpwMG7YyLa5e+UvFWxSuuSSoNAmIAxUZA/KCiBMTbVcZRTweg6OJL0q1zthl1ouicng5evkxhhAWyZk+JJx+kUieBALDaOi19yxqb/viSyxWFV038A+BdVqaNPoEufU8qAmIAxUZA/KD8B8Q71gLTbgJAcS0vSbfRHdePlatg054Sq8XLkHtnZyU+CQSAuHTWkSs01VX5sWYa2sGBaqORn2YaFedeXzuXl8HtbVItpFRPKiGEOkIo5WEYATGAYiMgflDOA+JE4l3aTQAonNVL0q3DG426b9+KNRpTlN7WPSWeFIXOJT4JBFBluVpHrrhWb8ao1rjKqXom/PHYubz0b27Mk5O+ZSV4sBSdfupHR72zs/L1pCIgBlBsBMQPynNAnGyDCNpNACiKdZakW/9xqnwJRSI9JVYr/UkggOoo3DpyxRV1pfixQccPhdil78uRB8Fs5lxeqn13/8ULq9ncx7PI6dT2PBVA987O8nnGvR0CYgDFRkD8oHwGxHuq+aXdBID8W39JunVE02nixbP5l2xPidXiJ4HJViEBwJ6Uch25QnvsNyJiAT2/kd3Fr/5J4dTbG43si4t9XMaUIQJiAMVGQPygHAbEe41xaTcBIM82XZJuTc7lpe15Rr1+/upVFc4q99dTYrVSngQCKIEqryNXaCtquuMtPqjpXlOGu+mUU+l9IyAGUGwExA/KW0CcTiMI2k0AyKGtl6Rbhz8et1+/FkKcv3qVnzk/cSn0lFhNrYReppNAAIXDOnIlFv1yl9sZL/xyyfrj8tDtIZ2+FukgIAZQbATED8pPQJxybS/tJgDkx+5L0q35LK1vv5WTyV6fJUNp9pRYrUwngQByjnXkKm798vBq9qbI21IB+1sZL00ExACKjYD4QTkJiDOJa2k3ASAPklqSbk3dt2/LN+9l1VNitfhJ4ODlywqelgNIHOvIYR3xdsbh3d2D7xOz0VDvluyGuXfO5aVzeSmE6Jye9s7OcnJ4IIRwr6+dy8vg9rb3/HmuBrYmAmIAxUZA/KA8BMTZNnyg3QSADCW7JN2aok7Hgy+/LPq8l3lPiSepvUx4d1fQk0AAGWIdOSQl/tFCOJ+XvtI8+uTYajb7lpXDPxDVk8p5906r1TK/8mlTBMQAio2A+EHZBsQ5qeGl3QSATOxpSbp1yOm0/fp1eHc3ePmyuA0Q8tNTYrVCnwQCSAfryCFlC+2MH+xVXcS3XDCbdd++9W9ucvvJcVz0Obd+dDR4+TLno40QEAMoNgLiB2UYEOcqls1JVA2gOva6JN06wvm8/fq1f3NTxJbE+ewpsVpBTwIB7Ik/HrOOHHJlnU8phBBmo5HPt2X849h02nYlxR+Pbc+Tk0lRelJ9lvUAAGAn6vKrrEexitVspj/C3vPnWW2W8O4unM/f//KXebjAWTs4GLx8adTrqk0VAOxb5ovFaQcHw6+/Vp+eZjWGrfnjsXZwkHmLpI3oh4fnr16pk0B/PC7QyAHsg+ovpILgzulpma7uR0FpBwfL4W/U5yS4vfVvbuRk4rx7d/+rX2U1yNXcq6siNnQyG433v/iF6kkVzGY5Ty0EFcQAAAAAAAC7K0QMBCzjrQsCYgAAAAAAAACoqE+zHgAAAAAAAAAAIBsExAAAAAAAAABQUSxSB2DvwjCUUgohDMPQNC0MQ03TEnnYRB4nWWEYttttXdcHg0HWY0mVlNJxnDAMhRCWZXU6naxHJEReRwWgZHI11UgpXdcNgkAIYRhGv9/PcDBrytUG3FQRNziAZBV6EkPF5fndm+exralYL4EexAD2y3Vd3/ctywrDMAgCleomcvrUbrfPz893f5zEffLJJ6ZpDofDrAeSHhWLn5+fq8jecRxd1y3LYlQASi9vU41t271eTw3G933f93MeWeZtA26qcBscQLKKPomhyvL87s3z2NZUuJdAQAxgjzzPk1LGz5Rs2xZCJHLu1Gq18hnCfvHFF6ZpVur8UEoZhqFpmtFP8hDf53NUAEom51NNbveVkZxvwE3lf4MDSFbJJjFUSp7fvXke25oK9xJoMQFgj1zXXThN6nQ66goLxfM8z/NUWbGu61ENjhBCNWrQdd3zPCGEruv9fl/dGgRBt9uVUrZaLfWPNU1TU626KbrG0/M813Xjl3yqfyCEGA6H6tmFEKZpqoG12201SPXJnm3bUspNrxh9sPGFlFKF4+qViq1S8hXbRHlse6rtIISIPsD0fd9xHCHEYDBQ49mFYRjxb6WUuz/m7vI5KgAlk+epRu0Bsx7FE/K8ATdViA0OIFllmsRQNXl+9+Z5bGsq3ku4B4C9MU1zxa3n5+edTif69v3795Zlxf+BpmmDwSC6Nf6PVz/4wk3L/9I0zV6v1+/31bfD4VB98f333y88i2VZHz58WPEqlvV6vegBI4ZhRI/z/v371VtmhRXbZPX2XHhd6t9HD5WgDx8+dDqdTTfavuVzVABKJj9TTafTMQxjeebPufxswE0VdIMDSFZxJzEgz+/ePI9tTYV4CZ9mHVADqC7P8+IruRmGYZqmqo2NfhL1cTcMQ60AkxRVYKu+jq77UJ/pRU/k+76u65suhdfv9+MXkkSP7Pu++towjK2XsFuxTVZvz2gA7XZb1Q5LKZfHuSNVhd3pdHK1fmA+RwWgZHI11QwGA3UFj7p8pBBytQE3VcQNDiBZhZ7EUHF5fvfmeWxrKspLoMUEgMz4vh/1iIik1rX9sSVEe72e4zgqaXVdd+skd8FgMHBd17btIAg0TdvHAqart6dpmlHeLaUUQgRBkOxFLmEY2rbd7/dzdYFtPkcFoGRyONWoK07UCUnWY3laDjfgpoq1wQEkqwSTGCorz+/ePI9tTQV6CQTEAPZI0zTVD/fBW03TzGGP9qiIOAzDLcqHH6TaLkcFy2EYtlqt9+/f7/7Icau3p2EYqnDYNE0ppcqpE3z2IAgcx1noiZy5fI4KQMnkearJe7c7IUS+N+CmCrHBASSrTJMYqibP7948j21NxXoJtJgAsEf9ft+27fiqdKqKVn1tWZaKLCNBEKzfR0LX9fgjx7+Oi/esWJMqIvY8L4p0dySljF9zuqfdw5PbU9M03/cty7Isy7btBIchpex2u/E938JIMpHPUQEomVxNNerzv+jbMAyT7c60D7nagJsq4gYHkKxCT2KouDy/e/M8tjUV7iV8cn9/n/UYAJSZlNK2bXU9haom7vf70a2O40SreaqEV02gqk2PlLLT6ah/b9u267rRt8uPLISI2kGoVFo9jq7rjuNEd3Qcx/d9KWV0iceDl3t0u92Foe7C933XdTVNU/uGIAhM09z0EtR1tslj21PdqjaL2ko///nPB4NBUj2IW63Wcqm46saYoXyOCkDJ5GqqCcPQcZxoPEEQDAaDnBet5GoDbqqIGxxAsgo9iaHi8vzuzfPY1lS4l0BADCANUsowDA3DePCsSS2e9titK4RhqNrpLt939TM+ybbtXq+X7DleNNrEl4ZbsPX2BACUgNrdaJqW/2535cAGBwAARUdADACLgiBwXTep8mEAAAAAAIDcYpE6APhRu91WDQQ1TfM8z7KsrEcEAAAAAACwR1QQAwAAAAAAAEBFfZr1AAAAAAAAAIrt+j//56yHAABbosUEACBJ7vW1fnhoNhpZD+R35HNUAErGvrgwG408TDXu9bUQovPsWdYD2YA/HgezWbHGHMeOBqggfzyWk0kwm8nJRE4m/+vPf/4fvv/eqNeNel0/PFRfaAcHWQ8TeII/Hvvjcf/Fi6wH8oCiHx6I4rwEAmIAQJK80cio1/N2hpzPUQEoGefdOyFEHqYabzQSBQyI5WRSrDHHsaMBSi+cz1UQHCXC6ucqCLaazT/7/PP/7Z//c3Wre3WlbtWPjozjY/3oyGw0yIuRT/547Lx7l9uAuNCHB6I4L4GAGAAAAEiAVqtlPYSPtIODcD7PehQAUGwqEfbH4+D2Vk6nwe2t+nmUCK/+WEiVDaq82Hn3Tn2IGOXFxvGxKjRO6cUAj+Nzi33zb26yHsLTCIgBAACABBj1etZD+Eg/OopK2wAAa3osETZPTqxm82PXiOPjNR9tITuW02lUfazCYiGEVqupuJm8GBnKzwEMMkRADAAAAAAAKifZRHg14/g4/lAf+1RMp6ofRXh3J2J5cbJPDQBPIiAGAAAAykZlDQCAuCiWTSERXk0/PNQPD61mc2Fg8bxYDYy8GOkI53N6TVQZATEAAACQjPw0dsjPSAAgQ/HgVU4m8ULdlBPh1Rby4nhpszcaxYNs9S9ZFROJk5MJb6p9KErrGAJiAAAAAABQBisS4c7paVFa/WoHB2ajEaV1v9MKYzJxr67Uz+P9KIj2gHzK/4SjEBADAAAAAIBCKkcivNpCXiyE8MfjaMm7eF6sHx7qR0dmo2HU67QLALA+AmIAAAAAAFAMKgv+mAuXNBF+0mN5cTCbOe/eOe/eCSH0oyPj+Ji8GMiDYDbL+bxEQAwAAAAkwKjXc9L5Nw/9NAEgKfFE2L+5UT9U6Wd1EuHVFvLi+BaL8mKVoauey2wxxBn1etZDKD8CYgAAAACp0mq1rIcAANtbkQj3nj+nHvZJxvFx/JPC+PZ0r67iNdfkxRBC8NcEQUAMAAAAAAAyRCK8Vwt5cbxrc5QXCyHMk5NoyTuuRAGqhoAYAAAASEZOWkwocjrlDB9APsXXWItmThLhdOiHh/rhodVsqm+jvDi4vfVGo+D2Vv08yov1w8N4/wqUlZxM+EVXGQExAAAAkIyoDisPcjUYABX3YCKsWhxYzab6gkQ4Ewt5cTify8nEH48X8mL1O1L1xcSIpRTO51kPoZyK8vdCQAwAAAAAAJL0ZCJclNCkarSDg/iSdyovjrcwVj+P58WE+0AJEBADAAAAAIDtLcSIJMKlsZAXi9+N/qO8WLUH0Y+OaA8CFBQBMQAAAJCA/JwPa7Va1kMAUHLxRgSqfa36OYlw6S3nxcFspvJi59075907EcuLjeNjVWic3XixFo4c9s0fj3M+JRIQAwAAAAkw6vWsh/ARa9MB2BP3+tq/uYknwubJidVsfmw1wORTPQuZl5xOo/piFRYLIbRaTX1y0H/xIosx4mn5OYZBVj65v7/PegwAgPKQ06l2cJC3MoF8jgpAyah6upyUh/jjcbEu8g1ms3A+L266xI4GFWFfXMjJ5Mfms4X9m0U6PrYcmU7lZKIdHJy/epX1iPCw3O7Fin54oPjjsVoNMuuBrEJADAAAAAAAAAAV9WnWAwAAAAAAAAAAZIOAGAAAAAAAAAAqioAYALClIAjCMNz0JgAAAAAAkB+fZT0AAEBRhWFo23av1zMMY/2btn4uKaUQwjAMTdPCMNQ0LZFHBoBCc11XfSbX6XQSmXKDIHBdVwjR7/d3f7SKY2+FcpBSOo6jPvu3LKvT6WQ9IhQJ75+qCYIgCAIhhDosie8Hi7tbXPGi9iTxA7wnsUgdAGB7YRi22+3BYKDr+vo3bcp1Xd/3LcsKw1DtJjVNI7kAgIht26ZpmqaZ1AO2Wq3hcJjUo1VWu90+Pz/PehTATtQR3fn5uQpEHMfRdd2yrKzHhWLg/VM18RM33/eDIHj//n10a0F3i6tf1F4lfoC3AhXEAIDtaZr2WIKw4qaNeJ4XBEH8SMK27d0fFgCAfaPbEkogCIJerxeVy/V6vXa7TcCHNfH+qRrP86JzQMuyvvjii/itBd0trn5RpUFADADINdd1F4LmTqcTP7aQUqoLcDRNU5XF6hi03W7ruq7ruud5Qghd16ObFNu2o3pkdWt005P3BYC4FfPJmqSU6tMv9Qjihw4Prut6ntfv9z3PU512hBC9Xm/NWpLVA3vsSRVVLxOG4RZzoOd5nudFzxulA57nqf4VhmGo52q322pKV1ecPHZHsfPMrLakerG9Xi/+6WMQBN1uVwgxHA7VAIQQpmlG10GvGNVjW1g9ppSy1Wqpn2iatlA29djDPjkeIE0LlzZLKXe/OAzVwfsnhx47e9p97xMEQfz3G9/x7bJbzPDgYfWLWj1ssfLYQyRx9JiwewAAcsw0zRW3vn//3jTNDx8+PPitpmmDwSC6qdPpxO8b/bP7+/t+vx/9y3XuCwBxq+eTdRiGsTCVRTcNh0PTNIfDYfRcnU7n/fv38bv3er3oH6w/sBVPussceH5+Hv/379+/tywr+vb7779feLTo1tV33GVU5+fnvV4v+rbX6y3f1zTNXq/X7/fVt9H2XD2q1Vt4xS7syRf72HiADKn5J/62B9bH+ycPVp893e+89+l0Ov1+//vvv3/w1q13i5kcPERWvKgVz/7kscc6R4+PHeDtw6cZ59MAAOzAdd2oqZkQwjAMy7LU57Tq2+gTb8Mw1MICEbXYnWojtXzr6vsCQNzq+WQduq77vq++NgxjMBjEbzUMIyoZVsU+juPsPrAVT7rLHOh53sJDmaYZzcy6rodhGF0I4vt+VF+2+o67jMp13XhhTq/Xe/C+qvBHfR1t8NWj2vpX/+SLfWw8QFZUM9lOp8M1VdgC75+cWH32pOyy9xkMBoZhuK7bbrdVyfCad8zhwcM6L2rFsz957LH70WOyaDEBACgwdWFU/CeWZa2Tm4Rh2O12o7tLKdNZHBZA+SQynwwGA9d11cWGmqYtXM658IDqjGL3ga1+0q35vh9dQBqJd5zsdDqO46izJtVAY807JiVqqbHgwS2wYlS7/OrXebH0lEB+hGFo23a/3+d4CVvg/ZMf65w97bj3iRZVW1iicLWcHzw89qLWf/aFY48cno0SEAMAck3lII8dVSxHJFLKdQ5But1ur9eLdsO+70dldACwkd3nEzWVRdU6YRi2Wq34Atlq7ezo24V2eNsN7Mkn3ZppmqvXKDdNU52LRg0Q17zj1hb2C0EQrF+ns2JUu/zq9/digcQFQaByGWo/sQXeP7my9dnTOjzPMwwjOkTRNM0wDCnlOmXIOTx4UFa/qBXPvvrYI4dno7SYAADkWr/ft207fiij6t3U16ZpRl8LIcIwdBxnnU+M1a49+nbhuioAWN/u84laLib+gAv/QJ1dR1+rk4odB/bkk25tuRZpOZC1LMt1Xcdx4mVK69xxO51Op9vtql2JKmRbv05nxaie/NWrS2Kjb+Nf7+/FAsmSUna73Xi6t2aLG0Dw/smfrc+e1iGlXNgVquYJ0bc77hZTPnhQVr+oFc+++tgjh2ejn9zf32c9BgAAVpFSRjtUVU0cb+fkuq7v++pDXSmlunhNXfsjpVRLCgghbNt2XTf61vM83/ejK3pM03Rd17Ksfr//5H0BIG7FfLLmI/i+77puVA4TBEF80XBVURIEgZrrwjDs9Xpq0ovW9Y5X06gVvZ8c2GNPujwHtlotKaVlWQudkVdwHCdaql6dGi3Xjn3xxRemaS5spcfuuPvMrF5vtAHjnQEdx/F9P35158J10I+N6slf/cL+S/12nnzYJ8cDpKnVai1fyzUcDrMaD4qF908OPXj2JNbYGz5JRc/Rb9z3/U6nEw9zt9stxp8i5YOHdV7UimGvOPZYfQix+gBvTwiIAQDFIKUMw9AwjOUytzAM1VoBG62ioO618OEtAGwhkfnksalMBcTbrVH25MC2mz/XpEb+4Ly9pzuur9vtrp93Kw+Oav0t/NjLSeHFAgAQt9e9/+oH399uca/70ye32DrPvnDskbezUQJiAAAAIL92CYjxINXpj4tCAABAOvJ/7MEidQAAAEBOua6rrjHUNG3flxaWm7rOVH1tGEaez9AAAEAJFOvYgwpiAAAAAAAAAKgoKogBAAAAAMAqwWwWzGbh3Z15cqIdHGQ9HBSMNxpptZpRr/PmAfKJgBgAAAAAACwK53P/5sYfj/3xOLi9jX5u1OvmyYnZaJiNRobDQ/7547E3GnmjUXh3p35iNZvmyUnn2bNsBwZgAS0mAAAAAADAR95oJKdT/+ZGTiZCCP3oyGw0jHrdajbDuztvNJKTiT8eq8jPPDlRNxnHxxmPG7khp1OVCwe3t1qt1jk9tZpN7eDAG43c62v1Q6vZNBsNq9nMerAAhCAgBgAAAACg4lSZsJxM/JsbIYRWq5mNhioT1g8PH7yLCpH98Th+F1VcTFhcTcFs5l5fR7nwYxHwQnxsNZud01PeM0C2CIgBAAAAAKicqEw4Kge2ms3tEl6VLy8XHa/Il1EawWymAl/127eaTfXfk3eU06l7daUaUOhHR+peJMVAJgiIAQAAAACohGA2U5XCUVtY1SMiqYbCqm3xQocKlfqxul3JhPO5Nxq5V1dRLmyenKhWEps+lDcaqW7FUVLcefaMjxaANBEQAwAAAABQWstrzUVlwnsNbaMweuF5Wd2u0FQu7N/ceKOREMKo16MWw7s/uKpEjh5Z1RSTFAMpICAGAAAAAKBsVqw1l34lr+pCwOp2habqfN2rK7Hn9HYhg1aFyZm8b4HqICAGAAAAAKAMtlhrLn2sblcsGfZ/COdz1aR49y4WAFYjIAYAAAAAoKgSXGsufaxul1tyOlUthvOwgtzCOnid01Oz0VhnyH8m9AAAFR1JREFUHTwAayIgBgAAAACgSPa91lz6WN0uJ1Qu7I1Gwe2tVqupXDg/b6pgNnOvr+PDIykGEkFADAAAAABA3mW11lz6WN0uffES3UIEr/EgO/MCZ6AECIgBAAAAAMipXK01lz5Wt9urhdYNRWzyu9wimfcGsAUCYgAAAAAAcqQQa82lj9XtkhLO595o5N/ceKORKGYuvCyeFKuPEKxms8p/L8BGCIgBAAAAAMhYodeaSx+r223Hvb6OcuFSpqgL2bd6jZ3T00Jn30AKCIgBAAAAAMhA+daaSx+r261joQ9D59mzkuXCy1RSrMJiUZYqaWB/CIgBAAAAAEhJddaaSx+r2y1QoXDF+/M+2Ge58+xZ1uMC8oWAGAAAAACA/ar4WnPpq/LqdnI6VZFocHur1Wqd09MqvOonqbeEe32tNovVbJqNhtVsZj0uIBcIiAEAAAAASB5rzeVERVa3C2Yz9/o6yoUJQB+zEKCrJsVleicAWyAgBgAAAAAgSe71te15rDWXQ8ur252/elWCX0379Wu1LJvqI0EuvA45nbpXV1ELjve/+AXl/KgsAmIAAAAAAJKkUsjKtr4thGh1u97ZWQliQff6WghBx5LtqA4w/Rcvsh4IkBkCYgAAAAAAAACoqE+zHgAAAAAAAAAAIBsExAAAAAAAAABQUQTEAAAAAAAAAFBRBMQAAAAAACQgCALXddvttu/7WY8FT5BSttvtVqvVarVc1816OFsqx6vIEBsQUAiIAQAAAADYVRiGjuNomqbretZjwRPCMLRtezAYDIfD4XAYhqHneVkPamPleBUZYgMCkU/u7++zHgMAAAAAACVh27ZpmqZpZj0QPEpKGYZh/HfUbrfPz88zHNIWyvEqMsQGBCKfZT0AAAAAAACA9BiGEf9WSlnEuu9yvIoMsQGBCC0mAAAAAABARYVh6Lpur9fLeiA7KceryBAbEBVHQAwAAAAAAKooDMN2u93pdDRNy3os2yvHq8gQGxAgIAYAAAAAAJWj1ijr9/sLrQaKpRyvIkNsQEDQgxgAAAAAAFRNEASO4/T7/UIXjZbjVWSIDQgoVBADAAAAAIAKkVJ2u914LOg4TrZD2kI5XkWG2IBA5JP7+/usxwAAAAAAQLGpNqZCiCAINE3TNE3X9cFgkPW48IBWqxWG4ULR6HA4zGo82ynHq8gQGxCIEBADAAAAAAAAQEXRYgIAAAAAAAAAKoqAGAAAAAAAAAAqioAYAAAAAAAAACrqs6wHAAAAAABASYTzuTca/ae//ut/8tOfWs2mfniY9YgAPCGcz/+v4fD3/8E/6J2dZT0WIBssUgcAAAAAwK680Uj9J4T485/97C9//WshhFGvW80mSXFu2RcXcjIZfv111gPZiT8et7799v5Xv8p6IIXkXl/bnvf3v/3t//jNb/Sjo8HLl2ajkfWggLRRQQwAAAAAwJa80cgfj73RKLy704+Oes+fd5490w8PVSmxf3Nje57teUa93jk9tZpN7eAg6yGjnILZjM8hNhLMZt23b/2bG/PkZPDyZTCb2Z7X+vZbq9nsWxYbE5VCBTEAAAAAAJuR06k3GrlXVyoXVmXCxvHx8r8M53P36sobjeRkIoSwmk3z5ISkOCfKVEE8/OYbSl/XZ19cOO/eabXa4OVLq9mMfu5cXjqXl0KIzulp/8WL7AYIpIqAGAAAAACAtahc2BuNgttbrVZTufCaqVwwm6n7khTnBwFxBfnjcfft2+D2tvf8ee/sbPkPMJzPbc9zr67oOIHqICAGAAAAAGAVle2619dRLmw2GvGqwy0ejaQ4DwiIKyVKfo16vW9ZqzeXPx7bnicnEzpOoAoIiAEAAAAAeMCDSW7n2bM9PX7n9HSX3BlbICCuDrUYndiwdwQdJ1ARBMQAAAAAAPwoWl/OG41EKhW+y50rSIrTUY6AWAjxyVdfERA/ZmExuk1rgek4gSr4LOsBAAAAAACQvYVcWF2EbjWbKVxabhwfG8fH/RcvoqTYvboiKQZ2Fy1Gd/7q1XZ/StrBgVrIzva81rff0nECpUQFMQAAAACg0rzRyB+PvdEovLsz6nW19Fy2ARA1xemggrjEnlyMbgt0nEBZERADAAAAAKpIhcIqF9aPjlQubBwfZz2u36GSYvfqKs+DLK4yBcS958+JLJWNFqPb+sHpOIEyocUEAAAAAKBCFopzO6eneY5co+4TqszZvbpy3r0jKQYeEy1Gt6fEnI4TKCUqiAEAAAAA5RfMZu71dQmaNsQbYpAU74gK4jLZcTG6LdBxAqVBQAwAAAAAKK1gNlP1wnIyEUJ0Tk8LmgsvW06KO8+eUcm4EQLi0ogWo1Plvak9Lx0nUA4ExAAAAACAsgnnc9W6V+XCVrNpnpxYzWYiC1XljUrAvdFICJGTRfaKojQB8c97PavZrGZAvI/F6LYYg+15cjKh4wQKioAYAAAAAFASKhf2b26itFS1GC5lLrxg+bWTFD+pNAFx67vvjHq9agHxXhej2wIdJ1BcBMQAAAAAgMKjijZCUrw+AuLiihajy1UaS8cJFBQBMQAAAACgqOjDu0KV66nXVKaAWD88HLx8mfVA0pD+YnSbouMECoeAGAAAAABQMHI6VS2Go1zYajaN4+Osx5VT4XzuXl1FK/WVuyPzRsoUEAshSvBCnpTVYnRboOMECoSAGAAAAABQJN5o1H79WqvVVC7MRdzrC2Yz1YhDTiZGvf7+F7/IekQZc6+vg9msBOGdfXEhhCjBC1nNvb7uvnmT4WJ0m4o6Tgy+/LLz7FnWwwEeRUAMAAAA4P9v7/5x3EaWAA5z/RYLPGxEA86NduYNe45AHYE6AnkEMtyQhE8gHkF9BHWy8U6H62wadmxgOjMW2AXmBQU3+CiLI2mkoTn8fcFAEv8VW00FNcUiMDPmzz9/8OLBH5z/8iV8/UrNNWbHff48u3k7x5ixNCSIAQAAAAAAAGChXk0dAAAAAAAAAABgGiSIAQAAAAAAAGChfp46AAAAAAAAHueca9s2hJAkSZ7nRVFMHdEsMYwXZ63tv9Vap2k6VTAhBOdcDCOEEIPpus57H0IoikJrfY2jd13nnLv4/mc6aWcaNpaJHsQAAAAAgB9dCGG9Xm+3W8l2tW2rlMrzfOq4ZoZhvDjvfdd1SZKEELz3Wussy7IsmySYruustXmeSzCSHW6apr9OXddXjfDi+5/ppJ1p2FgsEsQAAAAAgB+dcy6E0M86SfJlwpDmiGG8HmuttXaQjX1OxhjnXD+Auq6TJJl7gnimk3amYWOxaDEBAAAAAPjRDe5Yd84ppaYKZr4YxmdjjDHGSA2vUqqqqn7fibquY4WvUqqfw12v10oppZQxJkkSWXpMz4qu63a7Xf+Toiikv8ExDoXkvS/LUmstnxhjuq6Lbwdneo0K2ZlO2pmGjcUiQQwAAAAAmJMQQtd1E5ZqvgwM4/UYY6y1sVzUOVeWZb96tJ8vbtu267rYoHa73b5+/bppGsn2Oufqut5sNmeEcVJG8lBISqndbrdarWRRnud5nse3gzOVHhfXK0+e6aSdadhYFBLEAAAAAIDZkM6eR9ZU4hCG8aqMMf10sDQmNsbEAlt5fJxUlWqt9x9zF/PFWmvvfVwkD5obHO4iLR3GQzpkcKZFUUjh8zXMdNLONGwsDQliAAAAAMA8hBDqum6aZnD7Nk7CMF7ESEtZa22ssY1idjiEUJZlmqaSMXTOHf9FxMTxZZ0d0n4LiytNqplO2pmGjQUiQQwAAAAAmAHvfdu2FOI9EcMYyWPEpOHvGZuPtPfNsmzkcWRlWVZVFTOG8nS7MwIYkBLg877Ws0PaP9zxXY+PN9NJO9OwsUyvpg4AAAAAAIBHSBfXfqqlbdtpQ5ojhrFvvV6v1+uzk3dpmjrn4ltrbawRzvN8MLDe+9gaIk3Tfj3ppXoyNE1T13U/P9t1XV3Xx2x7Ukj9pUqpruviW+fcxVtMzHTSzjRsLNZPDw8PU8cAAAAAAMCY1Wq1Xx0pT/HC8RjGyDl3c3NTVdXZjw7rt2Xw3mdZ1u//0Lat9PNNvhXVxlyhPNgtNnPIsqzrujzPm6aRfrXOuaIoJLC6ruV5ccfEKU+0k1SvfNFxK2OMZHK997GVxGazkQhHQpLNJfUsRcpKqbZt+xEOJpW1tqqqmC5/oplO2pmGjcUiQQwAAAAAAJalbdu2be/u7p54+7+UBmutv7sfadSwv1QeBzeo270UaZ1xKKRDHg1pZLcyCGc36wAwORLEAAAAAABgWdbr9aDmFwAWiwQxAAAAAABYFufcNap3AWCOSBADAAAAAAAAwEK9mjoAAAAAAAAAAMA0SBADAAAAAGYjfP1qP36cOorZc58/u8+fp44COIH/8mWm1779+DF8/Tp1FMAYEsQAAAAAgNlwnz6tPnyYOorZq42pjZk6CuAE3R9/zPTaX3344D59mjoKYAwJYgAAAAAAAABYKBLEAAAAAAAsi377duoQgNNk799PHQLwYpEgBgAAAAAAAICFIkEMAAAAAAAAAAtFghgAAAAAgMWxf/01dQjAyezHj1OHALxAJIgBAAAAAAAAYKFIEAMAAAAAAADAQpEgBgAAAAAAAICFIkEMAAAAAMCyqDdvpg4BOA2TFrgeEsQAAAAAACwLuTbMDpMWuB4SxAAAAAAAAACwUCSIAQAAAABYIv/ly9QhAKdh0gLXQIIYAAAAAIAlIteG2WHSAtdAghgAAAAAAAAAFooEMQAAAAAAAAAsFAliAAAAAAAAAFgoEsQAAAAAACxL9v791CEAJ8t++23qEICX6T+///771DEAAAAAAHCs//7yC/nNp/v733+z9+/TX3+dOhDgWH//849++1a9eTN1IKfhWsOP76eHh4epYwAAAAAAAAAATIAWEwAAAAAAAACwUCSIAQAAAAAAAGChSBADAAAAAAAAwEL9PHUAAAAAAICXKYTgnJPXWus0TZ/nuF3Xee9DCEVRaK3H13TOPbraPu+99z5JEtnw0VPz3nddlyRJ0zQnHQh48fpXk1xK+59MxVorL9I0PfVXApgREsQAAAAAgKvw3kt6xTmnlNpsNs9z3KIokiSp6zqE8Oiax6w20HWdtTbP8xCCJKNvb2/HN1FKNU2zWq1OOhCwBNbaruuyLEuSRP5aa733zrmqquSTSXjvjTHJt/8Axf/xTJuzBq6BBDEAAAAA4Cq01lrrruu01pJneRmMMbvdTl7neX5zczNtPMCsFUXhvS+KQikVP5GU8YTZ4SRJlFJ5niff0tZJkjjn6rp+tv91Ac+GBDEAAAAA4Iq8903TSLGt1PbGz8uyTJJkt9sZYySDnGWZrLNer5VSSin5XCpw+4V7zjmp3k3TNE3T48v65FghhDRNJftz6unENFaSJGmabrfbwQpt28oN8nIKWZbFm9Ol+jiE0D8jY4wUJ263W/nEWtu2bZIkm81GDlfXtfTNSNNUtj1yJAdly1rr/rb90VBKVVUVh3H8KxjZUE4zll5WVWWM8d7HgRpfemjP46eJWZOMcH9mWmvjlBiZ/I/+UFyWtLxwzskVfSiwuq7lhdw8Ef9D1jSN1np8afLYxQVcxQMAAAAAANdxd3dXVZW8Lopif4Usy6qqappG3u52u7goTdPNZiOvb29v+5vf3t5mWXZ/f//dt6Kqqv7exHa77e9ns9lorfdXG1cURdM0d3d3+4vu7+/7kdzf3+d5Hvc/ckb7g7PdbuPKsqv4umma/iJxaCSzLOuvc3t72z/EYFTzPO/v81DA4xtut9v4pT88PFRV1V/50aXjIY1MGMxaf6Le3t7Gr/jhsck/clk93W63G8yx3W4XYxsJLF5rcYbf39/H2EaWPnoJANfwauoENQAAAADgxTLG9Kt0pa52QErk5HX/jnKtdSwO1Vr3t+26LhbbytI8z4/pYmGM6d8eXhTFGaV5klbuum69XpdlGR/EJ4H1y/2kQjZWHI+ckVJK+jWv12upHXbO9UcjTdMQgvRmHWzb38n+SMbzres6lih+dzS01lmW9YfxUMDjGw5KQauqGnx3I0sfDenQaWLu+pfw4G6D8ck/clldSexaPhKYNNiR1/IbmKZpXGFk6TGXAHBxtJgAAAAAAFyLdFSQ1yEEY0xM7UVntAiQthL9T/I8l7zquP3n0fUTpseL3VFDCOv1Omar5bFaZ+w/yzJrrVJKbmBP/r+XRQihLMt41vEO94HvjqTspCzLfk5KWGv3n5t3TNuNkzaU2+QP7Wqw9Jg901PiRcrzvK5refZj/wI/cvI/J5mxVwrs7KsSeAoSxAAAAACAq3DOFUXRT5iuVqv9BPEZ9vO8zrljaoH319nf1ThjjNY65jTTNNVax2pfpdSgSfGRtNaS4M6yzDknvZXj0rIsq6qK6SdrbUy7H0OeE7ifV82ybNBA+UjjGw4G2Xvfr6wcX3p2SJi72GnaWtsvDH/i5L+4+F+uKwXGJYBJ0GICAAAAAHAVg/vEk291sk/fc5Zl8UFPSZKEENq2PabITiklj4MTzrlT793e30TuLpfXRVH0A0u+PUzvmD2naWqtzfNcSin7iVRJQ8e3J8Usj/KLX8R6vY6L9suuB+naQ8Y3LIqiLEvJvIcQ6rruBz++9OyQ8ALkeS73HAyaq5w9+S+u67pY836lwLgEMImfHh4epo4BAAAAAPDS1HVtjFFKFUUhqdu6rq21IQR5LlPbttba/n3ZsUOu9G2QAmTpV1vXtaSbY/taSSRJpsY5F7c1xkhCVopwJc262WxiVW9d13IPewzVWltV1ZE3cUv+N+7BWlsURT8P3g9M0qAS8+CMVquVcy7P836P4BCCvH337t1ms4lpMmOMtTbeyZ5lWdd1eZ7LrkZG0nv/7t27frrNOXd/fx/ftm3rnBtEK51Vx7+CQxvGIe26TiklX/eg7/D40kN7HjlNvBg3Nzd5nvfvMxiZ/Mf8UDyF917+oSJHl9Rw3PNIYPLrF19ba7fbrVLq9evXTdN470eWym/jyMUFXAMJYgAAAADALIUQpF3vqU8qk4o8pdQZvSCOPLQUSmutL5XWkSMOihYv5exoj9ywLMv+c7eOWXrxAcR8XXXyP8WPeVUCZyBBDAAAAAAArkV6sx6q6BxfCgB4BjykDgAAAAAAXJLc+y+vtdaD/O/4UgDAM6OCGAAAAAAAAAAW6tXUAQAAAAAAAAAApkGCGAAAAAAAAAAWigQxAAAAAAAAACwUCWIAAAAAAAAAWKj/AW4wdAptCWIIAAAAAElFTkSuQmCC", + "text/plain": [ + "Tree('S', [Tree('2', [Tree('2', ['The']), Tree('2', ['Rock'])]), Tree('4', [Tree('3', [Tree('2', ['is']), Tree('4', [Tree('2', ['destined']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('2', ['to']), Tree('2', [Tree('2', ['be']), Tree('2', [Tree('2', ['the']), Tree('2', [Tree('2', ['21st']), Tree('2', [Tree('2', [Tree('2', ['Century']), Tree('2', [\"'s\"])]), Tree('2', [Tree('3', ['new']), Tree('2', [Tree('2', ['``']), Tree('2', ['Conan'])])])])])])])]), Tree('2', [\"''\"])]), Tree('2', ['and'])]), Tree('3', [Tree('2', ['that']), Tree('3', [Tree('2', ['he']), Tree('3', [Tree('2', [\"'s\"]), Tree('3', [Tree('2', ['going']), Tree('3', [Tree('2', ['to']), Tree('4', [Tree('3', [Tree('2', ['make']), Tree('3', [Tree('3', [Tree('2', ['a']), Tree('3', ['splash'])]), Tree('2', [Tree('2', ['even']), Tree('3', ['greater'])])])]), Tree('2', [Tree('2', ['than']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('1', [Tree('2', ['Arnold']), Tree('2', ['Schwarzenegger'])]), Tree('2', [','])]), Tree('2', [Tree('2', ['Jean-Claud']), Tree('2', [Tree('2', ['Van']), Tree('2', ['Damme'])])])]), Tree('2', ['or'])]), Tree('2', [Tree('2', ['Steven']), Tree('2', ['Segal'])])])])])])])])])])])])]), Tree('2', ['.'])])])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tree" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is what it actually looks like, of course:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(Tree('S', [Tree('2', [Tree('2', ['The']), Tree('2', ['Rock'])]), Tree('4', [Tree('3', [Tree('2', ['is']), Tree('4', [Tree('2', ['destined']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('2', ['to']), Tree('2', [Tree('2', ['be']), Tree('2', [Tree('2', ['the']), Tree('2', [Tree('2', ['21st']), Tree('2', [Tree('2', [Tree('2', ['Century']), Tree('2', [\"'s\"])]), Tree('2', [Tree('3', ['new']), Tree('2', [Tree('2', ['``']), Tree('2', ['Conan'])])])])])])])]), Tree('2', [\"''\"])]), Tree('2', ['and'])]), Tree('3', [Tree('2', ['that']), Tree('3', [Tree('2', ['he']), Tree('3', [Tree('2', [\"'s\"]), Tree('3', [Tree('2', ['going']), Tree('3', [Tree('2', ['to']), Tree('4', [Tree('3', [Tree('2', ['make']), Tree('3', [Tree('3', [Tree('2', ['a']), Tree('3', ['splash'])]), Tree('2', [Tree('2', ['even']), Tree('3', ['greater'])])])]), Tree('2', [Tree('2', ['than']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('1', [Tree('2', ['Arnold']), Tree('2', ['Schwarzenegger'])]), Tree('2', [','])]), Tree('2', [Tree('2', ['Jean-Claud']), Tree('2', [Tree('2', ['Van']), Tree('2', ['Damme'])])])]), Tree('2', ['or'])]), Tree('2', [Tree('2', ['Steven']), Tree('2', ['Segal'])])])])])])])])])])])])]), Tree('2', ['.'])])]),)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(tree,)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's a smaller example:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAALgAAACMCAIAAABAuvQrAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAHXRFWHRTb2Z0d2FyZQBHUEwgR2hvc3RzY3JpcHQgOS4xNnO9PXQAAAnVSURBVHic7Z0xbONGFoZnD1cE62tYeNsA3M4ueVtrA5CN3Jpq7TQUkkW6BGSbjoT6AGSTTXcgXR2wbsjCbrMaIM2q88DbWoAGCE5ODsFFVwxujktL1IgSNTPU+ypRIqk3nF9vZjjDX88WiwUCgHX8RXYAgB6AUAAhQCiAECAUQAgQCiCEHkIJgiAIAtlRHDQaCCVJEsuyMMayAzloVBcKpRRj7Lqu7EAOHdWFEkWR53myowDUFgohBCFkWZbsQAD0V9kB1BEEQRzHsqMAEFJZKBhjSmkURWyTEJJlGXRWZKGuUEzT9H2fbxZFYZqmxHgOnGe6zB47jpPnuewoDhelO7OMLMscx8EYwz03iWiTUQC5aJBRABUAoQBCgFAAITQQCn18/Ocvv5DpVHYgB42691EQQvTxMbq+Tm5u/vPnn7/+9pv3+rXf75vHx7LjOkQUHfVwidD53Hv9+qsvvvjHzz/zTZDL/lFOKBWJlDVR8xHQNgoJRVAHIBcpKCGUBnUPctkzkoWyZX2DXPaGNKHssI5BLntAglBaqleQS6vsVSh7qEuQS0vsSSh7rj+Qy85pXSgS6wzkskNaFIoi9aRIGLrTilAUrBsFQ9KLHQtF8fpQPDyV2ZlQNKoDjUJVhx0IRdPrrmnYsthKKB241h0own5oKJSOXd+OFacNmgiFTKd///777l3TslzSr792X72SHZFCNMwowdWV1+t1RiJlmFz8ft94/lx2LAqhxHoUQH00WIUPqAAIBRAChAIIsdlzPRjjKIoopQgh13U77K7GnBPCMJQdiCpskFEopcwrK8/zPM8ppVmWtReZRMCw9CkbCIUQ4vu+YRhs0/f9TgoFDEuXskHTU3FnxBh30isLDEuX0rAzSylNkqTssdYNwLB0FU2EQikdDAae5/FmqDMEQdA99e+Ejd0MWJc2DMPu/ezAsLSGzYRCCImiKAzD7uUSBIal9SyEGY/Htm3PZjP+ThiG4odrh23bskNQiA0mBR3HoZRWckknvV+zLEuSBGPseR7cc2PA7DEgBMz1AEKAUAAhQCiAEEq7QkqBLYX89x9/fGPbnVzr2Ywmndng6grf3+fffttGQHJJbm+DLKPz+d8+++xfv//esdXj29Awo3TPHji5vY2ur8nDg316Gl9cGEdHbEV+cnPjn53BWuuGGSV6927x449tBLR/iskkyDJ8f2+fnvr9vn1ywj9izVD07p1xdMSyy8HK5aD7KMVkEl1fFx8+mC9e5N99V5YIw3j+PDw/93o9Jpfk5uZg5XKgGYVMp6xlMV+88Pt9r9cTP+Qws8vBZZRyfftnZ+H5ueCB5vFxfHHh9/ssu2Tv3wsqrBs0ySjFZOKMRtplFP7EKEJoy5SAP34Msoy1WQcil0MRSnB1xZ9BD113J61GuYvTebl0v+nh496d3xSxT07skxMml+Hbt9H1dXh+3tVH25sLpZhMng4TlCJ7/z64uuK3RlqKtiyXwQ8/PB1jd4NuZhTeKLQqkTJMLix7OaNR9+TSNaGQ6XT400+s3xBfXu653+D1el6v10m5dEco5XHv/iVS5qlc4osL7SeMGiyfvHt4QJeX+YcPu1uRuRWz+dzPMnR5abx542fZbD6XHdH/iW9ujDdv0OWl9/bt3cOD7HCa0ySjqPPjKJtpqTl15/V67qtXfH5R3+nohmtmn3355dLJkX3ClwRocfV19xNsLhSJ/YDKkgCNrrj6KXAVzYWy0UTJrqhZEqAROq5eaDjqsU9P9/87JtOpMxqtWhKgEZXVC3Q+jy8uZAe1Bs2e68EfP1qffy47il3C1gqq33pqJhRAFvC4BiAECAUQ4pPOLCGEEGKaJrd7KIoCIWQYBjO/Y36QCKHyPk/3fLq5Dd22otTFfvIToWCMi6LAGOd5bhgGIYRtmqYZxzGz9mPeMqZpliuMec6g/ymDH2hZ1pZCYb49aZoyF4UoirpkbsPsJ5MkkR2IAJVb+nme+77v+z5/x/f9PM/55irXEOYpWn6nfJLGjMfjymld193+tCowm808z1toYsSypI9i2zallNneSceyLNu2+WaXrCj1sp9c3pkNw5C1nUrRJStK7ewnlwuFdTWU8hvumBWldvaTK4fHzJiaD3NWsZ/E0zErSm4/GQRBEAR8KKAydXM9nudxL81VlA3jK92atSITpHtWlDraT9bdcGO9WsH6tiyL3TthiB9YD8Z4OByWVbJWu+pjGIZdYid3m9rmk7keQshgMEAImaaZpilCiFL68uXLNE1t2x4MBuwPBSql4saQRVFEUcQ+xRjzmx/b0G0rSo3sJ3c/KVgUhRY/EWAjYPYYEAImBQEhQCiAECAUQAidhFJMJvTxUXYUO6aYTLRwTtRJKM5ohO/vZUexY5zRKLm9lR3FenQSCiAREIpk7NNT2SEIAUIBhAChAEKAUOSjRQ8dhAIIAUIBhAChAEKAUCSj/uPpDBCKZIyjI9khCAFCAYQAocgHJgUBIcjDg+wQ1gNCAYQAoQBCgFAAIUAoktHF3lInofhnZ7rcnhLHPD72z85kR7EeeK4HEEKnjAJIBIQCCKGuUMomlAqieHgVto9WXaEEQVA2X9knSZIEQTAcDmsC2Cg8QgjzzKnZp1XZbX8x1RUKMymV8tXMhMIwjJrK2yg80zTDMKyvquFwuFmUm7D9xVT3PwUVd0zceXitZpTto1VRKDxPLjVtwxizHG4YBvuVLLWgybKMedCx3Xzf5248g8GAOW8z5zT2cxf3/KkJL0kSbszMTPAIIcySiO9QFAWltPylhBDWzDmOw3YzDKN81DZlqYm2/thyWdI0bfLnk/uhYoTMsSxrNpux1+PxeKmbb5qmzOuX71axMTYMI45j/ml557UBrPo0TdOKkXPltPVfusqWuNWyrDq2UpbFYqGfUFzXTdOUb97d3S3dp/JOHMfloyq1srSSNhVK5SSz2az+W0RiWLRcllXHPj2Jik1PPXEcs1EJIcQwjKWtb1EUPI1z9uygz1vGLVGhLEjNPkoNrMfHvTcppY7jjMfjym62bZfb+P1Q6eWwvyrZ/rQqlAWpPDxeCsa4/F8Uq3qgrutWXEZ3VW01eJ43HA6ZlJmD8kaOh6Zplgc+/LUKZUEKTgoyR02EEGtZmBTiOGZpvCiKJEn4+4QQ27aXtj5RFPF/WGBF5bdGBoNB2bEzCIIkSfhmfQAi4bEq930/SRJ2zqdf6jgOxth13TiOWcBsNMe0xQLmH7VRlrXHlssSx7FyQhGB2d0ihMp/vLEU5pFsWZYU1+vhcMgrWwRerqUByy2LlkLRgqIoiqJQ3GZYHM06s4rD8jl7bVmW1ioplyXPc8gogBCajXoAWYBQACFAKIAQIBRAiP8CBCFm1Qc2iiMAAAAASUVORK5CYII=", + "text/plain": [ + "Tree('4', [Tree('2', ['NLU']), Tree('4', [Tree('2', ['is']), Tree('4', ['enlightening'])])])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Tree.fromstring(\"\"\"(4 (2 NLU) (4 (2 is) (4 enlightening)))\"\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### All-nodes readers\n", + "\n", + "In SST parlance, the __all-nodes task__ trains and assesses, not just with the full sentence, but also with all the labeled subtrees. We won't explore this task here, but it's good to know about it, and these readers will give you access to this version of the dataset:\n", + " * `sst.allnodes_train_reader`\n", + " * `sst.allnodes_dev_reader`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Methodological notes\n", + "\n", + "* We've deliberately ignored `test` readers. We urge you not to use the `test` set until and unless you are running experiments for a final project or similar. Overuse of test-sets corrupts them, since even subtle lessons learned from those runs can be incorporated back into model-building efforts.\n", + "\n", + "* We actually have mixed feelings about the overuse of `dev` that might result from working with these notebooks! We've tried to encourage using just splits of the training data for assessment most of the time, with only occasionally use of `dev`. This will give you a clearer picture of how you will ultimately do on test; over-use of `dev` can lead to over-fitting on that particular dataset with a resulting loss of performance of `test`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true, + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Modeling the SST labels\n", + "\n", + "Working with the SST involves making decisions about how to handle the raw SST labels. The interpretation of these labels is as follows ([Socher et al., sec. 3](http://www.aclweb.org/anthology/D/D13/D13-1170.pdf)):\n", + "\n", + "* `'0'`: very negative\n", + "* `'1'`: negative\n", + "* `'2'`: neutral\n", + "* `'3'`: positive\n", + "* `'4'`: very positive\n", + "\n", + "The labels look like they could be treated as totally ordered, even continuous. However, conceptually, they do not form such an order. Rather, they consist of three separate classes, with the negative and positive classes being totally ordered in opposite directions:\n", + "\n", + "* `'0' > '1'`: negative\n", + "* `'2'`: neutral\n", + "* `'4' > '3'`: positive\n", + "\n", + "Thus, in this notebook, we'll look mainly at binary (positive/negative) and ternary tasks.\n", + "\n", + "A related note: the above shows that the __fine-grained sentiment task__ for the SST is particularly punishing as usually formulated, since it ignores the partial-order structure in the categories completely. As a result, mistaking `'0'` for `'1'` is as bad as mistaking `'0'` for `'4'`, though the first error is clearly less severe than the second.\n", + "\n", + "The functions `sst.binary_class_func` and `sst.ternary_class_func` will convert the labels for you. Let's now use them to study the label distributions." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Train label distributions\n", + "\n", + "Check that these numbers all match those reported in [Socher et al. 2013, sec 5.1](http://www.aclweb.org/anthology/D/D13/D13-1170.pdf)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "train_labels = [y for tree, y in sst.train_reader()]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total train examples: 8,544\n" + ] + } + ], + "source": [ + "print(\"Total train examples: {:,}\".format(len(train_labels)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Distribution over the full label set:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3 2322\n", + "1 2218\n", + "2 1624\n", + "4 1288\n", + "0 1092\n", + "dtype: int64" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.Series(train_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Binary label conversion:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "binary_train_labels = [\n", + " y for tree, y in sst.train_reader(class_func=sst.binary_class_func)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total binary train examples: 6,920\n" + ] + } + ], + "source": [ + "print(\"Total binary train examples: {:,}\".format(len(binary_train_labels)))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "positive 3610\n", + "negative 3310\n", + "dtype: int64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.Series(binary_train_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Ternary label conversion:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "positive 3610\n", + "negative 3310\n", + "neutral 1624\n", + "dtype: int64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ternary_train_labels = [\n", + " y for tree, y in sst.train_reader(class_func=sst.ternary_class_func)]\n", + "\n", + "pd.Series(ternary_train_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Dev label distributions\n", + "\n", + "Check that these numbers all match those reported in [Socher et al. 2013, sec 5.1](http://www.aclweb.org/anthology/D/D13/D13-1170.pdf)." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "dev_labels = [y for tree, y in sst.dev_reader()]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total dev examples: 1,101\n" + ] + } + ], + "source": [ + "print(\"Total dev examples: {:,}\".format(len(dev_labels)))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1 289\n", + "3 279\n", + "2 229\n", + "4 165\n", + "0 139\n", + "dtype: int64" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.Series(dev_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Binary label conversion:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "binary_dev_labels = [\n", + " y for tree, y in sst.dev_reader(class_func=sst.binary_class_func)]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total binary dev examples: 872\n" + ] + } + ], + "source": [ + "print(\"Total binary dev examples: {:,}\".format(len(binary_dev_labels)))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "positive 444\n", + "negative 428\n", + "dtype: int64" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.Series(binary_dev_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Ternary label conversion:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "positive 444\n", + "negative 428\n", + "neutral 229\n", + "dtype: int64" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ternary_dev_labels = [\n", + " y for tree, y in sst.dev_reader(class_func=sst.ternary_class_func)]\n", + "\n", + "pd.Series(ternary_dev_labels).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Additional sentiment resources\n", + "\n", + "Here are a few publicly available datasets and other resources; if you decide to work on sentiment analysis, get in touch with the teaching staff — we have a number of other resources that we can point you to.\n", + "\n", + "* Sentiment lexica: http://sentiment.christopherpotts.net/lexicons.html\n", + "* NLTK now has a SentiWordNet module: http://www.nltk.org/api/nltk.corpus.reader.html#module-nltk.corpus.reader.sentiwordnet\n", + "* Stanford Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/index.html\n", + "* SemEval-2013: Sentiment Analysis in Twitter: https://www.cs.york.ac.uk/semeval-2013/task2/\n", + "* Starter code for a sentiment-aware tokenizer: http://sentiment.christopherpotts.net/code-data/happyfuntokenizing.py" + ] + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "widgets": { + "state": {}, + "version": "1.1.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/sst_02_hand_built_features.ipynb b/sst_02_hand_built_features.ipynb new file mode 100644 index 0000000..2277b66 --- /dev/null +++ b/sst_02_hand_built_features.ipynb @@ -0,0 +1,993 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Supervised sentiment: Hand-built feature functions" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "__author__ = \"Christopher Potts\"\n", + "__version__ = \"CS224u, Stanford, Spring 2018 term\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "0. [Overview](#Overview)\n", + "0. [Set-up](#Set-up)\n", + "0. [Feature functions](#Feature-functions)\n", + "0. [Building datasets for experiments](#Building-datasets-for-experiments)\n", + "0. [Basic optimization](#Basic-optimization)\n", + " 0. [Wrapper for SGDClassifier](#Wrapper-for-SGDClassifier)\n", + " 0. [Wrapper for LogisticRegression](#Wrapper-for-LogisticRegression)\n", + " 0. [Other scikit-learn models](#Other-scikit-learn-models)\n", + "0. [Experiments](#Experiments)\n", + " 0. [Experiment with default values](#Experiment-with-default-values)\n", + " 0. [A dev set run](#A-dev-set-run)\n", + " 0. [Assessing BasicSGDClassifier](#Assessing-BasicSGDClassifier)\n", + " 0. [Comparison with the baselines from Socher et al. 2013](#Comparison-with-the-baselines-from-Socher-et-al.-2013)\n", + " 0. [A shallow neural network classifier](#A-shallow-neural-network-classifier)\n", + "0. [Hyperparameter search](#Hyperparameter-search)\n", + " 0. [sst.fit_classifier_with_crossvalidation](#sst.fit_classifier_with_crossvalidation)\n", + " 0. [Example using LogisticRegression](#Example-using-LogisticRegression)\n", + " 0. [Example using BasicSGDClassifier](#Example-using-BasicSGDClassifier)\n", + "0. [Statistical comparison of classifier models](#Statistical-comparison-of-classifier-models)\n", + "0. [Exploratory exercise: The core development cycle](#Exploratory-exercise:-The-core-development-cycle)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Overview\n", + "\n", + "* The focus of this notebook is __building feature representations__ for use with (mostly linear) classifiers (though you're encouraged to try out some non-linear ones as well!)\n", + "\n", + "* The core characteristics of the feature functions we'll build here:\n", + " * They represent examples in __very large, very sparse feature spaces__.\n", + " * The individual feature functions can be __highly refined__, drawing on expert human knowledge of the domain. \n", + " * Taken together, these representations don't comprehensively represent the input examples. They just identify aspects of the inputs that the classifier model can make good use of (we hope).\n", + " \n", + "* These classifiers tend to be __highly competitive__. We'll look at more powerful deep learning models in the next notebook, and it will immediately become apparent that it is very difficult to get them to measure up to well-built classifiers based in sparse feature representations." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Set-up\n", + "\n", + "See [the previous notebook](sst_01_overview.ipynb#Set-up) for set-up instructions." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Applications/anaconda/envs/nlu/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + " from ._conv import register_converters as _register_converters\n" + ] + } + ], + "source": [ + "from collections import Counter\n", + "from sklearn.linear_model import LogisticRegression\n", + "import scipy.stats\n", + "from sgd_classifier import BasicSGDClassifier\n", + "from tf_shallow_neural_classifier import TfShallowNeuralClassifier\n", + "import sst\n", + "import utils" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Feature functions\n", + "\n", + "* Feature representation is arguably __the most important step in any machine learning task__. As you experiment with the SST, you'll come to appreciate this fact, since your choice of feature function will have a far greater impact on the effectiveness of your models than any other choice you make.\n", + "\n", + "* We will define our feature functions as `dict`s mapping feature names (which can be any object that can be a `dict` key) to their values (which must be `bool`, `int`, or `float`). \n", + "\n", + "* To prepare for optimization, we will use `sklearn`'s [DictVectorizer](http://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.DictVectorizer.html) class to turn these into matrices of features. \n", + "\n", + "* The `dict`-based approach gives us a lot of flexibility and frees us from having to worry about the underlying feature matrix." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "A typical baseline or default feature representation in NLP or NLU is built from unigrams. Here, those are the leaf nodes of the tree:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def unigrams_phi(tree):\n", + " \"\"\"The basis for a unigrams feature function.\n", + " \n", + " Parameters\n", + " ----------\n", + " tree : nltk.tree\n", + " The tree to represent.\n", + " \n", + " Returns\n", + " ------- \n", + " defaultdict\n", + " A map from strings to their counts in `tree`. (Counter maps a \n", + " list to a dict of counts of the elements in that list.)\n", + " \n", + " \"\"\"\n", + " return Counter(tree.leaves())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "In the docstring for `sst.sentiment_treebank_reader`, I pointed out that the labels on the subtrees can be used in a way that feels like cheating. Here's the most dramatic instance of this: `root_daughter_scores_phi` uses just the labels on the daughters of the root to predict the root (label). This will result in performance well north of 90% F1, but that's hardly worth reporting. (Interestingly, using the labels on the leaf nodes is much less powerful.) Anyway, don't use this function!" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def root_daughter_scores_phi(tree): \n", + " \"\"\"The best way we've found to cheat without literally using the \n", + " labels as part of the feature representations. \n", + " \n", + " Don't use this for any real experiments!\n", + " \n", + " \"\"\"\n", + " return Counter([child.label() for child in tree])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's generally good design to __write lots of atomic feature functions__ and then bring them together into a single function when running experiments. This will lead to reusable parts that you can assess independently and in sub-groups as part of development." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Building datasets for experiments\n", + "\n", + "The second major phase for our analysis is a kind of set-up phase. Ingredients:\n", + "\n", + "* A reader like `train_reader`\n", + "* A feature function like `unigrams_phi`\n", + "* A class function like `binary_class_func`\n", + "\n", + "The convenience function `sst.build_dataset` uses these to build a dataset for training and assessing a model. See its documentation for details on how it works. Much of this is about taking advantage of `sklearn`'s many functions for model building." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset = sst.build_dataset(\n", + " reader=sst.train_reader,\n", + " phi=unigrams_phi,\n", + " class_func=sst.binary_class_func,\n", + " vectorizer=None)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train dataset with unigram features has 6,920 examples and 16,282 features\n" + ] + } + ], + "source": [ + "print(\"Train dataset with unigram features has {:,} examples and {:,} features\".format(\n", + " *train_dataset['X'].shape))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that `sst.build_dataset` has an optional argument `vectorizer`:\n", + "\n", + "* If it is `None`, then a new vectorizer is used and returned as `dataset['vectorizer']`. This is the usual scenario when training. \n", + "\n", + "* For evaluation, one wants to represent examples exactly as they were represented during training. To ensure that this happens, pass the training `vectorizer` to this function:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "dev_dataset = sst.build_dataset(\n", + " reader=sst.dev_reader,\n", + " phi=unigrams_phi,\n", + " class_func=sst.binary_class_func,\n", + " vectorizer=train_dataset['vectorizer'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dev dataset with unigram features has 872 examples and 16,282 features\n" + ] + } + ], + "source": [ + "print(\"Dev dataset with unigram features has {:,} examples and {:,} features\".format(\n", + " *dev_dataset['X'].shape))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Basic optimization\n", + "\n", + "We're now in a position to begin training supervised models!\n", + "\n", + "For the most part, in this course, we will not study the theoretical aspects of machine learning optimization, concentrating instead on how to optimize systems effectively in practice. That is, this isn't a theory course, but rather an experimental, project-oriented one.\n", + "\n", + "Nonetheless, we do want to avoid treating our optimizers as black boxes that work their magic and give us some assessment figures for whatever we feed into them. That seems irresponsible from a scientific and engineering perspective, and it also sends the false signal that the optimization process is inherently mysterious. So we do want to take a minute to demystify it with some simple code.\n", + "\n", + "The module `sgd_classifier` contains a complete optimization framework, as `BasicSGDClassifier`. Well, it's complete in the sense that it achieves our full task of supervised learning. It's incomplete in the sense that it is very basic. You probably wouldn't want to use it in experiments. Rather, we're going to encourage you to rely on `sklearn` for your experiments (see below). Still, this is a good basic picture of what's happening under the hood.\n", + "\n", + "So what is `BasicSGDClassifier` doing? The heart of it is the `fit` function (reflecting the usual `sklearn` naming system). This method implements a hinge-loss stochastic sub-gradient descent optimization. Intuitively, it works as follows:\n", + "\n", + "0. Start by assuming that all the feature weights are `0`.\n", + "0. Move through the dataset instance-by-instance in random order.\n", + "0. For each instance, classify it using the current weights. \n", + "0. If the classification is incorrect, move the weights in the direction of the correct classification\n", + "\n", + "This process repeats for a user-specified number of iterations (default `10` below), and the weight movement is tempered by a learning-rate parameter `eta` (default `0.1`). The output is a set of weights that can be used to make predictions about new (properly featurized) examples.\n", + "\n", + "In more technical terms, the objective function is \n", + "\n", + "$$\n", + " \\min_{\\mathbf{w} \\in \\mathbb{R}^{d}}\n", + " \\sum_{(x,y)\\in\\mathcal{D}} \n", + " \\max_{y'\\in\\mathbf{Y}}\n", + " \\left[\\mathbf{Score}_{\\textbf{w}, \\phi}(x,y') + \\mathbf{cost}(y,y')\\right] - \\mathbf{Score}_{\\textbf{w}, \\phi}(x,y)\n", + "$$\n", + "\n", + "where $\\mathbf{w}$ is the set of weights to be learned, $\\mathcal{D}$ is the training set of example–label pairs, $\\mathbf{Y}$ is the set of labels, $\\mathbf{cost}(y,y') = 0$ if $y=y'$, else $1$, and $\\mathbf{Score}_{\\textbf{w}, \\phi}(x,y')$ is the inner product of the weights \n", + "$\\mathbf{w}$ and the example as featurized according to $\\phi$.\n", + "\n", + "The `fit` method is then calculating the sub-gradient of this objective. In succinct pseudo-code:\n", + "\n", + "* Initialize $\\mathbf{w} = \\mathbf{0}$\n", + "* Repeat $T$ times:\n", + " * for each $(x,y)$ in $\\mathcal{D}$ (in random order):\n", + " * $\\tilde{y} = \\text{argmax}_{y'\\in \\mathcal{Y}} \\mathbf{Score}_{\\textbf{w}, \\phi}(x,y') + \\mathbf{cost}(y,y')$\n", + " * $\\mathbf{w} = \\mathbf{w} + \\eta(\\phi(x,y) - \\phi(x,\\tilde{y}))$\n", + " \n", + "This is very intuitive – push the weights in the direction of the positive cases. It doesn't require any probability theory. And such loss functions have proven highly effective in many settings. For a more powerful version of this classifier, see [sklearn.linear_model.SGDClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier). With `loss='hinge'`, it should behave much like `BasicSGDClassifier` (but faster!)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Wrapper for SGDClassifier\n", + "\n", + "For the sake of our experimental framework, a simple wrapper for `SGDClassifier`:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_basic_sgd_classifier(X, y): \n", + " \"\"\"Wrapper for `BasicSGDClassifier`.\n", + " \n", + " Parameters\n", + " ----------\n", + " X : 2d np.array\n", + " The matrix of features, one example per row.\n", + " \n", + " y : list\n", + " The list of labels for rows in `X`.\n", + " \n", + " Returns\n", + " -------\n", + " BasicSGDClassifier\n", + " A trained `BasicSGDClassifier` instance.\n", + " \n", + " \"\"\" \n", + " mod = BasicSGDClassifier()\n", + " mod.fit(X, y)\n", + " return mod" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Wrapper for LogisticRegression\n", + "\n", + "As I said above, we likely don't want to rely on `BasicSGDClassifier` (though it does a good job with SST!). Instead, we want to rely on `sklearn`. Here's a simple wrapper for [sklearn.linear.model.LogisticRegression](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) using our \n", + "`build_dataset` paradigm." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_maxent_classifier(X, y): \n", + " \"\"\"Wrapper for `sklearn.linear.model.LogisticRegression`. This is also \n", + " called a Maximum Entropy (MaxEnt) Classifier, which is more fitting \n", + " for the multiclass case.\n", + " \n", + " Parameters\n", + " ----------\n", + " X : 2d np.array\n", + " The matrix of features, one example per row.\n", + " \n", + " y : list\n", + " The list of labels for rows in `X`.\n", + " \n", + " Returns\n", + " -------\n", + " sklearn.linear.model.LogisticRegression\n", + " A trained `LogisticRegression` instance.\n", + " \n", + " \"\"\"\n", + " mod = LogisticRegression(fit_intercept=True)\n", + " mod.fit(X, y)\n", + " return mod" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Other scikit-learn models\n", + "\n", + "* The [sklearn.linear_model](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.linear_model) package has a number of other classifier models that could be effective for SST.\n", + "\n", + "* The [sklearn.ensemble](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.ensemble) package contains powerful classifiers as well. The theme that runs through all of them is that one can get better results by averaging the predictions of a bunch of more basic classifiers. A [RandomForestClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier) will bring some of the power of deep learning models without the optimization challenges (though see [this blog post on some limitations of the current sklearn implementation](https://roamanalytics.com/2016/10/28/are-categorical-variables-getting-lost-in-your-random-forests/)).\n", + "\n", + "* The [sklearn.svm](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.svm) contains variations on Support Vector Machines (SVMs)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Experiments\n", + "\n", + "We now have all the pieces needed to run experiments. And we're going to want to run a lot of experiments, trying out different feature functions, taking different perspectives on the data and labels, and using different models. \n", + "\n", + "To make that process efficient and regimented, `sst` contains a function `experiment`. All it does is pull together these pieces and use them for training and assessment. It's complicated, but the flexibility will turn out to be an asset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Experiment with default values" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.617\n", + " precision recall f1-score support\n", + "\n", + " negative 0.634 0.696 0.664 997\n", + " neutral 0.239 0.106 0.147 483\n", + " positive 0.666 0.772 0.715 1084\n", + "\n", + "avg / total 0.573 0.617 0.588 2564\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_maxent_classifier,\n", + " train_reader=sst.train_reader, \n", + " assess_reader=None, \n", + " train_size=0.7,\n", + " class_func=sst.ternary_class_func,\n", + " score_func=utils.safe_macro_f1,\n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A few notes on this function call:\n", + " \n", + "* Since `assess_reader=None`, the function reports performance on a random train–test split. Give `sst.dev_reader` as the argument to assess against the `dev` set.\n", + "\n", + "* `unigrams_phi` is the function we defined above. By changing/expanding this function, you can start to improve on the above baseline, perhaps periodically seeing how you do on the dev set.\n", + "\n", + "* `fit_maxent_classifier` is the wrapper we defined above. To assess new models, simply define more functions like this one. Such functions just need to consume an `(X, y)` constituting a dataset and return a model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### A dev set run" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.602\n", + " precision recall f1-score support\n", + "\n", + " negative 0.628 0.689 0.657 428\n", + " neutral 0.343 0.153 0.211 229\n", + " positive 0.629 0.750 0.684 444\n", + "\n", + "avg / total 0.569 0.602 0.575 1101\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_maxent_classifier,\n", + " class_func=sst.ternary_class_func,\n", + " assess_reader=sst.dev_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Assessing BasicSGDClassifier" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.572\n", + " precision recall f1-score support\n", + "\n", + " negative 0.624 0.589 0.606 428\n", + " neutral 0.293 0.170 0.215 229\n", + " positive 0.601 0.764 0.673 444\n", + "\n", + "avg / total 0.546 0.572 0.552 1101\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_basic_sgd_classifier,\n", + " class_func=sst.ternary_class_func,\n", + " assess_reader=sst.dev_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Comparison with the baselines from Socher et al. 2013\n", + "\n", + "Where does our default set-up sit with regard to published baselines for the binary problem? (Compare [Socher et al., Table 1](http://www.aclweb.org/anthology/D/D13/D13-1170.pdf).)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.772\n", + " precision recall f1-score support\n", + "\n", + " negative 0.783 0.741 0.761 428\n", + " positive 0.762 0.802 0.782 444\n", + "\n", + "avg / total 0.772 0.772 0.772 872\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_maxent_classifier,\n", + " class_func=sst.binary_class_func,\n", + " assess_reader=sst.dev_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### A shallow neural network classifier\n", + "\n", + "While we're at it, we might as well see whether adding a hidden layer to our maxent classifier yields any benefits. Whereas `LogisticRegression` is, at its core, computing\n", + "\n", + "$$\\begin{align*}\n", + "y &= \\textbf{softmax}(xW_{xy} + b_{y})\n", + "\\end{align*}$$\n", + "\n", + "this model inserts a hidden layer with a non-linear activation applied to it:\n", + "\n", + "$$\\begin{align*}\n", + "h &= \\tanh(xW_{xh} + b_{h}) \\\\\n", + "y &= \\textbf{softmax}(hW_{hy} + b_{y})\n", + "\\end{align*}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_nn_classifier(X, y):\n", + " mod = TfShallowNeuralClassifier(hidden_dim=50, max_iter=100)\n", + " mod.fit(X, y)\n", + " return mod" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Iteration 100: loss: 3.178435742855072" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.645\n", + " precision recall f1-score support\n", + "\n", + " negative 0.623 0.595 0.609 964\n", + " positive 0.662 0.687 0.674 1112\n", + "\n", + "avg / total 0.644 0.645 0.644 2076\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_nn_classifier,\n", + " class_func=sst.binary_class_func)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It looks like, with enough iterations (and perhaps some fiddling with the activation function and hidden dimensionality), this classifier would meet or exceed the baseline set up by `LogisticRegression`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Hyperparameter search\n", + "\n", + "The training process learns __parameters__ — the weights. There are typically lots of other parameters that need to be set. For instance, our `BasicSGDClassifier` has a learning rate parameter and a training iteration parameter. These are called __hyperparameters__. The more powerful `sklearn` classifiers often have many more such hyperparameters. These are outside of the explicitly stated objective, hence the \"hyper\" part. \n", + "\n", + "So far, we have just set the hyperparameters by hand. However, their optimal values can vary widely between datasets, and choices here can dramatically impact performance, so we would like to set them as part of the overall experimental framework." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### sst.fit_classifier_with_crossvalidation\n", + "\n", + "Luckily, `sklearn` provides a lot of functionality for setting hyperparameters via cross-validation. The function `sst.fit_classifier_with_crossvalidation` implements a basic framework for taking advantage of these options. \n", + "\n", + "\n", + "This method has the same basic shape as `fit_maxent_classifier` above: it takes a dataset as input and returns a trained model. However, to find its favored model, it explores a space of hyperparameters supplied by the user, seeking the optimal combination of settings.\n", + "\n", + "__Note__: this kind of search seems not to have a large impact for SST as we're using it. However, it can matter a lot for other data sets, and it's also an important step to take when trying to publish, since __reviewers are likely to want to check that your comparisons aren't based in part on opportunistic or ill-considered choices for the hyperparameters__." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Example using LogisticRegression\n", + "\n", + "Here's a fairly full-featured use of the above for the `LogisisticRegression` model family:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_maxent_with_crossvalidation(X, y):\n", + " \"\"\"A MaxEnt model of dataset with hyperparameter \n", + " cross-validation. Some notes:\n", + " \n", + " * 'fit_intercept': whether to include the class bias feature.\n", + " * 'C': weight for the regularization term (smaller is more regularized).\n", + " * 'penalty': type of regularization -- roughly, 'l1' ecourages small \n", + " sparse models, and 'l2' encourages the weights to conform to a \n", + " gaussian prior distribution.\n", + " \n", + " Other arguments can be cross-validated; see \n", + " http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\n", + " \n", + " Parameters\n", + " ----------\n", + " X : 2d np.array\n", + " The matrix of features, one example per row.\n", + " \n", + " y : list\n", + " The list of labels for rows in `X`. \n", + " \n", + " Returns\n", + " -------\n", + " sklearn.linear_model.LogisticRegression\n", + " A trained model instance, the best model found.\n", + " \n", + " \"\"\" \n", + " basemod = LogisticRegression()\n", + " cv = 5\n", + " param_grid = {'fit_intercept': [True, False], \n", + " 'C': [0.4, 0.6, 0.8, 1.0, 2.0, 3.0],\n", + " 'penalty': ['l1','l2']} \n", + " return sst.fit_classifier_with_crossvalidation(X, y, basemod, cv, param_grid)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best params {'C': 2.0, 'fit_intercept': True, 'penalty': 'l2'}\n", + "Best score: 0.755\n", + "Accuracy: 0.772\n", + " precision recall f1-score support\n", + "\n", + " negative 0.762 0.742 0.752 966\n", + " positive 0.781 0.798 0.789 1110\n", + "\n", + "avg / total 0.772 0.772 0.772 2076\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_maxent_with_crossvalidation, \n", + " class_func=sst.binary_class_func)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Example using BasicSGDClassifier" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The models written for this course are also compatible with this framework. They [\"duck type\"](https://en.wikipedia.org/wiki/Duck_typing) the sklearn models by having methods `fit`, `predict`, `get_params`, and `set_params`, and an attribute `params`." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_basic_sgd_classifier_with_crossvalidation(X, y):\n", + " basemod = BasicSGDClassifier()\n", + " cv = 5\n", + " param_grid = {'eta': [0.01, 0.1, 1.0], 'max_iter': [10]}\n", + " return sst.fit_classifier_with_crossvalidation(X, y, basemod, cv, param_grid)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best params {'eta': 0.01, 'max_iter': 10}\n", + "Best score: 0.743\n", + "Accuracy: 0.752\n", + " precision recall f1-score support\n", + "\n", + " negative 0.717 0.787 0.750 980\n", + " positive 0.791 0.722 0.755 1096\n", + "\n", + "avg / total 0.756 0.752 0.753 2076\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " unigrams_phi,\n", + " fit_basic_sgd_classifier_with_crossvalidation, \n", + " class_func=sst.binary_class_func)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Statistical comparison of classifier models\n", + "\n", + "Suppose two classifiers differ according to an effectiveness measure like F1 or accuracy. Are they meaningfully different?\n", + "\n", + "* For very large datasets, the answer might be clear: if performance is very stable across different train/assess splits and the difference in terms of correct predictions has practical import, then you can clearly say yes. \n", + "\n", + "* With smaller datasets, or models whose performance is closer together, it can be harder to determine whether the two models are different. We can address this question in a basic way with repeated runs and basic null-hypothesis testing on the resulting score vectors.\n", + "\n", + "The function `sst.compare_models` is designed for such testing. The default set-up uses the non-parametric [Wilcoxon signed-rank test](https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test) to make the comparisons, which is relatively conservative and recommended by [DemÅ¡ar 2006](http://www.jmlr.org/papers/v7/demsar06a.html) for cases where one can afford to do multiple assessments.\n", + "\n", + "Here's an example showing the default parameters values and comparing `LogisticRegression` and `BasicSGDClassifier`:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model 1 mean: 0.515\n", + "Model 2 mean: 0.505\n", + "p = 0.074\n" + ] + } + ], + "source": [ + "_ = sst.compare_models(\n", + " unigrams_phi,\n", + " fit_maxent_classifier,\n", + " stats_test=scipy.stats.wilcoxon,\n", + " trials=10,\n", + " phi2=None, # Defaults to same as first required argument.\n", + " train_func2=fit_basic_sgd_classifier, # Defaults to same as second required argument.\n", + " reader=sst.train_reader, \n", + " train_size=0.7, \n", + " class_func=sst.ternary_class_func, \n", + " score_func=utils.safe_macro_f1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In general, one wants to compare __two feature functions against the same model__, or one wants to compare __two models with the same feature function used for both__. If both are changed at the same time, then it will be hard to figure out what is causing any differences you see." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Exploratory exercise: The core development cycle\n", + "\n", + "In order to get a feel for the codebase and prepare for the in-class bake-off, we suggest some rounds of __the basic development cycle for models based in hand-built feature functions__:\n", + "\n", + "0. Write a new feature function. We recommend starting with something simple.\n", + "0. Use `sst.experiment` to evaluate your new feature function on the binary and ternary versons of SST, with at least `fit_basic_sgd_classifier` and `fit_maxent_classifier`.\n", + "0. If you have time, compare your feature function with `unigrams_phi` using `compare_models`.\n", + "0. Return to step 1, or stop the cycle and conduct a more rigorous evaluation with hyperparameter tuning and assessment on the `dev` set.\n", + "\n", + "Error analysis is one of the most important methods for steadily improving a system, as it facilitates a kind of human-powered hill-climbing on your ultimate objective. Often, it takes a careful human analyst just a few examples to spot a major pattern that can lead to a beneficial change to the feature representations.\n", + "\n", + "To bring error analysis into your development cycle, you could improve `sst.experiment` by adding a keyword argument `view_errors` with default value `0`. Where the value is `n`, the function prints out a random selection of `n` errors: the underlying tree, the correct label, and the predicted label." + ] + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "widgets": { + "state": {}, + "version": "1.1.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/sst_03_neural_networks.ipynb b/sst_03_neural_networks.ipynb new file mode 100644 index 0000000..7541586 --- /dev/null +++ b/sst_03_neural_networks.ipynb @@ -0,0 +1,1187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Supervised sentiment: Dense feature representations and neural networks" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "__author__ = \"Christopher Potts\"\n", + "__version__ = \"CS224u, Stanford, Spring 2018 term\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "0. [Overview of this unit](#Overview-of-this-unit)\n", + "0. [Set-up](#Set-up)\n", + "0. [Distributed representations as features](#Distributed-representations-as-features)\n", + " 0. [GloVe inputs](#GloVe-inputs)\n", + " 0. [IMDB representations](#IMDB-representations)\n", + " 0. [Remarks on this approach](#Remarks-on-this-approach)\n", + "0. [RNN classifiers](#RNN-classifiers)\n", + " 0. [RNN dataset preparation](#RNN-dataset-preparation)\n", + " 0. [Vocabulary for embedding](#Vocabulary-for-embedding)\n", + " 0. [Pure NumPy RNN implementation](#Pure-NumPy-RNN-implementation)\n", + " 0. [TensorFlow implementation](#TensorFlow-implementation)\n", + "0. [Tree-structured neural networks](#Tree-structured-neural-networks)\n", + " 0. [TreeNN dataset preparation](#TreeNN-dataset-preparation)\n", + " 0. [Pure NumPy TreeNN implementation](#Pure-NumPy-TreeNN-implementation)\n", + "0. [Exploratory exercises](#Exploratory-exercises)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview of this unit" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set-up\n", + "\n", + "See [the first notebook in this unit](sst_01_overview.ipynb#Set-up) for set-up instructions." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Applications/anaconda/envs/nlu/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + " from ._conv import register_converters as _register_converters\n" + ] + } + ], + "source": [ + "from collections import Counter\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import random\n", + "from rnn_classifier import RNNClassifier\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import classification_report\n", + "import tensorflow as tf\n", + "from tf_rnn_classifier import TfRNNClassifier\n", + "from tree_nn import TreeNN\n", + "import sst\n", + "import vsm\n", + "import utils" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "vsmdata_home = 'vsmdata'\n", + "\n", + "glove_home = os.path.join(vsmdata_home, 'glove.6B')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Distributed representations as features\n", + "\n", + "As a first step in the direction of neural networks for sentiment, we can connect with our previous unit on distributed representations. Arguably, more than any specific model architecture, this is the major innovation of deep learning: __rather than designing feature functions by hand, we use dense, distributed representations, often derived from unsupervised models__." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our model will just be `LogisticRegression`, and we'll continue with the experiment framework from the previous notebook. Here's is `fit_maxent_classifier` again:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_maxent_classifier(X, y): \n", + " mod = LogisticRegression(fit_intercept=True)\n", + " mod.fit(X, y)\n", + " return mod" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### GloVe inputs\n", + "\n", + "To illustrate this process, we'll use the general purpose GloVe representations released by the GloVe team, at 50d:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "glove_lookup = utils.glove2dict(\n", + " os.path.join(glove_home, 'glove.6B.50d.txt'))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def vsm_leaves_phi(tree, lookup, np_func=np.sum):\n", + " \"\"\"Represent tree as a combination of the vector of its words.\n", + " \n", + " Parameters\n", + " ----------\n", + " tree : nltk.Tree \n", + " lookup : dict\n", + " From words to vectors.\n", + " np_func : function (default: np.sum)\n", + " A numpy matrix operation that can be applied columnwise, \n", + " like `np.mean`, `np.sum`, or `np.prod`. The requirement is that \n", + " the function take `axis=0` as one of its arguments (to ensure\n", + " columnwise combination) and that it return a vector of a \n", + " fixed length, no matter what the size of the tree is.\n", + " \n", + " Returns\n", + " -------\n", + " np.array, dimension `X.shape[1]`\n", + " \n", + " \"\"\"\n", + " dim = len(next(iter(lookup.values()))) \n", + " allvecs = np.array([lookup[w] for w in tree.leaves() if w in lookup])\n", + " if len(allvecs) == 0:\n", + " feats = np.zeros(dim)\n", + " else: \n", + " feats = np_func(allvecs, axis=0) \n", + " return feats" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def glove_leaves_phi(tree, np_func=np.sum):\n", + " return vsm_leaves_phi(tree, glove_lookup, np_func=np_func)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.732\n", + " precision recall f1-score support\n", + "\n", + " negative 0.715 0.718 0.717 981\n", + " positive 0.746 0.744 0.745 1095\n", + "\n", + "avg / total 0.732 0.732 0.732 2076\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " glove_leaves_phi,\n", + " fit_maxent_classifier,\n", + " class_func=sst.binary_class_func,\n", + " vectorize=False) # Tell `experiment` that we already have our feature vectors." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### IMDB representations\n", + "\n", + "Our IMDB VSMs seems pretty well-attuned to the Stanford Sentiment Treebank, so we might think that they can do even better than the general-purpose GloVe inputs. Here are two quick assessments of that idea:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "imdb20 = pd.read_csv(\n", + " os.path.join(vsmdata_home, 'imdb_window20-flat.csv.gz'), index_col=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "imdb20_ppmi = vsm.pmi(imdb20, positive=False) " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "imdb20_ppmi_svd = vsm.lsa(imdb20_ppmi, k=50) " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "imdb_lookup = dict(zip(imdb20_ppmi_svd.index, imdb20_ppmi_svd.values))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def imdb_phi(tree, np_func=np.sum):\n", + " return vsm_leaves_phi(tree, imdb_lookup, np_func=np_func)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.751\n", + " precision recall f1-score support\n", + "\n", + " negative 0.737 0.739 0.738 984\n", + " positive 0.764 0.762 0.763 1092\n", + "\n", + "avg / total 0.751 0.751 0.751 2076\n", + "\n" + ] + } + ], + "source": [ + "_ = sst.experiment(\n", + " imdb_phi,\n", + " fit_maxent_classifier,\n", + " class_func=sst.binary_class_func,\n", + " vectorize=False) # Tell `experiment` that we already have our feature vectors." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Remarks on this approach\n", + "\n", + "* Recall that our `ungrams_phi`created feature representations with over 16K dimensions and got about 0.77.\n", + "\n", + "* The above models have only 50 dimensions and come close in terms of performance. In many ways, it's striking that we can get a model that is pretty competitive with so few dimensions.\n", + "\n", + "* The promise of the Mittens model of [Dingwall and Potts 2018](https://arxiv.org/abs/1803.09901) is that we can use GloVe itself to update the general purpose information in the 'glove.6B' vectors with specialized information from one of these IMDB count matrices. That might be worth trying; the `mittens` package already implements this!\n", + "\n", + "* That said, just summing up all the word representations is pretty unappealing linguistically. There's no doubt that we're losing a lot of valuable information in doing this. The models we turn to now can be seen as addressing this shortcoming while retaining the insight that our distributed representations are valuable for this task." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## RNN classifiers\n", + "\n", + "A recurrent neural network (RNN) is any deep learning model that process its inputs sequentially. There are many variations on this theme. The one that we use here is a __RNN classifier__.\n", + "\n", + "\n", + "\n", + "For a sequence of length $n$:\n", + "\n", + "$$\\begin{align*}\n", + "h_{t} &= \\tanh(x_{t}W_{xh} + h_{t-1}W_{hh}) \\\\\n", + "y &= \\textbf{softmax}(h_{n}W_{hy} + b)\n", + "\\end{align*}$$\n", + "\n", + "where $1 \\leqslant t \\leqslant n$. As indicated in the above diagram, the sequence of hidden states is padded with an initial state $h_{0}$ In our implementations , this is always an all $0$ vector, but it can be initialized in more sophisticated ways (some of which we will explore in our unit on natural language inference).\n", + "\n", + "This is a potential gain over our sum-the-word-vectors baseline, in that it processes each word independently, and in the context of those that came before it. Thus, not only is this sensitive to word order, but the hidden representation give us the potential to encode how the preceding context for a word affects its interpretation.\n", + "\n", + "The downside of this, of course, is that this model is much more difficult to set up and optimize. Let's dive into those details." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RNN dataset preparation\n", + "\n", + "SST contains trees, but the RNN processes just the sequence of leaf nodes. The function `sst.build_binary_rnn_dataset` creates datasets in this format:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "X_rnn_train, y_rnn_train = sst.build_binary_rnn_dataset(sst.train_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, each member of `X_rnn_train` train is a list of lists of words. Here's a look at the start of the first:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['The', 'Rock', 'is', 'destined', 'to', 'be']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_rnn_train[0][: 6]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because this is a classifier, `y_rnn_train` is just a list of labels, one per example:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'positive'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y_rnn_train[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For experiments, let's build a `dev` dataset as well:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "X_rnn_dev, y_rnn_dev = sst.build_binary_rnn_dataset(sst.dev_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Vocabulary for embedding\n", + "\n", + "The first delicate issue we need to address is the vocabulary for our model:\n", + "\n", + "* As indicated in the figure above, the first thing we do when processing an example is look up the words in an embedding (a VSM), which has to have a fixed dimensionality. \n", + "\n", + "* We can use our training data to specify the vocabulary for this embedding; at prediction time, though, we will inevitably encounter words we haven't seen before. \n", + "\n", + "* The convention we adopt here is to map them to an `$UNK` token that is in our pre-specified vocabulary.\n", + "\n", + "* At the same time, we might want to collapse infrequent tokens into `$UNK` to make optimization easier.\n", + "\n", + "In `sst`, the function `get_vocab` implements these strategies:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can extract the training vocab and use it for the model embedding, secure in the knowledge that we will be able to process tokens outside of this set (by mapping them to `$UNK`)." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "sst_full_train_vocab = sst.get_vocab(X_rnn_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sst_full_train_vocab has 16,283 items\n" + ] + } + ], + "source": [ + "print(\"sst_full_train_vocab has {:,} items\".format(len(sst_full_train_vocab)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This frankly seems too big. Let's restrict to just 3000 words:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "sst_train_vocab = sst.get_vocab(X_rnn_train, n_words=3000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Pure NumPy RNN implementation\n", + "\n", + "The first implementation we'll look at is a pure NumPy implementation of exactly the model depicted above. This implementation is a bit slow and might not be all that effective, but it is useful to have available in case one really wants to inspect the fine details of how these models process examples." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "rnn = RNNClassifier(\n", + " sst_train_vocab,\n", + " embedding=None, # Will be randomly initialized.\n", + " embed_dim=50,\n", + " hidden_dim=50,\n", + " max_iter=50,\n", + " eta=0.05) " + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Finished epoch 50 of 50; error is 1.3376949593831804" + ] + } + ], + "source": [ + "rnn.fit(X_rnn_train, y_rnn_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "rnn_dev_predictions = rnn.predict(X_rnn_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " negative 0.48 0.65 0.55 428\n", + " positive 0.48 0.30 0.37 444\n", + "\n", + "avg / total 0.48 0.48 0.46 872\n", + "\n" + ] + } + ], + "source": [ + "print(classification_report(y_rnn_dev, rnn_dev_predictions))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### TensorFlow implementation\n", + "\n", + "The included TensorFlow implementation is much faster and more configurable. Its only downside is that it requires the user to specify a maximum length for all incoming sequences: \n", + "\n", + "* Examples that are shorter than this maximum are padded (and the implementation ignores those dimensions)\n", + "* Examples that are longer than this maximum are clipped from the start (on the assumption that later words in the sentences will tend to be more informative).\n", + "\n", + "The function `utils.sequence_length_report` will help you make informed decisions:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Max sequence length: 52\n", + "Min sequence length: 2\n", + "Mean sequence length: 19.30\n", + "Median sequence length: 19.00\n", + "Sequences longer than 50: 6 of 6,920\n" + ] + } + ], + "source": [ + "utils.sequence_length_report(X_rnn_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Thus, if we choose `max_length=52`, no training examples will get clipped. (When making predictions about new examples, some might still get clipped.)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The class `TfRNNClassifier` takes a parameter for specifying this maximum length. It has many others as well:\n", + " \n", + "* `hidden_activation`: the activation function for the hidden layers (default: `tf.nn.tanh`).\n", + "* `cell_class`: which TensorFlow cell-type to use: \n", + " * The default is an LSTM, which should help ensure that we get a good gradient signal all the way through even long sequences.\n", + " * `tf.nn.rnn_cell.BasicRNNCell` is the same as what we defined in pure NumPy.\n", + " * `tf.nn.rnn_cell.GRUCell` should be similar to the LSTM.\n", + "* `train_embedding`: whether to update the embedding during training." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "tf_rnn = TfRNNClassifier(\n", + " sst_train_vocab,\n", + " embed_dim=50,\n", + " hidden_dim=50,\n", + " max_length=52,\n", + " hidden_activation=tf.nn.tanh,\n", + " cell_class=tf.nn.rnn_cell.LSTMCell,\n", + " train_embedding=True,\n", + " max_iter=500,\n", + " eta=0.05) " + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Iteration 500: loss: 3.9221649765968323" + ] + } + ], + "source": [ + "_ = tf_rnn.fit(X_rnn_train, y_rnn_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "tf_rnn_dev_predictions = tf_rnn.predict(X_rnn_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " negative 0.73 0.54 0.62 428\n", + " positive 0.64 0.81 0.72 444\n", + "\n", + "avg / total 0.69 0.68 0.67 872\n", + "\n" + ] + } + ], + "source": [ + "print(classification_report(y_rnn_dev, tf_rnn_dev_predictions))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It can be challenging to make sense of the errors that these models are making, but we should try. Here's a function for viewing random errors:" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "def view_error(model):\n", + " data = list(zip(X_rnn_dev, y_rnn_dev))\n", + " for _ in range(len(data)):\n", + " ex, label = random.choice(data)\n", + " pred = model.predict([ex])[0]\n", + " if label != pred:\n", + " print(\" \".join(ex))\n", + " print(\"Correct label: {}\".format(label))\n", + " return" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Affleck and Jackson are good sparring partners .\n", + "Correct label: positive\n" + ] + } + ], + "source": [ + "view_error(tf_rnn)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It can also be informative to invent examples and see how the model deals with them:" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "def process_new_examples(model):\n", + " examples = [\n", + " ['great'], ['excellent'], ['bad'], \n", + " ['boring'], ['not', 'good']\n", + " ] \n", + " for ex in examples:\n", + " ex = ['This', 'was'] + ex\n", + " prediction = model.predict([ex])[0]\n", + " print(\"{0:<30} {1:}\".format(\" \".join(ex), prediction))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This was great negative\n", + "This was excellent negative\n", + "This was bad negative\n", + "This was boring positive\n", + "This was not good negative\n" + ] + } + ], + "source": [ + "process_new_examples(tf_rnn)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Such experiments often reveal that the model is more sensitive to minor variation in the inputs than one would like!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With `embedding=None`, the `TfRNNClassifier` creates a random embedding space in which the values are drawn from a uniform distribution with bounds `[-1, 1)`. You can also pass in an embedding, as long as you make sure it has the right vocabulary:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "sst_glove_vocab = sorted(set(glove_lookup) & set(sst_train_vocab))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "glove_embedding = np.array([glove_lookup[w] for w in sst_glove_vocab])" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "# Add $UNK and its random representation:\n", + "\n", + "sst_glove_vocab.append(\"$UNK\")\n", + "\n", + "glove_embedding = np.vstack(\n", + " (glove_embedding, utils.randvec(glove_embedding.shape[1])))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "tf_rnn_glove = TfRNNClassifier(\n", + " sst_glove_vocab,\n", + " embedding=glove_embedding,\n", + " hidden_dim=50,\n", + " max_length=52,\n", + " hidden_activation=tf.nn.tanh,\n", + " cell_class=tf.nn.rnn_cell.LSTMCell,\n", + " train_embedding=True,\n", + " max_iter=500,\n", + " eta=0.05) " + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Iteration 500: loss: 3.3262188732624054" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf_rnn_glove.fit(X_rnn_train, y_rnn_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "tf_rnn_imdb_dev_predictions = tf_rnn_glove.predict(X_rnn_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " negative 0.76 0.70 0.73 428\n", + " positive 0.73 0.79 0.76 444\n", + "\n", + "avg / total 0.75 0.75 0.75 872\n", + "\n" + ] + } + ], + "source": [ + "print(classification_report(y_rnn_dev, tf_rnn_imdb_dev_predictions))" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This was great positive\n", + "This was excellent positive\n", + "This was bad negative\n", + "This was boring negative\n", + "This was not good positive\n" + ] + } + ], + "source": [ + "process_new_examples(tf_rnn_glove)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tree-structured neural networks\n", + "\n", + "Tree-structured neural networks (TreeNNs) are close relatives of RNN classifiers. (If you tilt your head, you can see the above sequence model as a kind of tree.) The TreeNNs we explore here are the simplest possible and actually have many fewer parameters than RNNs. Here's a summary:\n", + "\n", + "\n", + "\n", + "The crucial property of these networks is the way they employ recursion: the representation of a parent node $p$ has the same dimensionality as the word representations, allowing seamless repeated application of the central combination function:\n", + "\n", + "$$p = \\tanh([x_{L};x_{R}]W_{wh} + b)$$\n", + "\n", + "Here, $[x_{L};x_{R}]$ is the concatenation of the left and right child representations, and $p$ is the resulting parent node, which can then be a child node in a higher subtree.\n", + "\n", + "When we reach the root node $h_{r}$ of the tree, we apply a softmax classifier using that top node's representation:\n", + "\n", + "$$y = \\textbf{softmax}(h_{r}W_{hy} + b)$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### TreeNN dataset preparation\n", + "\n", + "This is the only model under consideration here that makes use of the tree structures in the SST:" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "def get_tree_dataset(reader):\n", + " data = [(tree, label) for tree, label in reader(class_func=sst.binary_class_func)]\n", + " X, y = zip(*data)\n", + " return list(X), list(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "X_tree_train, y_tree_train = get_tree_dataset(sst.train_reader)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAB2AAAALRCAIAAAACyCQVAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAHXRFWHRTb2Z0d2FyZQBHUEwgR2hvc3RzY3JpcHQgOS4xNnO9PXQAACAASURBVHic7N1PbCNpmh/oqNrEeCzN9m60LdkG3CuCDdiABBvGsnRcZB5ILFC1gC9FXXv6QgGGj9UWb9NzI921N2MAxqWrb4tkHbcKBsSD8raGKhaGdyXsAJ4AhRljDNGYgL0rYTA2nHv4slksScnUH4oRZDwPEgky+O9liBlM/vTy/T56+/ZtBAAAAABA9XxcdAEAAAAAABRDQAwAAAAAUFECYgAAAACAihIQAwAAAABUlIAYAAAAAKCiXhRdAACVk2VZlmVRFDUajSiK4jguuiIAAACoqI/evn1bdA0AVEiSJKPRqN1u53k+Go2yLPvuu++KLgoAAAAqSkAMwFK1Wq3j4+NwOs/zTz755E/+5E+KLQkAAAAqywxiAJYny7J6vT49G8fx69evC6wHAAAAKk5ADMDyhHS43++HGcTRb8cQAwAAAIUwYgKAZRuNRmH6cBzHnU5HRgwAAABFERADUJg8zw8ODl6/fh3HcdG1AAAAQBUZMQHA8gyHw+lwiSiK4jhuNBppmhZYEgAAAFSZgBiA5UnTdDgczm7JssyICQAAACjKi6ILAKBa8jw/PDwMMyVGo1Gn0zFfAgAAAIpiBjEAy5bneRgr0Ww2i64FAAAAKk1ADAAAAABQUWYQAwAAAABUlIAYAAAAAKCiLFIHwGLk19fpeHxz49VVenFxe2M2mbz4+OP/8l//a7yxUd/evnGF+tZWfWvrPhsBAACApzCDGKCisskkm0xubEzH4/z6+vbG2zcfnZ094kGbe3vT0yEg/sFDX1099A4btVq8uXl74x0Pvbt7Y0u8udnY2XnoIwIAAMA6ERADrIbR+fmNLXc250Z35bnZZJJdXj70Eevb27c7du/MXhs7O7dT2tuB7COkFxe3U+PbuyK661nnV1d3RtvzxZubt59jfWvr9hO8s6O5UavFGxsPfVAAAAAoioAYYPHubM69e+Pl5Z0du4/opZ1tzp26nXXGGxt3bKxML60fDQAAAMwSEANoU+VJqtncDQAAwHoQEAMr5qErod3YaNAt5Vf4eOjA+oEAAABVICAGnpeoCwrn1yoAAAC8j4AY+AFflgfuw2AWAACA9SAghhVmuS1gbTigAQAAFEJADMuj4Q5gmXwlAgAA4IMExHA3IzsBuMFQdQAAYP0IiFkTPrQDsHL8MhIAACicgJhi+NovADwH44wAAIAHERDzYRYOAoDK8t8AAABYbwLiNaR1CAAoM18kAgCA8hAQF8zwQQCAJ7IUAQAAPJqA+GF8/AAAWHt+hQ8AQHWseUDsC4wAAJSBIWAAAJRTiQJiS6AAAMA9+c8zAAALUWRAPDo/b/3qV/e/viYIAAB4Psv8+t3RZ5/1Pv/8gQUCALB4RQbE2WSSvHkTGaMGAADraM4CHs3dXdPVAADKoEQjJgAAAAAAWKaPiy4AAAAAAIBiCIgBAAAAACrqxfIfMk3Tfr+f53kURe12u9PpLL8GAACgQN1uN4qiXq9XdCEAAFW37A7iPM+73e5gMDg+Pj4+Ps7zfDgcLrkGAACgQEmSNBqNNE2LLgQAgKUHxFmWHR0dxXEczh4dHQmIAQCgOvI8T9O03W4XXQgAAFG0/BETjUZj9myapvV6fck1AAAARen3+6bMAQCUR5GL1OV5niTJ0dFRgTUAAABLk2VZdKtrBACAAhUWEOd5fnBw0Ol0puMmAACA9dbtdjWIAACUyrJHTARhqbper6d3AAAAKiJN0zzP+/1+OJtl2XA4NIwYAKBYBQTEWZb1+/1er6d3GAAAqqNer8+2D49GI+uRAAAU7qO3b98u8/HSNO12u69fv56mw/1+37fMAACgalqt1vHxcdFVAABU3bID4larlef5jd5h/y8EAIDqGA6HSZKkadrpdHq9XtHlAABU2rIDYgAAAAAASuLjogsAAAAAAKAYAmIAAAAAgIoSEAMAAAAAVJSAGAAAAACgol4U9cDpxcX/+i//5X/7u7/ba7fjjY2iygAAAIL8+jodj+dcIZtMsslk3hUuL/Pr6zlXSMfj/OoqiqLa3/yb4//wH6Ioqm9v17e25tykvrUVb27OuUJjZ2f+FZq7u3MuBQCouI/evn27/EdN3rzpDoe/8+LFX/7n//w3fu/3Xv+Tf9LY2Vl+GQAAUB5LyGdHZ2ePq22O5t7enEvjjY369vbt7X/1X/7L77y4o1tl/h7Ir67mX+ER4s3NRq025woSagBgvS07IM6vr7vDYXJy0t7fH/z+72eTyeFXX6Xjca/dPvr002VWAgAAsyqVz07Vt7bmN/A2arUV+sLfB39G6Xj8wR7nOZdKqAGA9bPUgDi9uLgdB9+IjFfof58AACyQfPZOq5XPIqG+k4QaAMpseQFxGCsRb27eOVBi/qUAADy3VQy2Ivks/JB/yHfyDxkA5lhGQHzPHuE7+4sBACpiFWMdjYfADb4KcCcJNQBl9uwB8YNiX+MmAICiyGfvJJ8FVouE+k4SagDmeN6A+HGDI4ybAIAKSi8u8qureVeQzwJQehLqx10BgAI9V0D8xF5g4yYAYMk+mM+Ozs8/cA8f/Dx8efmIwuaob29/8OOofBaASllCQp2Ox/P/z/AIjVrtA2/Zc38jG29sfPBXthJqgPd5loB4IfGucRMAVMpa5rPzP6pFH4pf481N3yUCgJUz/z8t+dVVenEx/wofHvokoQZYnMUHxIsdEGHcBADLIZ+9TT4LAKwiCfVtEmpgjkUGxM/U82vcBEAVPPH/8ZF8FgCAcqhgQh35ny2ssoUFxM8a4xo3AfDc/C/2Nv+LBQBgFem9uM3/7WGOxQTEyxkEYdwEsMYqmM/6HhwAAKwl09tuk1BTZk8NiJfc22vcBPBM5LO3yWcBAIBVJKG+TULNHE8KiAuJa42bgPWTX19/+N13bvyaXV7m19dzriCfBQAAYDnWMqGub2194CPwzs78K8yPsCnQ4wPiYgc+GDcBC7SEfHZ0dva42uZo7u3NuTTe2Khvb8+5wgfjV/ksAAAAq+iDCXU6Hn+wy2rOpfnV1fwrPEK8ufnBJioJ9TN5TEBckh5e4yZYG/LZOzVqNV8RAAAAgJXzwRxDQl0qDw6ISxXLliSqZm28L6iVzwIAAACsCgn1rA8G0w8OiEfn54e/+U2pBjskb970v/32uz/4A2EWTzQ6P2/96lf3vLJ8FgAAAIDbSpVQv/31r+df4UmL1MGame0gLm3bPwAAAADM8aCMS0AMAAAAAFBRHxddAAAAAAAAxRAQAwAAAABU1It7Xi9N036/n+d5FEXtdrvT6TxnVfdVzqpYOWmaJkmSZVkURY1Go9frFV0RAAAAADzSg1LTe80gzvP84ODg9evXcRxHUdTv9+v1ervdXlTFj1POqlhF3W736OgovJBGo9FoNJIRAwAAALCKHpqa3isgTtM0z/NmszndEh5jIRU/WjmrYg20Wq3j4+OiqwAAAACAB3toanqvERONRuPGY9Tr9UeXuCjlrIpVlyTJjZcWAAAAAKyKh6am951BPJXneZIkZfsCfjmrYrUcHh6madpoNAaDQdG1AAAAAMBT3Sc1/fih93hwcNDpdMIAi5IoZ1WsnMFgECZLJElSdC0AAAAA8CT3TE3vNYN4eo/dbrfT6ZTqC/jlrIqVZpg1AAAAACvt/qnpfUdMZFnW7/d7vV6punTLWRWrzjBrAAAAAFbXg1LTe42YSNP08PBw9h77/f6TalyEclbFyknTNMuy6dk8z2fPAgAAAMAKeWhqeq8RE61WK8/zG3lzmNZaoHJWxcrJ87zf709fS1mWDQYDPekAAAAArKKHpqYPmEEMayzP8zRN4zg2zBoAAACA6hAQAwAAAABU1L1mEAMAAAAAsH4ExAAAAAAAFSUghu+d/PEfZ5NJ0VUAAAAAwJK8eOgNkjdv6ltbzd3d56jm0cpZFaU1Oj+f/p2Ox+Hv/OpqeoV4c7NRq8UbG/Xt7Xhjo1GrxZubjZ2dguoFAAAAgAe7T2r64IB4eHraqNXKFsWWsyoKdzsIHp2dzV6hvr1d39qqb201arX61tb2j370e3/tr6XjcX59nV1e5tfXycnJ7eA4iqLwd3jJeeEBAAAAUEL3SU0fHBCXVoj/qKD8+jr0/6YXF9Fve4FvvB5CC3B9a+vos89CIhz+3HmHt//NpBcX+dVVNplkk0k4MTw9zS4v+9988/2t9vai3wbHjZ2daQ/yIp8qAAAAACzU+gTErL37B8HNvb3m3t4CU9r3DZcIkXH4E0oanZ3dWdJscDwnmwYAAACAZRIQUzqzqeu0bze7vJy9TmjXXXgQ/FDvi3pvZ9khO/7BbX873WIaGQuOAQAAAFgyATGFuT234X1BcHt/P1qpgb/xxkaoM1Q+68ZY5GwyGZ2c/OC21scDAAAAYFnWJCCub22F7/hTQiEInl35LXTXTq8wjURXLgh+hPc9u9t7yfp4AAAAADy3NQmI483NSEBctGlX7AeD4M6rV3pjbwj7wfp4AAAAACzTmgTELNONIQlRFL1vum6jVpuO2RUEP4718QAAAAB4PusTEBsxsVi3l1mL3h8EH332mWXWlsz6eAAAAAA83RoFxD9c3Ix7up0nhkm4s9eZNqI2ajWNqCVnfTwAAAAA7m99AmLmuz2R4H1BcHNvr7m3Z5Tt+rE+HgAAAAA3CIjXzWwQPF3c7EZ7dVjTLATBwj6sjwcAAABQWQLiVXW76/N9QXAYNSAI5qGsjwcAAACw9tYkIG7u7s52Na6T20FwmA4xvcK0Z7O9v29uLEtgfTwAAACAtbEmAfEauLGAWPTbMcHTK4RkLd7Y6Lx6JQimhKyPBwAAALByBMTLdjsIfl+LZaNW02LJerA+HgAAAEA5rVVAPDo/L0lydPu79iEFm71O6JGsb20dffaZIJhqsj4eAAAAQLHWKiAuUPLmTTaZzAmCm3t7zb09MRbcx0LWx6tvbXVevlxCtQAAAACr66O3b98+6AbpxUW8sVG2RtfQsVtg8Nr68stwQj8jLN/tnv2w/fiLL4osCwAAAKBQ98lyHxwQAwAAAACwHj4uugAAAAAAAIohIAYAAAAAqCgBMQAAAABARb2YnkrTNM/zGxfX6/UoipIkiaKo1+stoaDRaDT76KGAheh2u1mWvX79elF3mKZpv98PO63dbnc6nUXdM/B03W43WtaBCwAAAGBFfb9I3eHhYRzHURSlaVqv18Pp6LfxSqvVOj4+fu5qsixLkmQ4HLbb7bAlTdNer9doNBZy/wt8FnmeHxwcvH79Ouyofr9fr9enZQPFSpIkjuMkSZZw4AIAAABYXd93EMdxHLLgbrfbbDabzWb02xa8panX671eL4TCYUvIYUsY8WRZdnR0NI3Rj46ODg4OBMRQBnmep2k6GAzCtx8AAAAAeJ/vA+I7JyTMbkySZDQa5XkeYtxpNhpF0XA4HA6HeZ7HcVyv12eT0yeK4/hG+3CapkmSZFkWx3EItWcfK8uyfr+fZVn02wkVzWbzdgPywcFBqPbREyduV7XAaRjAU/T7fSNfAAAAAO7j+4D4znxzujFN03a7HeLUNE273e5gMAgXDYfD0Wg0TVrTND08PFzUqN/RaBTS3umdd7vd6WCHNE1n5zzkeR4eevbs7XQ4TA1e4OSKPM+TJDHqFMogHDEW9a8bAAAAYL19fM/rNRqNaUdeo9GYDW2Hw+E0LA6XNpvN4XD46JrSNG21Wq1W66c//els9BxFUZIk0/w3PFa73Z4+VpIks83LcRwfHR3dCL5DpjwYDBaYDh8cHHQ6nUU1TQNP0e12j46Oiq4CAAAAYDW8+PBVPmQ0GrVarRsbnzKNt9FohKHDw+EwTdPZi8JYiRsP1O/3w+k0TW8EQ7dT4CRJGo3GosLcPM+73e4Cm5GBp0jTNM/z6TEhy7LZRS8BAAAAuGEBAXGz2VzUQIkbQqwTEtiwJUyHmJWm6TTtrdfrWZbNnwU8GAxCh/JsJ/LjhHnHN4YgAwUKM9CnZ0ejkeHgAAAAAHPcd8TEHLM9vEGWZbMzKJ5457P31mw2u93u9NLQKjhtD+x0OrOXRr9d0e7GfTYajV6vF9ape3RhYdTybDp8YycAyxfHcXPG7VUuAQAAAKogy7Kf/vSntwc/3PbR27dvp2eSJBkOh1mWhUkOYXJCmLGbpmmn0wmdvK1WK6xZNx093O/30zQNnXrTJeAe0VebZdnh4WGapiHDDclOlmUHBwdHR0chCE6SZNoVmKbpjfEOs5feqOTg4GA0Gv3FX/xFuM+wd9rt9uMWl2u1Wnme33iOYTIGULjhcJgkyeyBCwAAAKA60jT95JNP4jgOcegcPwiIn2g0GkVRtMAJv++T53mYTdxsNoutBAAAAACghMJs3g+O31xkQAwAAAAAwApZwAxiAAAAAABWkYAYAAAAAKCiXhRdAMBiZJNJNpmE06Pz8yiKmru74Wx9a6u+tVVYZQAAAABlZQbxAnS//rq+tdV5+bLoQmA9pRcX+dVV9MMIOB2PpyfCpfcRb242arVwenpiGh/Hm5uNnZ1FlQ0AAABQfjqIF2AaVAEPEvp8oxDyXl9HUZRfXX3fBXx2duetboS84fT7Qt454XI6Hs8Jl5t7e+FEfWsr3tyMoije2Jg+7rQ3GQAAAGClCYiBBbs96iGKouzyMkTA2WSSXV7eecP69va7kHdjo769HdLYJ46JuGdH8Jyas8kkOz+/Z81ho9EWAAAAwKowYmIBWl9+2ajVep9/XnQh8LwePephnbpxF9L1HE4YbQEAAAAUTgcxsIxRD2tjGmfPz7WNtgAAAABWgoAY1lapRj1UjdEWAAAAwEowYmIBjJhgyYx6qCCjLQAAAIDnoIMYSsSoB97HaAsAAADgOQiI4dkZ9cDSGG0BAAAAPIgREwtgxERlGfXA2jPaAgAAANabDmK4g1EPEBhtAQAAAOtNQEyFGPUAz8RoCwAAAFhRRkwsgBEThTPqAdaM0RYAAACwHDqIKTWjHqCajLYAAACA5RAQUwCjHoCFMNoCAAAAnsiIiQUwYmLKqAdgpRltAQAAQNXoIOZejHoAqsBoCwAAAKpGQFxpRj0APILRFgAAAKwNIyYWYIVGTLS+/DIy6gGgZJ74LY3jL75YRpUAAACsIwFxtYQsO5w26gFgtbxvtIWAGAAAgEcTEAMAAAAAVNTHRRcAAAAAAEAxBMQAAAAAABUlIH6wNE3zPC+6CgAAAACApxIQP1i3203TtOgqHiZN04ODg1ar1Wq1kiQpuhwAFsCxHQAAgKcTED9Yu92u1+tFV/EAeZ53u93BYHB8fHx8fJzn+XA4LLooAJ7EsR0AAICF+Ojt27dF18DzCjMxms3mdMvBwcHr168LLAmAJ3JsBwAAYCFeFF3AKpkOl+j1eo1G48alaZp2u90oiuI4Di3GvV5v+UXedqPUNE1XqwMagNsc2wEAAFgIAfEDhMC32+3euUjd4eHh8fFxHMfRTFhcNnmeJ0lSkuQagIVwbAcAAODRzCBemHq9PhqNwulGozEYDIqt57Y8zw8ODjqdTkixAVgDju0AAAA8hYB4YQaDQZZl3W734ODg8PDwzi7jAoXljO4cjgHAinJsBwAA4ImMmFiMEAcfHR1Nz7Zare+++67Qor6XZVm/3+/1evrLANaGYzsAAABPp4N4MdI0TZJkerZUn9XTND08PJxNEPr9frElAfBEju0AAAAsxEdv374tuobVMBwOQwScZVkcx+ED+WAwCKvGj0ajJEmm27MsazabnU6n2JqDVquV5/mNzPr4+LioegB4Osd2AAAAFkJAvEh5nqdpGkVRs9ksuhYAAAAAgA8QEAMAAAAAVJQZxAAAAAAAFSUgBgAAAACoqBdFFwAAPFh+fZ2Ox6Pz8/zqKoqieHOzubvbqNXijY2iSwMAAGCVmEFcIa0vv4yi6PiLL4ouBIAHSy8ussvL9OIiHY/T8TjkwlEU/YO/+3ejKPq//uzPwtl4c7NRqzVqtcbOTn17u7GzU1jFAAAArAIdxABQOqFBOB2Ps8kkm0xGZ2dhe8h/O69e1be2Qgo8vUnIjrPJJB2Pk5OTaYLc3Nurb229u74WYwAAAH5IB3GFdL/+Oh2PdRADlNBsg3A2mWSXl2H7u1T34RMkZmdQhKw5bK9vb0/DZS3GAAAA6CAGgGWb3yDc3t+/3SD8UPHGRnN3t7m7O92ixRgAAIDbBMQA8OxCLDttE55tEK5vbR199llzdzektM9XQ2NnZzZxfpdNn59nl5chMg7bQ1tx+DuU93wlAQAAUDgBcbVMv2IMwLManZ+n4/G7OQ93NQjXt7Zm23uX73YNo/PzkBrfbjEOncWNWq3YmgEAAFg4AXG1TD/tA7BAZWgQfrob4e9si/Ho7Gx2irEWYwAAgLUhIAaAB7uzQTiKoubeXnt/fz2abe9sMZ4+6/4330y3azEGAABYXQJiAPiA2V7a0Cwctocu2qPPPqtIL+2NVe9m+6aHp6c3+qbr29sr0TcNAABQcR+9ffu26BpYkv6333aHw7e//nXRhQCU3ew03nQ8No33nuZMXn6XGhc9eRkAAIAbdBBXSKNWK7oEgDKa3yDcefWqIg3CT6fFGAAAYOXoIK6Q0fl561e/0kEMkF5cpOPxnQ3CIa8MHa/xxkaxda6Z0Fkc9nw2mdzZYtyo1Ro7O8XWCQAAUCk6iCsnv74WeQCV8m7iwfl5fnUVAsqwvb69Xd/aCg3CoVm40DLXX7yxcaPFODQXh7z+Rotxo1aLNzebu7uSegAAgGelg7hCQgfx8S9+Yf4jsN40CK8oLcYAAADLp4MYgNX2vgbhkCpqEF4h81uMk5OTadavxRgAAGBRBMQArJjZ0HC2QTiEhs29PaHh2mjs7DR2dtr7++Hs7V8G9L/5JpppMfbLAAAAgIcSEANQavPHDnRevTJ2oDrubDGejhOZbTE2TgQAAOCeBMQV0qjVii4B4MNmG4SzyeTGwmVHn32mQZip0GI8PXujxTg5OQnbw4KEWowBAABus0hdtbS+/LLXbvtgDJTW4W9+E0I965KxEHeuWNh59Wrws58VXRoAAEApCIgBKJHR+XkURWE4QNG1sIbCoJIoimbnVAAAAFSZgBgAAAAAoKI+LroAAAAAAACKISCurjRN8zwvugoAAAAAoDAC4hJJkuTw8DBN06ff1X2S3263u5DHutMCnwtQBWmaHhwctFqtVquVJEnR5bCe0jQ9PDwML7Nut1t0OQAAAKUgIC6RTqcTx/FCunoPDw8/eJ12u12v15/+WHda4HMB1l6e591udzAYHB8fHx8f53k+HA6LLoo1NBwOe71eeJk1m00ZMQAAQCQgXlf3SWY7nc7zBcQA95dl2dHRURzH4ezR0ZGAmOfQ6/WmL7Nms+lrLgAAAFEUvSi6gKobDofD4TDP8ziO2+32nEvr9fpsgBJFUZqmofspXBpFUa/Xy7IszHZotVrhanEcv379evZup8Mler1eo9GYvejg4KBer9fr9ZDO1Ov12Y/T8+uZ/1wA3ufGgShNU7++4rklSXLjhQcAAFBNAuIiDYfD0Wg0TW+TJBmNRs1m885Lw+TE2aj38PDw+Pg4RLTTsLherx8fH7darePj4/c9bq/Xi6Ko2+3ebjR+/fr1j3/84/AN3OndDgaDD9Yz/7kA3FOe50mShMMUPIfwa9RGoxHe3QAAACrOiIkiDYfD2U+nYW7v+y5tNBrNZnP2a9f1en00Gk0vXdQH3Uaj0el0pqezLLtPPfOfC8B95Hl+cHDgAMKzCtOuoyiyHCIAAECkg7hYtxt4Z7/uOhqNpmMipmZHNwwGgyRJut1ulmVxHE9T3Wcyv575zwXgg8JSdbdH38DCxXE8GAzCbyOKrgUAAKBgAuIi3W6Rm41Zm83mjdnBt695dHQ0Pdtqtb777rtnKPNe9cx/LgDzZVnW7/dnh57DczPqGgAAIDJiolj1en32+61pms5OkGi32/1+f/b6WZZNBz6kaTp72xuRSr1en81nF5LVzq9n/nMBmCPMNJ9Nh28cbeDp0jSdvmdFUZTn+exZAACAyvro7du3RddQaWGluNl4dzQaHR0dhdEN/X4/TdPQ4hRC3mmAMhqNkiSJ4ziczbKs2WxOvyobFpcLX9MON5wOCB4OhyHJDYMpws0Hg0HIlA8ODtI07XQ604XskiSZnp1TzwefC8D7tFqtG0ePKIrmrLQJj5Dneb/fn77SsiwbDAY61gEAAATExQt9uPV6/X3fdQ0r0TUajTvHOKRpGkVRs9l830V33vAp5tTzwecCAAUKb45xHBt1DQAAEAiIAQAAAAAqygxiAAAAAICKEhADAAAAAFSUgBgAAAAAoKJeFF0AAJWWTSbZZDI6P8+vrrLJ5P/7y7/8vd/93frWVry52dzdrW9t1be2iq6R9TE6Px+env6bP/3Tf/iTn7T395u7u0VXBAAAUDCL1AGwPDfi4NHZ2fSi5t5efWvr+q/+auN3fufOi0TGPFp+fZ2cnCRv3mSXl/Xt7b//t//2H//7fx9Od16+7Lx6FW9sFF0jAABAMQTEADyXOXFwo1arb23Vt7cbOzvh9J03T8fj9OIiu7wMp6cXiYy5p9AynJycRFHU3t8Pf8JFw9PT8CeKos6rVxqKAQCAahIQA7AY+fV1Oh6n4/H3J66uwkWzcXD4+3EPEcLi25FxvLnZqNUatVq8sTE9sZAnxYrKJpPh6em0Zbi9v995+fJ9v4dI3rwZnp5OG4rb+/t+5QAAAFSHgBiAx1hCHHwfImNumO0LvtEy/Ew3BAAAWGkCYgA+rCRx8H2IjKtpUY3A9289BgAAWA8CYgDuMDo/vzMOrm9v17e23oXCW1srMbN1dH4epiGn43E2mWSXl2H7jch4JZ4Ltz3TKOE5w4sBAADWiYAYgPdGqKsYB99H1Z7vWrrdMtx59WrhXeH59XVycqKhGAAAWGMCYoDKEY/eZp+skJALj87OooW2DM8321DcSekdzQAAIABJREFU3NsLSfFzPygAAMASCIgB1twHByyIPu80u99WfcjGeggtw8nJSX511ajVQjS85EHS+fV1iInT8Tje3Oy8eqWhGAAAWHUCYoC1Ml2ibf68XUu0PdScZfqmkXFJlulbP9OW4Xhzs72/33n1qvCdnF5cJCcnw9PT/OpKQzEAALDSBMQAK2waB2eXl6HXNWwXBy/BnMj4XX/x9rbI+ClmQ9iiWobnu9FQXJLwGgAA4EEExAArQxxcciLjhVjF1LX8WTYAAMD7CIgBSup9cXAURc29vTAAVxxcctOkOIwzDuuqBSLj29ZgbkMJp2EAAADMJyAGKIV3AeL5eX51FSLF6UUhDo43N5u7uyEXLq5Mnmr2B307Mm7UahX8Qa/fym9lWE8PAADgngTEAAWYkxKKg6umyi+G0fl5iIajKFrRluH5pg3FURSFmLi5u1t0UQAAAD8gIAZ4dlVOAHmEtX/B5NfXyclJ8uZNdnlZ394OufCKPpf7CA3Fw9PT8Hw7L192Xr3SUAwAAJSEgBhgwebPEJiOnQ2nC6yTFRKGUM8ZSL0qkfFsy3B7fz/8Kbqo5RmenoY/kYZiAACgNATEAE81Oj9Px+PpcmT51VXYbhUyns/71jCMNzenSxc2arWS5I9VaxmeT0MxAABQKgJigKf65A//MB2PxcEU63Zk3KjVvvuDPyi6riiKomwy+ek/+2cVbBmeb9pQ/Cf//J9XNi4HAAAKJyAGeKpsMhHuUEKlemXm19eaZO9kzwAAAMUSEAMAAAAAVNTHRRcAAAAAAEAxBMQAAAAAABX1ougCAFZVmqb9fj/P8yiK2u12p9MpuiKIopK9MktVTNmkaZokSZZlURQ1Go1er1d0RQAAQBWZQQzwGHmeHxwcvH79Oo7jKIr6/X69Xm+320XXRdWV6pVZqmJKqNvtHh0dhZ0zGo1Go5GMGAAAWD4BMcBjpGma53mz2ZxuCUFYgSVBVLJXZqmKKb9Wq3V8fFx0FQAAQOUYMQHwGI1GY/Zsmqb1er2oYmCqVK/MUhVTckmS3NhdAAAAyyEgBniqPM+TJPHdcMqmVK/MUhVTKoeHh2maNhqNwWBQdC0AAEAVfVx0AQCrLUxZ7XQ6YZAolESpXpmlKqZsBoNBmCyRJEnRtQAAAFVkBjHA4+V53u12O52O74ZTKqV6ZZaqmDIzoBkAACiEERMAj5RlWb/f7/V6miIplVK9MktVTMkZ0AwAABTCiAmAx0jT9PDwcDb26vf7xZYEUclemaUqpmzSNM2ybHo2z/PZswAAAEtjxATAY7RarTzPbzRFhkGiUKBSvTJLVUzZ5Hne7/en+yfLssFgoM8aAABYPgExAEAx8jxP0zSOYwOaAQCAogiIAQAAAAAqygxiAAAAAICKEhADAAAAAFSUgBjgSbLJ5OSP/7joKoBVlU0m2WRSdBUAAEB1vSi6AICym8Y3o/PzKIryq6t3Z8/OwhX+x1rt/xyPoyiqb2/Xt7aiKGrUalEU1be2wtnm7m4hlVNx3a+/bu7uluHll7x5E0VR5+XLogspkfz6enh6Ojo7G56eRlHU3t9v7u219/fjjY2iSwMAAKrFInUAH46Ap+LNzRD+zkbA//4//se//d/9dx+8rfiYJfvo5z8/+uyz3uefF11I1PryyyiKjr/4ouhCSiF582aaCzdqtf/lH/2jKIr+93/9r9PxOPptUixMBwAAlkYHMVAJj4uAG7VavLExPTu/s+/OnPfOx03HY/ExSxBvbhZdwjvxxkZ+fV10FQUbnp6Ozs+Hp6f51VWjVuu12+39/fDPPIqiX/7jf5xNJsPT0+Hp6eFXX3WHw/b+fnN3t72/X2zZAADA2tNBDKyJ/Po69N+l43GIor4/e3U1e80bXcD3j4AX7v6xtfiYR2h9+WWjVitDB3H366/T8biaHcQhFA65cH17u72/397fb+zszLlJenERbpJdXsabm+Em/rEDAADPRAcxsDIeFwE/qAt4yebnvLqPYXXdCHk7r159MBeeauzsNHZ2ep9/Hu4kOTlJTk7uGS4DAAA8lIAYKJH7R8BRFDX39qIoqm9tlTkCfgrxMevk9j/htZRNJsmbN7PNv83PP3/0mIhpUhzGUyQnJ/1vvglJcefly+l4CgAAgKcwYgJYthBoPigCDqNUQ6A5zTeZw/AKojItDdf9+uv+N9+8/fWviy7kuUzHB4cDWufVq2caHzxNiqMoatRqoafYIREAAHgKATGweCGUnGaUITHJJpPs8vLGNUXAhRAfV4SA+Lnl19dhBEQ4yrX395t7e+39/ef+HkN43NHZ2fD0NIqiRq0WRliszfcnAACAZRIQA48hAl5j4uO1ISB+JuXJZ4tKqAEAgHUiIAbudv8IuFGrxZub8cZGfXs7iqLGzk68uSkCXlfi4xUiIF640k54WNqMCwAAYP0IiKG60ouL/Ooqv7pKLy6ih0fA8eZmY2dn6VVTauLjUul+/XU6HpchIB6enh780R+tbkAccuHh6Wl+dVXyNeLuWCVPUgwAAMwlIIZ1diMCzi4v8+vr/OoqZMGzRMAsgfh4ycoTEI/Oz1u/+tXKBcTpxUUY4DDNhdv7+6tyVAzFT5PiMARjVYoHAACWSUAMq00EzNoQHy+cgPhxbkSrIRde3VdX6H2ebX+WFAMAALMExFB294+ApWasMfHxI3S//jo5OfmLf/Evii7kXUD83S9/WeZccnaM71oOZ5gdlFGqAcoAAECxBMRQvDuTLxEw3J/4+E7lWRouBMTHv/hFCffzjeXd2vv7zb29zsuXRdf1jJI3b0ZnZ8PT06hkS+0BAACFeFF0AVAJj0ivmnt7zb299U6vYFHm/0u58x9gOh5XPD6uuPz6enh6Os1J2/v7YUpvvLFRdGnPrvPyZefly+ke6A6H3eEwJOMV2QMAAMAsATEsxv0j4HhzM2RPjVqtUatJoOC5iY+ZdaN/ttduV7N/Nt7YCEnxtIf68KuvDr/6qgo91AAAwCwjJuC+HhcBR1EUb2xMz+rMgtWy0sMr+t9+2x0OyzBiIr24+OSXvyx2xIQJvB+09lOYAQCAOwmI4Q759XX/22+jKAojKdPxOL+6mr2CCBiIHhsfxxsbR59+uoTywuTfMgTEURR99POfFxUQH/7mNyEXrm9vh1y4zGvllUF6cRGS4uzyMiTFg5/9rOiiAACA5yIghrv9+J/+UxEw8BTvi4+jKDr+4oslFJBfX6fjcUkmYIzOz4s6fra+/DK0DMuFHyokxel4vJxXLAAAUAgBMQAAAABARX1cdAEAAAAAABRDQAwAAAAAUFECYgAAAACAihIQw/fSND04OGi1Wq1WK0mSossB1ko5jzBJkhweHqZp+qBb5Xl+e2OWZd1ut9vtPrqYO+/20cq5w1dImqaHh4dhBz7lxwoAAJScgBjeyfO82+0OBoPj4+Pj4+M8z4fDYdFFAWuitEeYTqcTx/FDk9nDw8PbG+v1eq/Xe2jW/MG7fZzS7vAVMhwOe71e2IHNZlNGDAAA60pADO9kWXZ0dBTHcTh7dHQkTQAWZc2OMItt9X2Ou12zHV6IXq833YHNZvMp0T8AAFBmL4ouAMqi0WjMnk3TtF6vF1UMsGYWfoTpdrtZluV5HsdxaN2dXnRwcFCv1+v1eohEw6XTpC+KouFwOBwOw23b7faDHjfLsjCSotVqhS1xHL9+/Xr2OkmSjEajPM9vP/T7yr7P3T6IQ/piJUlyY5cCAABr46O3b98WXQOUTvhu8o1cA2AhFnKECRlrON3v9+M47nQ600t//OMf93q9sCVN0yRJBoNBuGg4HI5Go+nZJEmSJOn1es1m8/6P3mq1jo+P77xozkN/sOw5d/sUDulPEYL7RqMx+3MEAADWiRETcFOe5wcHB2EuZ9G1AOtmUUeYMDh4NBplWdZoNLIsm7200WhMg9cblw6Hw9mkb+HHujkP/cGyn4ND+hOFOc5RFFnoDwAA1pURE/AD00Yz36UFFm5RR5g8zw8PD+M4DqFnaPC8/21vbFna4e4pZT/6ER3Sny6O48FgEHL2omsBAAAWT0AM38uyrN/v+xoy8BwWeIQ5PDw8Ojqahp6j0Wg0Gt3ztrcf/ZlWnLvtKWU/gkP6YhniDAAA68qICXgnTdPDw8PZKKHf7xdbErA2FnuEieN4tiU2LEZ3T/V6fXZWQJqmD7r59E5mY+V7RswfLPtxd3snh/QnStN0dgBInudLmAcCAAAUwiJ18E6r1ZpdPSl4juWSgApa7BEmLDQ3HdTQbDaTJGm3271eL4zcTdO00+n0er0oirrdbpIk07Nhy41iRqPR0dFRu92+ZwFpmna73ZD2hhh3MBjcfuhWq5WmabvdDlOP55Q9524ft4sc0p8oz/N+vz/dh1mWDQYDvdgAALCWBMQAsHryPE/T9EZP7v1lWZZlWb1ef/TcgFBAFEWNRuP+ueEHy37c3fJMnvgyAwAAVoKAGAAAAACgoswgBgAAAACoKAExAAAAAEBFvSi6ACidbDL5v//dv/uf/t7fizc2iq4FAIqUXlz8H//2327/6EfNvT1viwAAsJYExPC90fl5cnIyPD390V//6//Nxx+39/ePPv20vrVVdF3A+kjevMkmk97nnxddSFUkb94MT0+Pv/ii6EJWSXpxMTo7S8fj0fl5fnW19aMfTf7Tf4qiqFGrNff2Gjs7wmIAAFgnAmKIot8mCKOzs/r2dq/d/p//wT/43/7Vv0pOTpKTk86rV+39/ebubtE1Ausgm0xGZ2eRgHhZ3u1wPuRGKBxFUaNW67x6FeLgsBvT8Tg5OZleKiwGAID1ICCm0vLr6+Hpaf/bb7PLy0atNvj93++8fBku+kc/+cnRp5/2v/12eHqanJw09/Y6L1+29/eLLRhYA+l4XHQJFeJbIHPMD4VnY9/Gzk5jZyecziaT0fl5Oh4PT0/733wTbhX+NHd37XAAAFg5H719+7boGqAA+fV1/9tvQydUc2/v6NNP5/QIJ2/ehBC5vr199Omn0xAZ4KG6X3/d/+abt7/+ddGFVMXo/Lz1q1/Z4VN3hsKP7gWehsWj8/Ps8jKKovr2dnN3V1gMAAArREBM5WSTSYiGoyjqvHp1/ynDo/Pz/rffjs7O4s3NcENfqgUeSkC8ZALiaNGh8PsIiwEAYEUZMUGFTNegizc3jz77rPPy5YM+rzZ3d5u7u+nFRXJy0v/mm+TkxCp2ACshm0yqdqy+//iIRalvbXVevoxevoxmwuIwtjgSFgMAQInpIKYSZteg67x82Xn16omfjbPJJHnzJkyosIodcH/D09ODP/qjije0LlPoID7+xS+qcJReTqfwQ+XX16Ozs2ltURTFm5vvwuK9veloYwAAoCgCYtbZjTXoOq9eLXZ8cBhkPDw9zS4vrWIH3IeJB0u29gFxOUPh9xEWAwBACQmIWU8PWoPu6axiB9yTgHjJ1jIgXq1Q+H2ExQAAUBICYtbNo9egezqr2AEfFPLK7375S/nXcqxNQLweofD7TMPidDwenZ1FM2FxGFtcdIEAALDOLFLH+njiGnRPZxU74J5CwMcSrHS2uPyF5ooSb2y09/enY5pG5+dhjbvucBi2NPf2QlK80j9QAAAoJx3ErIOFr0H3dFaxA+60Ng2tK+Sjn/98hXb4encKP8I0LA6dxZGwGAAAFk1AzAp77jXons4qdsANAuLlK39ALBS+J2ExAAA8BwExK2nJa9A9nVXsgCC9uPjkl78seV65Zj76+c+PPvus9/nnRRfyA0LhJwpJ8ewODGGxHQgAAA8lIGbFFLgG3dNZxQ6IVqGhdc2UJyAWCj8TOxYAAJ7CInWsjMLXoHs6q9gBVE11FporUGNnp7GzE05Pd3j4mlEkLAYAgA8RELMCZteg67XbZViD7ikaOzuDn/3s6NNPwyp2ycmJVeygakJuxboSChdIWAwAAA9lxATlVf416J7OKnZQQeWZeFARPz06au/vP/cON+Wg5LLJ5N0Cd+fn2eVlFEWNWi38ae7u+ioPAABVJiCmpLLJ5JM//MNVWYPu6aar2DVqte/+4A+KLgd4RgLiJWt9+WWjVnu+HT48PT38zW+Ewivkdlhc394+/uILMTEAANUkIKa8ul9/vYqDhp9idH6eTSbr1ygNsMbSi4vh6alQeEVNw+LBz35WdC0AAFAMATEAAAAAQEV9XHQBAAAAAAAUQ0AMAAAAAFBRAmIAAAAAgIoSEFM6aZoeHBy0Wq1Wq5UkSdHlLEk1nzWsvTzPW63W4eHhIy7lQbIs63a73W53aY/ouL0G0jQ9PDwMP8RlvngAAKBULFJHueR5fnBw8Pr16ziOoyjq9/v1er3dbhdd1/Oq5rOGivjoo4+azebx8fEjLuWhWq3Wcnam4/Z66Ha7R0dH4Yc4Go1Go1Gv1yu6KAAAWDYdxJRLlmXTj2pRFB0dHQ2Hw2JLWoJqPmuoiEaj0Wg0HncppeW4vR56vd70h9hsNtM0LbYeAAAoxIuiC4AfuBGUpGlar9eLKmZpqvmsoSKm8dMjLuURkiQZjUZ5ntfr9dn4bzgcDofDPM/jOK7X67Px7iM4bq+fJEn8tgYAgGoSEFNeeZ4nSVK1L3tW81nDGms0Gs1m83GX8lBpmrbb7devX4fT3W53MBhEUTQcDkejUdge/Xby7PTsEzlur7rDw8M0TRuNRni1AABA1ZhBTEmF8Y69Xq9S7TzVfNYAi3JjBvH0bJgXPHvNJEniOH761GDH7fWQ53m32200Gp1Op+haAABg2XQQU0bhc1rVPm9X81kDLMFoNGq1Wjc2LiQddtxeD3EcDwaDg4MDATEAABUkIKZ0sizr9/uzgyOroJrPGmA5ms3mogZKTDlurx+DpAEAqKaPiy4AfiDMhZz9vN3v94staQmq+awBlqbdbt84rmZZlmXZo+/QcXsNpGk6+xrI8/wpLwkAAFhdZhBTLq1WKywxP7txdqDkWqrmswZYoDALOE3TTqcT1otrtVphzbqw8li/30/TNLSI5nkeRdFTmn8dt9dAnuf9fn/6c8yybDAY6AcHAKCCBMQAQFWMRqMoihqNhhyQIM/zNE3jODZIGgCAyhIQAwAAAABUlBnEAAAAAAAVJSAGAAAAAKioF0UXAHwvefPm//nzP/9bP/pR59WreGOj6HIAYJ2Nzs+zySSbTP74z//87/+dv9PY2alvbzd2doquCwAAlsoMYkqq+/XX6Xh8/MUXRReyDOnFxfD0NDk5ya+u/uFPfvJv/vRPoyjqvHrV3N1t7+8XXR0A71Wpd6tVl15cZJeXs3+H7fXt7b/3t/7Wv8qy/OoqbGnUavWtrfr2dnN3t761Vd/aKq5qAAB4djqIoTD59XXIhdPxON7cbO/vd169auzsZJNJ8uZNuKi+vd3e3++8fOnTKQDcX359nY7Ho/Pz/Ooqm0xGZ2dhe7y52ajV2vv78cZGo1Zr7u6+7yb9b77pf/NNuKi5txeS4kat1qjVfMsHAIB1ooOYklrvnqzR+XnIf6Moau7thQj40VcDoCjr/W61WqbzItLxOB2PZ9uBG7VavLn5iHbgd/d2cZFdXobTYXtImd81Gm9tzabMAACwcgTElNRafuSetgZnl5f3bw1+X6PxcmoGYI61fLdaCXPmRbzr832egcLpxUU6HoewOJtMZh93+ogGGQMAsFqMmIBlSN68GZ2dDU9PozBc+PPP7z9cON7Y6Lx82Xn5cjqqODk5CV+PtZYdAFXwiHkRz6Sxs3Mj/B2dn6fjcagwLCfw7ppP6FwGAIBl0kFMSa1HT9bs6nMLjHRvxs3WsgMoyHq8W5XQc8yLWI5Q9u0sO4qi5t5emF9skDEAAKWigxgW77mHQoSGYmvZAbAe5s+LCO+hqzK34fZU4tnnNTo7M8gYAICy0UFMSa1oT1Yhy8pZyw6gKCv6blWs+fMipj22axyYzvZHzw4yfhcWG2QMAMByCYgpqdX6yP241ecWy1p2AMu3Wu9WRVndeRHLERLz6SDjG1Mp6ltbdhEAAM9KQExJrcpH7hKOA36mwccA3LYq71bLNH9eRKNW0x77QbODjEN2HLbfaLI2yBgAgIUQEFNSJf/IvRIhbAnDa4A1U/J3qyUwL2I5Phi7T8P3YusEAGAVWaQOHmC1xjhYyw6AhZs/L+Los88MQ3gOjZ2dxs7O7C96Z38Q4TfW7645M8g4nC6oZAAAVoYOYkqqbD1Za7AQ3Bo8BYCyKdu71cKZF7EqpoOM342neM8gY1MpAAC4TUBMSZXkI3cZVp9brNVqggYouZK8Wy2KeRHrZDrIOLu8DI3GYbufJgAANwiIKanCP3Kv/QDflRijDFByhb9bPdH8eRGh59S8iLUx7QR/12isHxwAgCiKBMSUVlEfuSsYm659FA7wfFYrIDYvghvm/IYgDDL2GwIAgCqwSB1EUbUHL1jLDmAtzZ8X0d7fN2GAGz/9G6+Z/jff9L/55t019/ZCUjwdT1FEvQAAPAsdxJTU0nqyLN12gx0CcH+l6iA2L4KFe/dyurh43yDj8IryawYAgJUmIKakkjdvssmk9/nnz/oow9PTgz/6Iw2zt822VB999tlz/yAAVtRy3q3u46Of/zycMC+CZzUdYXxjkPHbX/+62MIAAHg0ATFVNzw9NXJ3jvTiIt7YEJ0DlFzy5o1GTgoRWtd93wgAYHUJiAEAAAAAKurjogsAAAAAAKAYAmIAAAAAgIoSEFO8PM+X/Ihpmh4cHLRarVarlSTJkh99VdhLADfked5qtQ4PD4su5B0HaoqVpunh4WF4BXa73aLLAQDgkcwgpngHBwevX79e2sPleR4eMY7jKIr6/X69Xm+320srYCXYSwB3+uijj5rN5vHxcdGFOFBTvG63e3R0FF6Bo9FoNBr1er2iiwIA4MEExBSv1Wot85N2mqZ5njebzemWJSfUK8FeArjTJ5980mw2y5CCOVBTNkv+Hx0AAIvyougCqLQsyw4PD9M0bbVaYUscxzc+3KZpmiRJlmVxHMdx3Ov1QqPKozUajRv3X6/Xn3KHa8leArjT+96D0jQNX7GP4zgcMJ87RHagplSSJLnxmgQAYFUIiClSvV4/Pj6e028SPm9Pvz8bhi1Ozz5dnudJkpShEazM7CWAqUajMdu0O3V4eHh8fDx9t1ryPFYHagoUftnfaDQGg0HRtQAA8BgWqaPUkiSZjYMbjUa73R4Ohwu58zC9sdPpLCpuXkv2EsCsXq93Z0Bcr9dHo1E4veSkzIGaYg0Gg/CbfislAgCsKAExpRbGSsxuabfbWZY9/Z7zPO92u71ez9ch57CXAO5pMBhkWdbtdg8ODg4PD/M8X87jOlBTBnEcDwaD6e9IAABYLUZMUGq3P2Cnafr0Dqksy/r9/tPHGa83ewngnsK71dHR0fRsq9X67rvvnvtxHagpFVOwAQBWlA5iilev12eD4NnTzWZzdoxjnuf9fr/dbj/l4dI0PTw8nP043e/3n3KHa8leAri/sJ7q9Oxy4loHaoqVpunsl7ryPF/Id7wAAFi+j96+fVt0DVRdWMwnfDc2pMOzoxuTJBmNRqEnJU3Tp3+LttVq5Xl+49P7+1bJqyx7CeD+RqNRkiTTsUhZljWbzU6n86wP6kBNscKv7acvwizLBoOBZnYAgFUkIKYU8jxP0zSKokajcfujxfTSO9cFAoAy8G5FBYWXfRzHpmADAKwuATEAAAAAQEWZQQwAAAAAUFECYgAAAACAihIQAwAAAABU1IuiC4AipRcX8cZGfWur6ELKKJtMRufno7Oz//cv//J/+Bt/o7m729zbizc2iq4LgB/Ir6/T8bhRqzlE89zCiy2KotH5eRRF2eVlfn2dTSb//cbGjzc3oyhq1GpRFNW3tupbW/HmZmNnp8hyAQC4H4vUUWmtL79s1Gq9zz8vupASySaT4enp8PQ0fAJs1Go/+fGP/82f/Vl2eRlFUXt/v7m3197fF0MAlMTo/Lz1q18d/+IXzd3domthHbwvBQ7/E5iqb2/Xt7bijY369vb1X/3Vxu/8zrtbnZ3duMPm3l4kOwYAKDEdxEAURVF6cTE6O5vmwu39/fBn2l6dXlwMT09HZ2eHX311+NVXISlu7u7qvwaAVRTy33Q8zq+v86urbDKZkwK39/ejKAq/hLhPu3p6cTG9z/Ao6XgsOwYAKCcBMVRaiH2Hp6fTBuHOq1d3Ngg3dnYaOzvR559PW4wPv/oqiv5/9u4mtpE0z+/8U4Vyj0ecaSPKI3kGMw0R0cCuQXnn4CgCC/iQOgQvysseMnjNqgsJJHysMuOwQPeR4arFYgFPAoxLZ94WCsG3FGwwDtJpDTOfyyxE2IArQKLHmLVoZGAGFtGYQUN7eLKiokmJ4kswXr8fFAqSmCQfhqgnIn78x/8RRr2+ECUDANJE4TBWWE6Bw7s79WFwRKvVVESrUmDj+Fir1VRcu/Xzrgh5yY4BAADyhhYTqLTKtpjwRiMVDQe3t1qtZjWbKufdqHGESorlZOKNRkIIo15X3Sc4eQOAlH3y1Ve0mKiyjVLgj/9PIgVO3HJ2LOhZAQAAsH9UEAMV4o1G/njsjUbh3Z3Khc0XL1S50Bb0w8Pe2ZkQIpzPVfcJ59075907/ehI1RRzqgYAQFI2SoHNkxPz5CSfKfAK1B0DAABkggpiVFoVKohVeqvqfMO7O5Xemo3GPgrNwvncv7mJMmj1XMbx8dYZNABgHZ989VXv+fNy784qYosS2sKlwImj7hgAAGBHVBAD5RRV9ar+D/rRkWouvNcTIbWIjdVsDl6+VNXK7tXVj9XKjQZJMQAAmwaaRr1OoLkCdccAAAA7IiAGSkX1BfbHY3XyY9TrfctSF5mmPJIoKfbHY1VT7F5dabWa2WioVsUb9TsGAKBYSIHzgOwYAABgHQTEQBmoXFi1khA/5MJWs5mHq01VO4v+ixdqWTz1X/fNG6vZJCkGABSaihdJgYuI7BgAACB3jg/GAAAgAElEQVRCD2JUWtF7EKvI1b+5UWekRYlcF+Js8+REdZ/IQ5wNAEX0817PajaLuzvLORUUhnd3cjoVP6TAcjIJ7+7i/8yo11Uv4KgjMHFh+dDvGAAAlBIBMSqtoAFxVIob3N6K4uTCy5YLn1VjCpJiANhIQXdnubJFCqwdHKgEcB/rvqJwyI4BAEBx0WICKAzVXNgbjX5c9u3Fi0Iv+6YfHvbOznpnZ9GSerbn2Z6nHx2ppJjzJQBAgjZNgY16nRQYa6JnBQAAKC4CYiDv4rmwSk5VQ4asx5Uk7eCg8+xZ59mzKCl2r66cd+9IigFgTQsRZ5WF8/nH4s3xWJACIwfIjgEAQM4REAN5FOWk3mgkhNCPjjqnp1XISaOkWPyQjKuk+GPFdOmScQBIikqXqmMhBQ5ub8P5PJjNVPOliH50pB8eagcHndNT8UP+SwqM/CA7BgAAeUBADORIMJv543GUCxv1eu/58yrkwg9StcODly9VUqzC4igpNk9OCtdzGQCwkU1TYPUhosp/VV1wBoMGEkJ2DAAAUkNADGRvea22vmWxVltEJcUitjqfe3Ulirw6HwAgQgoMbIrsGAAAJOuT+/v7rMcAZCbbZd/ldKqKhdWJsdVsGvU6ufA6VFLs39xEm848OTEbDTYdgGpqffedEGL49ddZD2SVqCNwOJ9HAdZjKbB+dCRIgYFELWfHQgiyYwAAIKggBtIXlcGqs2Kr2VT9hTn7XZ9xfGwcH4sXL6Li6+6bN0IIlbATsgOoGqNeV1lP5lT2FE+Bw7u7hbFptZrKnlQtsHF8rBaLY+oG9mofdcd8fgMAQDlQQYxKc6+v9cPDlBercS4vncvLqF6Yo+pEqKRYTibeaPT+l7+kzgVApfjjcTCbqRU+Mx9J69tvRSwF/vh/UmCgmFbXHd//6lcZjg0AACSFgBhIWzifEwrvD5sXADKkCodJgYEqkNMpH8kDAFAOBMQAAAAAAAAAUFGfZj0AAAAAAAAAAEA2CIgBAACKTUoZhmHWowAAAABQSATEKD/f923bllJmOwwpZbvdbrVarVbLdd1sB1NKbGEAlZX5bo4ZGKgg27Zt2856FAAAIAEExCg/0zRN08y2tCoMQ9u2B4PBcDgcDodhGHqel+F4yoctDKDKLMvSdT2rZ2cGBirIdV3DMDKvwAAAAIkgIAbSEARBr9fTNE192+v1OHlOFlsYQJV1Op0MA2JmYKBqwjCUUlqWlfVAAABAMj7LegBAeqIrcDVN6/V6hmGon3ue53leGIaapum6Hj/LTUr0XIqUMsMz+VJiCwMoLtd1VaKqdk+e5wVBcH5+rm6VUrquGwSBpmmapvX7/fhOKtq19fv9+EzYbrd1Xdd1XT2yrusLd1z9pBthBgaqxnGcTqeT9SgAAEBiCIhRFY7j9Hq9fr8vhAjDsN1uDwYDdebs+378PLzb7W53hrymMAxd11UjwT6whQEUiEpmh8Oh+ta27TAM43sl27bPz89Vtqta/UbfCiHUXKfuFX/Y8/Pzzz//vN/vq0dWjzMYDNZ50l0wAwOlFwSBWPpkCAAAFBotJlAVhmGYpqm+1jRtMBg4jiOE8DwvOmGO/tn+ro1V2XSn00m8SBkKWxhAsSzEqb1eT4Uv0a3xONgwDMuy1txJGYYRlfgZhrHwsCuedGvMwEAV2Lbd6/WyHgUAAEgSFcSoioUyB13X1cmw7/utVmvhH++ppZpaxmfhKmAkiC0MoOhUs6P4twthq2VZ6gPO/T3pdpiBgSqQUoZhGM1CQRB4nkczYgAAio6AGFWxsJKG6jgshDBNc68NJSJBEDiOs9ACEgliCwMoooUpKwiCeDHvQuMIIYSUcvdZbvWTboEZGKgItVxH9K3v+/QcBwCgBGgxgarwfT9+mm3btrrwdrkUa/fz5GWqtXH8zDnx+q+KYwsDKKhOp9PtdtUeShXhxitwTdO0bTv6VhXu7V6st/pJN8UMDFSHpmlmjKZpXDQAAEAJfHJ/f5/1GID9chzH8zy1RLuqcQiCwDTNqDOj4zjRkuvqbDnxGqhWqxXVLEei1YGwO7YwgOLyfd91XV3XwzDs9XoLDYJd141q9KSU8R4Onue5riuECIIgakYxGAw0TWu321LKTqcTrWLnum707ZNPuhFmYKCC1PwTn2cAAEBxERCjWlTfNMMwlvNf3/eFEA/eBABAarrdbnz1VCFEGIZSSiFEtNpqCk8KAAAAoCLoQYxqWXER3P7OugEAWJPv+8ufU6prulN+UgAAAAAVQUAMAACQpTAM2+22+towjHQu1s7kSQEAAADkEC0mAAAAAAAAAKCiqCAGUCrhfC6E0A4Osh4IAABAacnp9L+G4T/70z/VDw+zHgsAANgVATGQNvviovPsGQfTyfJGIzmd+jc3cjL56e///h/94R+ajYZRr1vNJmExAKTDH4+D2azz7FnWAwGwE388XvgiuL1Vn8HLySS8u1M/rP/RH03++39XX+tHR+rgVj881Go1IYR2cGDU60IIrVYzjo9TfQEAAGBDtJgA0vbJV18Nv/nGbDSyHkjh+eOxPx7LycS/uRFCaLWa2WiYJyc/+eyz/+e//Bd/PA5ub4UQRr1unpwYx8fmyQlhMQDsj31xISeT4ddfZz0QAA8L53M5mQghwrs7OZ2qH6qfCCHUAdUyo15Xma8KfIUQxvGxVqvN/+7vDn7yk2A2C2Yz9Zjqi2A2U8dgC7Ra7cdH+OGL6JCYY2MAADJEBTGAIonKhP3xWBWwWM1m37JUBBz9sy//xb8QQgSzmUqQ3asr9Y/NkxOjXjcbDU5CAABAmcjpVB3tbJ3Yqq8TTGxX5NHRB/zOu3cL93osjxZC6IeHXIQHAMA+UEEMpI0K4k1FOa83Gm2d8z6YLEfFxXscPQBUBhXEwJ6s2fMhrkA9H8r96gAAKAQqiAHkUTif+zc3qolE1Cmic3q6dacI4/g4OluIelPYnic8L+pNYTYalKUAAIDU7N7zQaWiha6xXadm+bH66GA2C344VlxARwsAANZHQAwgR+JrzQkh9KMjs9Ewzs6SXWsuXnocPWP3zZsfn5HV7QAAwG5y2POhuNYpCqajBQAAW6PFBJA2WkwseGytuZTreR+sWWZ1OwBYHy0mUB10RSgufncAACyjghhABtZcay5N2sGB1WxazaZgdTsAAKqKng+lR0cLAACWUUEMpK2yFcSJrDWXPla3A4A1UUGMnNu954P6gtQPIonPEj7+hM8SAAA5QAUxgD1KfK259LG6HQAA+bdj34CP7QLoG4C1aQcH0ScE6hK0Bz32zpSTCR0tAAD5QUAMIHnprDWXPla3AwAgZfR8QKHR0QIAUAi0mADSVtYWEzlZay59rG4HAAotJrApej4Aa6KjBQBgr6ggBrC9HK41lz5WtwMAYBk9H4AE0dECALBXVBADaSt6BXFB15pLH6vbAagaKogrgkpGoNCo3AcALKOCGMDTSrDWXPpY3Q4AUDi7J0fqa5IjILfWqVRY8TlQ1FDOefdu8ZH5HAgACouAGMCjyrrWXPpY3Q4AkDl6PgBYEx0tAKBqaDEBpC3nLSYqu9Zc+ljdDkDJ0GIiK/R8AJBbdLQAgEKgghgAa81lg9XtAABPoucDgEKjowUAFAIVxEDa8lNBHMxmzuUla83lzYN5fef0lN8LgJyjgngX9sWF2LDnA1dnA6iUHfvkaLWafnjYefYsrfECQJEQEANp88djo17PQwOBYDZrv35NT4M8izp+WM0mh7MAci6YzcL5nIxyO5989RXVcACwo9VXXeiHh3yKCQAPIiAGAAAAAAAAgIr6NOsBAAAAAAAAAACyQUAMAAAAAAAAABVFQAwAAAAAAAAAFfVZ1gMAyiMMQ03Tln8eBIHrukKIfr+f+qAeIKV0HCcMQyGEZVmdTifrEeFR/LIA5NNjuzxsSkrpum4QBEIIwzBycqgAAGXCETUAPIlF6oDEtNvt8/Pzx25ttVrD4TDN8TwoDEM1TnVi7ziOruuWZWU9LjyAXxaA3Fq9y8P6bNvu9Xpqnvd93/d9MmIASBBH1ACwDlpMAIlRH0rnXBAE0YmoEKLX63mel+2Q8Bh+WQByqxC7vELo9/vRPG+appQy2/EAQMlwRA0A66DFBJCAIAi63a6UstVqqZ9omrZcWuW6ru/7YRjquh4/IRRCeJ7neZ66YlfX9fhBTLIMw4h/K6XUdX0fT4Td8csCkEPr7PKitgmapmmatrDLw2Nc112Y+QEAO+KIGgDWQYsJIDGrm0h8/vnn/X5fdbxSZ86DwUDd5Hme7/vRt6pJVgqX7oZhaNs25+2FwC8LQK6s2OVJKW3bji7mXfgWD1KZu2EY0cEAACBxHFEDwGNoMQGkxDCMaD0EwzDUcjSK53nxE0LDMEzT3PelT6obV6fT4fAo//hlASgQ13XjcbBhGJZlcT3vaoPBQAXualVbAEDiOKIGgBVoMQFkz/f96ELdyF5XTog+POdS1vzjlwWgWFRbifhPLMtyHCer8RSFpmmDwUCFF1mPBQDKhiNqAFiNgBjInmmaaa4FHwSB4zhcWlUI/LIAFM7y+nVSSiaxNdEZEwASxxE1ADyJFhNAYnRdj58Vr7/C+3JpVRAE8R4UCZJSdrvd+OERVV25xS8LQG6t2OWZpmnbdvwmx3H2ellMcUkp47v7MAz3tPcHgMriiBoA1sEidUBi1Do86qoldaqsOgurdldSyk6n0+/3hRCtVktKaVlW1HrYcZxoRV113z19xN1qtcIwXHjkFWvrIUP8sgDk1mO7PMV1Xd/31U5NSsklvY9R6Xk01QdBMBgMKHADgARxRA0A6yAgBpIUhqGUUghhGMYWJ3i+7299XwAA0rR6lxfdappmBoMrFLWtNE0jRgcAAEAmCIgBAAAAAAAAoKLoQQwAAAAAAAAAFUVADAAAAAAAAAAV9VnWAwAAPEFOp/81DP/Zn/6pfniY9VgAAMnzRiOtVjPqde3gIOuxAECpyOk0vLsTQgSz2X/667/+Jz/9qVGvq5vMRiPLkQFAnhAQA9VlX1zoh4edZ8+yHggeFs7n3mjkXl3JyeSP/9E/+v/+5m+sZtM8OeFXBgAloCZ5/+bGG42iHxr1utVsWs0mnwgCwGr+eKy+kJNJOJ8LIcK7u2A2+3jrzc3yXeqHh5Mf/kGcfnSkZl3t4EA/OlI/jOJj/fCQORlA6bFIHVBdre++M+r1/osXWQ8Ei/zxWEXDQgjz5ESFBe7VlTcayclEq9WsZrNzemocH2c9UgDAZuR0qkJhOZkIIYx6Xc3z2sGB+qHKi/WjI6vZNBsNCtwAVEo4n6vpUcQi4OD2NoqAo1sXrB/yxp9iOVxe8RTqUg/1dfRF9MharcbBOYDiIiAGqqv13XdCiOHXX2c9EHwUzGbu9bU3GgW3tyoa6Dx7tlCwIKdTlR2Hd3f60VHn2TMKzQAg//zxWH34F9zeCiGsZlMVCy9P4KqyWCXF4d2d+lDQbDTMkxMaUAAormA2+zGBnU7VD6MoNpjN1PS4zKjXtVpNCKEfHqovtIODdNpExNtTRLXJvxMu3909eEfz5ER9kf6YAWA7BMRAdREQ54d7fR1dZdw5PTUbDavZXH0XbzRS/wkhaD0BADkUzuf+zY3KhbeLepdjZfPkxGw0+FwQQH6sTlEfbPWgRClq0atxH8y+o6rnFdk3rS0A5AcBMVBdBMSZi5cDq1KyzunpRgVi4XxO6wkAyJVgNvPH4+hjv0SaRSw3plBhMRM+gP1Z0eR30z4MxvFxVAVc2dCT1hYA8oyAGKguAuKsxFefSyrVpfUEAGRLzcP+zc1eM9wH02fj+PjJ604AQFnd5HedVg8Plrsa9TptcJJCawsA6SMgBqrLvriQkwkBcZqWV59LvC8ErScAIE3eaKSi4ZS7QDzYv0Kl0mQ0QDVFqeKDTX5JFcuH1hYAEkRADFQXAXFq1ll9Llm0ngCA/XlsHbmsynhVQr3OCngAiujHOt9NmvzSlwAKrS0ArIOAGKguAuIUbLH6XLJoPQEASQlmsygXFj+0d1AdHrIe2kdqhPFWxepqlfyMEEBk69guKv8UNPlFomhtAVQZATFQXfbFhXt19eHf/JusB1JCu68+lzhaTwDAdh5sLpzzD9tUjXO8VbFaJY9WxUAK4uHadk1+o5RN0OQX+UNrC6CUCIiB6rIvLpx37+5/9ausB1Ie+1h9Llm0ngCANXmjkT8e++NxcHur1Wpmo6GqcQsX0JTmhQCZ272+MqqppL4SpUdrC6BYCIiB6iIgTlAKq88li9YTALAsKrz1x2M1PZap8HahFFotpsfkD4hYnW8UYwma/AJpobUFkAcExEB1ERDvLv3V5xJH6wkAUK17/fFYxUClb91btdeLyooXMEYR8EZNfuMXwtPkF8gWrS2AvSIgBqrLuby0PY+AeDuZrz6XLFpPAKggVVGrPuQTlayoXaiYVvN/0fdoqIKtc6LlJr/xekOa/AIlQGsLYDsExEB1+eNx69tvCYg3ksPV55JF6wkApaeKZ73RKIpE1Xxepsl8C95opHYBKllT15SwWZCyBJv8xqsCudIcwDJaWwBxBMRAdREQry//q88ljtYTAMokKpVV05pqCmQcH1Mqu2yhVbFKz/mwEDtabvIbFfSJDZv8Rq0eKOgDkAJaW6AiCIiB6lIB8Ye/+AuKg1Yo3OpzyaL1BIBCk9OpCoWjrJNmu+tTrYrlZBJP1dWqfVkPDXnxYJPfKDfZtMkvuQmA4qK1BYqOgBioLhUQD7/5hjO9ZSVYfS5ZtJ4AUCD+eKw+3ou6JagyWGat7aj6a5UUx1sVmycnfMZcVg9WzEXpxtZNfjnmBABaWyCfCIiB6iIgfoxavk+UYvW5xMVbT/Qtq3d2lvWIAOBH/njcfv2aEHN/FsJ3dgTlIKdTdeTzWKsHsbLJLwVuAJC4HVtbGPV6/8WL1EaLEiAgBoBF/ngsJ5OSrT6XLNV6wqjX+XQBQK6E87lzeUkbhBSo9h3sCMpBBcTLTX5p9QAAOfdYawutViMgxkYIiAEAAAAAAACgoj7NegAAAAAAAAAAgGwQEAMAAAAAAABARREQAwAAAAAAAEBFfZb1AAAkQ0rpOE4YhkIIy7I6nU781iAIfN/3fb/T6ZimudEjh2GoaVqSY82r1dsQClsJQA4xNaVMSum6bhAEQgjDMPr9ftYjwjb4wwGAkmFix9aoIAbKIAxD27YHg8FwOBwOh2EYep4Xv9VxHE3TdF3f4sG73W5yI82v1dsQClsJQA4xNaXP87x+v682uGmatm1nPSJsjD8cACgZJnbs4pP7+/usxwBgV1LKMAzjpcHtdvv8/Hzhn9m2bZrmphXErVZrOBwmMMp8W3MbVhxbCUAOMTVlriKHCiXDHw4AlAwTO3ZBiwmgDAzDiH8rpdyoWFhKqWp/oipjda1oEATdbldK2Wq11L/UNK2sO5gdt2FFsJUA5BBTU7Zc1134FaAQ+MMBgJJhYscuCIiBsgnD0HXdjboBdrvd4XCoGg1HYbEQQtf14XBYwbKgLbZhBbGVAOQQU1Oa1KfIhmEMBoOsx4Kd8IcDACXDxI5N0YMYKJUwDNvtdqfT2WhZOV3Xfd9XX3Oat902rBq2EoAcYmpKmepyKIRwXTfrsWB7/OEAQMkwsWMLBMRAeaie9P1+f9MrPQeDQRAEtm232+1ut6vWPK2mrbdhpbCVAOQQU1MmNE0bDAbRx8woHP5wAKBkmNixHVpMACURBIHjOP1+f9MPCVUc3Ov1om9brdb79++TH2Lubb0NK4WtBCCHmJqyRYvDguIPBwBKhokdW6OCGCgDKWW3243vBhzHWf++8StDl3ckuq7Ha4rLWl+8yzasDrYSgBxiakqZlDIIgujbMAzj36Io+MMBgJJhYscuPrm/v896DAB21Wq1wjBcyHajleVUByIhRBAEmqZpmqbretRo2Pd913XVz9W/MU2z0+lEj6OWrVPXp6h0uJRNildvQyhsJQA5xNSUsjAMHceJtnkQBIPBgEqlwuEPBwBKhokduyAgBiCEEGEYSimFEKZprrjVMAzOAAEAgDo20DSNFocAAABFR0AMAAAAAAAAABVFD2IAAAAAAAAAqCgCYgAAAAAAAACoqM+yHgAA5It7fS2E6Dx7lvVAci2Yzf7v//gf/+gP/oANBQDVFMxm3mj03/72b//pn/yJ1WxqBwdZjwgAgEoL53M5mcjJJJjNhBBarWY2Gka9zj4a66AHMQB8JKfT7ps3cjIRQhj1+uDLL43j46wHlTvhfO5cXjrv3h385Cfzv/s7/eho8PKl2WhkPS4AEEKI1nffGfV6/8WLrAdSWioX9kYjtbv885/97C9//WshhNVsmicnJMVF5I/HrW+/vf/Vr7IeCABgM8FsJicTOZ0Gt7fq/+rn/8uf/ZkQ4v/9q79S3+pHR/rhoVGvG8fH+tERJ7l4EBXEACCEEM7lpXN5KYQYfvONEKL79u0Xv/xl7/lzUoY49/ra9rzw7q73/Hnv7ExOJs7lZevbb82Tk75lcagBIA/Cu7ush1BCcjpVubA6+bSazc7pqYqDo8i4++ZN980bo163mk2r2dQPD7MeNQAApSKnU1UgrCqF1TGPVqupnW+UAj/4792rq+gYyTw5UZXFRr1OrQ8UKogBVF04n3ffvPFGI6vZHHz5pSp9iupkKSVW/PHY9jw5mZgnJ4OXL+On/e71tXN5Gdzedk5Pe2dnJAIAMtT67jshxPDrr7MeSEnEc2F1peqKMmGVFPvjsX9zI4QgKS4KKogBIJ/iLSPUF+rnqgpY/d+o19ffzwazWTCb+ePxQsVx9IBmo6EfHrLjriYCYgCV5o/H7devhRC9s7Pe2dnyrd23b8O7uwdvrYhgNnMuL92rq9XdJOyLC/fqSgihYmIuMQaQCQLiRCzkwlazaTYaVrO55t3D+dwbjfybG280EiTFuUdADAA58VjLCKNeV0Gw+iLBUy1/PJaTiUqi1ee74oeS5B+fsfLFUhVBQAyguuyLiydrhMP53PY89+pquXK29FQZtYp914nIo7JrrVbrWxbr1wFIHwHxLvzx2B+Pt86Flz2YFJsnJ5xq5goBMQBkZUXLiEzy2SibVqNaSKhZ9a7cCIgBVFG0Ht2aXYa90aj79q1YLycth3jjiL5lrX8QEMxmtud5oxHr1wFIHwHxFlRfCG80Cu/uEsmFly0kxfrRkaopJinOAwJiAEhH4i0j9u1jZfF4HN7dLQyYVe/Kh4AYQOVE69Gdv3q1fnwZtSo2T07OX70q8aemcjq1Pc+/udll6Tl/PHYuL3d8EADYlH1xIScTAuJ1xHNhldgax8fJ5sLLVFIsJ5P485IUZ4uAGAD2JP2WEfv2YMmzYNW7UiAgBlAhD65Ht5GolHjw8uW+z6LTF/XT0I+O+i9e7P4CWb8OQMoIiJ+0nAtnlc/mZyQVR0AMAEnJW8uIfWPVuzIhIAZQFavXo1tfOJ+3X7/2b262TpnzKb7K3DptN7Z7ZNavA7BXBMQPynndbia1zIgQEAPAdgrXMiIFrHpXXATEACphnfXoNhL1qShBKbE3GtkXF3ut82X9OgDpULM9UZdSuM6/KXRDxjICYgBYU/laRuwbq94VCAExgJLbdD269QWzWfftW//mZtNl3PIjegnmyUnv7Gzf7aJYvw7AvhEQi6Vc2KjXrWbTPDnJbS68zB+PVVIc3N6SFO8bATEAPKZqLSP2jVXv8oyAGECZbbce3RZPodVqxUo8o5Je/eiod3aWZkkv69cB2J8qB8QP5sJWs1noK1vldOqNRiTFe0VADAAKLSPSx6p3+UFADKCcdl+Pbn3xIuVC9NhVoXZ4d5fhgFm/DsA+VDAgDmYz1ZlBdforRy68bCEpNhsN8+TEajbzv8/NPwJiAJVFy4i8YdW7DBEQAyihpNaj24hKJXLePMEfj7tv3wa3t1az2beszPesrF8HIFnu9XX3zZsqRF0qF1ZLz4ny5sLL4kmxEEJ1zyAp3gUBMYDqoGVE4bDqXWoIiAGUTeLr0a1vf/2Odxf1/zXq9b5l5SfCZv06AAkqfdS1kAtXOR5lUySl9H81ACqLlhHlw6p3+0NADKA8cpLPZphQP6gQCSzr1wFIRFmjLsLQFdg4OyrrXw2ACqJlRNWw6l2CCIgBlEQK69GtL+rkkHkpsXt9bXtetu2G18f6dQB2pKKu7//1vy5HNRDtFDZS2bYbOyIgBlBctIzAAla92xoBMYDCS3M9uvVFdbtZlRL747HteXIyMU9OBi9fFuj0mPXrAGxNRV3Db74p9KE/C7LtqCIL9yWFgBhAUdAyApti1bv1ERADKDy10Fma69GtT5US64eHw6+/TvmpP/+X/1Kr1YrbrkH9Ws1G4/zVq6zHAqAw5HRqe16hL0HwRqP269darWY1m2ajYTWbWY+owML53BuN/JsbbzQSQnz4i78gZF+m/mrSP1ABgI0Es9nP/9W/Ul/TMgJbe3DVu8GXX+azE2OaCIgBlEEwm+X2Q79wPhdCpH/UIqfT4uYjSjifh3d3uf3NAsCeqGtish5FqajzwIJ+YgoAUNzra1pGIFmqrJjCc0FADAAAAAAAAACV9WnWAwAAAAAAAAAAZIOAGAAAAAAAAAAq6rOsBwAAW5JSOo4ThqEQwrKsTqeT9Yh+RybDk1K6rhsEgRDCMIx+v5/CkyYu579ZAPln23YQBOfn51kPZAPlmMBzqGr7lDAMNU1b/nkQBK7rCiF4awEolqpN40gNb60F9CAGUEhhGLbb7fPzc3UW5DiOruuWZWU9ro+yGp5t271eTz2p7/u+7xfuPDDnv1kARdFqtYbDYdaj2EAJJvAcquA+Rb3ex24t3N8FgIqr4DSOdPDWWkZADKCQpJRhGJqmGf1k9RlRynIyvCKeB+Zk0wEouiJOgHFFH39OVHCfsvqdw/sKQLFUcBpHOnhrLaPFBIBCMgwj/q2UUri/4/IAACAASURBVNf1rAazLA/Dc113YRiFkIdNByBztm2rL9QkYBiG53lCiH6/H80Sqo+EuqBe1/XH6m3b7bb6N/GDfs/zPM+L7huV7uZEQSfwHKrUPiUIgm63K6VstVrqJwtve8V1Xd/3wzBUfzXxd/5jf1PtdlvXdV3X1Z/h8h0BYE8qNY0jTby1HnAPAAX34cOHTqfz4cOHrAfysPSH1+l0DMPodDqpPeOe5Pw3C2CvTNN8//79/f19r9dTE5qaE6J/EJ8c+v3+YDBYuLv6N5ZlqceJnJ+fxx/n/fv3lmXt50VsrDQTeA5VZJ+i3vmP0TQt+kt5//79wjttxd/U6jsCQAoqMo0jfby1lE+zDqgBYCeqeVCn08lnJUsmwxsMBuoCUrUcTUHl/DcLYN8Mw4iKO1RLOE3T1BpuiqZpYRj6vh8EgWEY8ZsUKWW73R4MBgtFIp7nDQaD+BOZpqlKIzNXjgk8h9inKOrjh+jrhb+aFX9Tq+8IAPvGNI494a0VocUEgAILw9C27fgVx7mS4fBUpY/a1aX81InI+W8WQObCMOx2u5qmqaN5KeXydKEaNSwf7vu+H12DH8nPsiRFn8BziH3KOtb5mwKATDCNY094a8UREAMoqiAIHMfJbRe8PAyvoH2U8rDpAORct9vt9XrR0bzv+77vL/ybwWCg+rFGS1QrpmnmfxGSgk7gOcQ+ZU3r/E0BQPqYxrEnvLUW0GICQCFJKbvdbnw2dxwn2yHFZTI8KWX8ks8wDIt4BWjOf7MAckLTtHitx2MNIgzD6Pf7ap266IeWZS1MLEEQZDthlmMCz6EK7lN0XY+/2+Nfr7bm3xQApKmC0zjSwVtr2Sf39/dZjwEANtZqtdQq2/Efqr6NeZDJ8MIwdBwnet4gCAaDQeE+Ds35bxZAOmzb9jzPsqx+v2/btu/75+fnuq5//vnn/X6/0+l4nuf7fnQtvGmaruuqfy+EaLfbvu9/+PBBCBEEgWooEd0qhHAcJ1quWiVo2daPlGMCz6EK7lOklLZtq6hXvbdVx23VY1FK2el01B9Cq9WSUlqWpf7BY39TvV5v4Y62bbuuG30LAPtTwWkc6eCttYyAGABKJQxDKeVCHRAAlM/u0526gv7BPsWZYAJHItQbSWz+3uYdCABAZREQAwAAAAAAAEBF0YMYAAAAAAAAACqKgBgAAAAAAAAAKoqAGAAAAAAAAAAq6rOsBwAAOwlms3A+N46Psx7Io+R0qh0c6IeHaT6pPx4b9bp2cJDmkyZLTqf/x7/7d3/4D/9h37IK/UIAYH3BbPa//9t/y9SXOH88/j///b+v/d7v9S0r5T0yAGA7cjoNbm/V/4PZ7O9/+9ujn/5UPzzUajWz0dBqtTyfA6IonMvL//D99//zn/xJ7+ys4odeLFIHoNjsiws5mQy//jrrgTyq9d13Rr3ef/EizSf95Kuvht98YzYaaT5pgtzra9vzfvLZZ7/5+7//x3/wB+evXnH8B6D05HTa+vbbv//tb//Hb35j1OuDL79k6ttdMJvZnueNRv/TH//xf/vbv/2b+bz3/DkngQCQN/54HMxmwWwmJ5NgNgtub9XPtVrNqNf1w8Pa7/3eX334EM7n/s1NdC91k350ZBwfq+A4o+GjePzxuPv2bXB7++c/+9lf/vrXWq3WOzvrnZ1lPa7MUEEMAMiRcD63Pc+9urKazcGXXwazWffNmy9++cu+ZVV5bw2g9Nzr6+6bN0a9Pvzmm2A2a79+3fr228HLl1azmfXQCsy+uHCvroQQaicSzufO5aXz7p17dVXxk0AAyFA4n8vJRE4m6ovlONhqNrWDA6Nef+yaSBUl++NxeHcXzGbu1VV4d6du0o+O9MNDdccVj4AqC2az7tu3/s2NOu4yGw31cbLtee719eDly2p+0kAFMYBio4L4QQWtIJbTaffNGzmZxOPghciYIzwA5WNfXDjv3nVOT6POEuF83n792r+56T1/nvIepByisqD4VlWimmL96KiyJ4EAkJoozBVCfMyFl8Jc/fBQPzzcZUJ+MnQ26nUhhNloqOfa9VWhmKKPirVarW9ZnWfP4rf647HteXIyMU9OBi9fVu19QkAMoNgIiB9UxIBYtZXQarUHG0qsvhUACiqcz7tv3nij0YNBsAqOzZOT81ev+HhsTVH+a9Trfct6bFcYnQRazSaNiQEgKXI6De/uotrebNtBRG0rVBdjOZlEN5knJ6qdsXF8rIa078Egc+qMMry7W91sas1/Vj4ExACKjYD4QcUKiNesEX6wvhgAiku1kghms+Ualggfj20k6imxZgcJ5/LSubys4EkgACRiYR25/Cew0YBX5Ne7lzMjbzYtDV5daFxWBMQAio2A+EEFCog3in1pNwGgNNSSdEKI4TffrD5tVvPk6hwZK3pKrFbNk0AA2MKT68ipuuBi9XBYpwMG7YyLa5e+UvFWxSuuSSoNAmIAxUZA/KCiBMTbVcZRTweg6OJL0q1zthl1ouicng5evkxhhAWyZk+JJx+kUieBALDaOi19yxqb/viSyxWFV038A+BdVqaNPoEufU8qAmIAxUZA/KD8B8Q71gLTbgJAcS0vSbfRHdePlatg054Sq8XLkHtnZyU+CQSAuHTWkSs01VX5sWYa2sGBaqORn2YaFedeXzuXl8HtbVItpFRPKiGEOkIo5WEYATGAYiMgflDOA+JE4l3aTQAonNVL0q3DG426b9+KNRpTlN7WPSWeFIXOJT4JBFBluVpHrrhWb8ao1rjKqXom/PHYubz0b27Mk5O+ZSV4sBSdfupHR72zs/L1pCIgBlBsBMQPynNAnGyDCNpNACiKdZakW/9xqnwJRSI9JVYr/UkggOoo3DpyxRV1pfixQccPhdil78uRB8Fs5lxeqn13/8ULq9ncx7PI6dT2PBVA987O8nnGvR0CYgDFRkD8oHwGxHuq+aXdBID8W39JunVE02nixbP5l2xPidXiJ4HJViEBwJ6Uch25QnvsNyJiAT2/kd3Fr/5J4dTbG43si4t9XMaUIQJiAMVGQPygHAbEe41xaTcBIM82XZJuTc7lpe15Rr1+/upVFc4q99dTYrVSngQCKIEqryNXaCtquuMtPqjpXlOGu+mUU+l9IyAGUGwExA/KW0CcTiMI2k0AyKGtl6Rbhz8et1+/FkKcv3qVnzk/cSn0lFhNrYReppNAAIXDOnIlFv1yl9sZL/xyyfrj8tDtIZ2+FukgIAZQbATED8pPQJxybS/tJgDkx+5L0q35LK1vv5WTyV6fJUNp9pRYrUwngQByjnXkKm798vBq9qbI21IB+1sZL00ExACKjYD4QTkJiDOJa2k3ASAPklqSbk3dt2/LN+9l1VNitfhJ4ODlywqelgNIHOvIYR3xdsbh3d2D7xOz0VDvluyGuXfO5aVzeSmE6Jye9s7OcnJ4IIRwr6+dy8vg9rb3/HmuBrYmAmIAxUZA/KA8BMTZNnyg3QSADCW7JN2aok7Hgy+/LPq8l3lPiSepvUx4d1fQk0AAGWIdOSQl/tFCOJ+XvtI8+uTYajb7lpXDPxDVk8p5906r1TK/8mlTBMQAio2A+EHZBsQ5qeGl3QSATOxpSbp1yOm0/fp1eHc3ePmyuA0Q8tNTYrVCnwQCSAfryCFlC+2MH+xVXcS3XDCbdd++9W9ucvvJcVz0Obd+dDR4+TLno40QEAMoNgLiB2UYEOcqls1JVA2gOva6JN06wvm8/fq1f3NTxJbE+ewpsVpBTwIB7Ik/HrOOHHJlnU8phBBmo5HPt2X849h02nYlxR+Pbc+Tk0lRelJ9lvUAAGAn6vKrrEexitVspj/C3vPnWW2W8O4unM/f//KXebjAWTs4GLx8adTrqk0VAOxb5ovFaQcHw6+/Vp+eZjWGrfnjsXZwkHmLpI3oh4fnr16pk0B/PC7QyAHsg+ovpILgzulpma7uR0FpBwfL4W/U5yS4vfVvbuRk4rx7d/+rX2U1yNXcq6siNnQyG433v/iF6kkVzGY5Ty0EFcQAAAAAAAC7K0QMBCzjrQsCYgAAAAAAAACoqE+zHgAAAAAAAAAAIBsExAAAAAAAAABQUSxSB2DvwjCUUgohDMPQNC0MQ03TEnnYRB4nWWEYttttXdcHg0HWY0mVlNJxnDAMhRCWZXU6naxHJEReRwWgZHI11UgpXdcNgkAIYRhGv9/PcDBrytUG3FQRNziAZBV6EkPF5fndm+exralYL4EexAD2y3Vd3/ctywrDMAgCleomcvrUbrfPz893f5zEffLJJ6ZpDofDrAeSHhWLn5+fq8jecRxd1y3LYlQASi9vU41t271eTw3G933f93MeWeZtA26qcBscQLKKPomhyvL87s3z2NZUuJdAQAxgjzzPk1LGz5Rs2xZCJHLu1Gq18hnCfvHFF6ZpVur8UEoZhqFpmtFP8hDf53NUAEom51NNbveVkZxvwE3lf4MDSFbJJjFUSp7fvXke25oK9xJoMQFgj1zXXThN6nQ66goLxfM8z/NUWbGu61ENjhBCNWrQdd3zPCGEruv9fl/dGgRBt9uVUrZaLfWPNU1TU626KbrG0/M813Xjl3yqfyCEGA6H6tmFEKZpqoG12201SPXJnm3bUspNrxh9sPGFlFKF4+qViq1S8hXbRHlse6rtIISIPsD0fd9xHCHEYDBQ49mFYRjxb6WUuz/m7vI5KgAlk+epRu0Bsx7FE/K8ATdViA0OIFllmsRQNXl+9+Z5bGsq3ku4B4C9MU1zxa3n5+edTif69v3795Zlxf+BpmmDwSC6Nf6PVz/4wk3L/9I0zV6v1+/31bfD4VB98f333y88i2VZHz58WPEqlvV6vegBI4ZhRI/z/v371VtmhRXbZPX2XHhd6t9HD5WgDx8+dDqdTTfavuVzVABKJj9TTafTMQxjeebPufxswE0VdIMDSFZxJzEgz+/ePI9tTYV4CZ9mHVADqC7P8+IruRmGYZqmqo2NfhL1cTcMQ60AkxRVYKu+jq77UJ/pRU/k+76u65suhdfv9+MXkkSP7Pu++towjK2XsFuxTVZvz2gA7XZb1Q5LKZfHuSNVhd3pdHK1fmA+RwWgZHI11QwGA3UFj7p8pBBytQE3VcQNDiBZhZ7EUHF5fvfmeWxrKspLoMUEgMz4vh/1iIik1rX9sSVEe72e4zgqaXVdd+skd8FgMHBd17btIAg0TdvHAqart6dpmlHeLaUUQgRBkOxFLmEY2rbd7/dzdYFtPkcFoGRyONWoK07UCUnWY3laDjfgpoq1wQEkqwSTGCorz+/ePI9tTQV6CQTEAPZI0zTVD/fBW03TzGGP9qiIOAzDLcqHH6TaLkcFy2EYtlqt9+/f7/7Icau3p2EYqnDYNE0ppcqpE3z2IAgcx1noiZy5fI4KQMnkearJe7c7IUS+N+CmCrHBASSrTJMYqibP7948j21NxXoJtJgAsEf9ft+27fiqdKqKVn1tWZaKLCNBEKzfR0LX9fgjx7+Oi/esWJMqIvY8L4p0dySljF9zuqfdw5PbU9M03/cty7Isy7btBIchpex2u/E938JIMpHPUQEomVxNNerzv+jbMAyT7c60D7nagJsq4gYHkKxCT2KouDy/e/M8tjUV7iV8cn9/n/UYAJSZlNK2bXU9haom7vf70a2O40SreaqEV02gqk2PlLLT6ah/b9u267rRt8uPLISI2kGoVFo9jq7rjuNEd3Qcx/d9KWV0iceDl3t0u92Foe7C933XdTVNU/uGIAhM09z0EtR1tslj21PdqjaL2ko///nPB4NBUj2IW63Wcqm46saYoXyOCkDJ5GqqCcPQcZxoPEEQDAaDnBet5GoDbqqIGxxAsgo9iaHi8vzuzfPY1lS4l0BADCANUsowDA3DePCsSS2e9titK4RhqNrpLt939TM+ybbtXq+X7DleNNrEl4ZbsPX2BACUgNrdaJqW/2535cAGBwAARUdADACLgiBwXTep8mEAAAAAAIDcYpE6APhRu91WDQQ1TfM8z7KsrEcEAAAAAACwR1QQAwAAAAAAAEBFfZr1AAAAAAAAAIrt+j//56yHAABbosUEACBJ7vW1fnhoNhpZD+R35HNUAErGvrgwG408TDXu9bUQovPsWdYD2YA/HgezWbHGHMeOBqggfzyWk0kwm8nJRE4m/+vPf/4fvv/eqNeNel0/PFRfaAcHWQ8TeII/Hvvjcf/Fi6wH8oCiHx6I4rwEAmIAQJK80cio1/N2hpzPUQEoGefdOyFEHqYabzQSBQyI5WRSrDHHsaMBSi+cz1UQHCXC6ucqCLaazT/7/PP/7Z//c3Wre3WlbtWPjozjY/3oyGw0yIuRT/547Lx7l9uAuNCHB6I4L4GAGAAAAEiAVqtlPYSPtIODcD7PehQAUGwqEfbH4+D2Vk6nwe2t+nmUCK/+WEiVDaq82Hn3Tn2IGOXFxvGxKjRO6cUAj+Nzi33zb26yHsLTCIgBAACABBj1etZD+Eg/OopK2wAAa3osETZPTqxm82PXiOPjNR9tITuW02lUfazCYiGEVqupuJm8GBnKzwEMMkRADAAAAAAAKifZRHg14/g4/lAf+1RMp6ofRXh3J2J5cbJPDQBPIiAGAAAAykZlDQCAuCiWTSERXk0/PNQPD61mc2Fg8bxYDYy8GOkI53N6TVQZATEAAACQjPw0dsjPSAAgQ/HgVU4m8ULdlBPh1Rby4nhpszcaxYNs9S9ZFROJk5MJb6p9KErrGAJiAAAAAABQBisS4c7paVFa/WoHB2ajEaV1v9MKYzJxr67Uz+P9KIj2gHzK/4SjEBADAAAAAIBCKkcivNpCXiyE8MfjaMm7eF6sHx7qR0dmo2HU67QLALA+AmIAAAAAAFAMKgv+mAuXNBF+0mN5cTCbOe/eOe/eCSH0oyPj+Ji8GMiDYDbL+bxEQAwAAAAkwKjXc9L5Nw/9NAEgKfFE2L+5UT9U6Wd1EuHVFvLi+BaL8mKVoauey2wxxBn1etZDKD8CYgAAAACp0mq1rIcAANtbkQj3nj+nHvZJxvFx/JPC+PZ0r67iNdfkxRBC8NcEQUAMAAAAAAAyRCK8Vwt5cbxrc5QXCyHMk5NoyTuuRAGqhoAYAAAASEZOWkwocjrlDB9APsXXWItmThLhdOiHh/rhodVsqm+jvDi4vfVGo+D2Vv08yov1w8N4/wqUlZxM+EVXGQExAAAAkIyoDisPcjUYABX3YCKsWhxYzab6gkQ4Ewt5cTify8nEH48X8mL1O1L1xcSIpRTO51kPoZyK8vdCQAwAAAAAAJL0ZCJclNCkarSDg/iSdyovjrcwVj+P58WE+0AJEBADAAAAAIDtLcSIJMKlsZAXi9+N/qO8WLUH0Y+OaA8CFBQBMQAAAJCA/JwPa7Va1kMAUHLxRgSqfa36OYlw6S3nxcFspvJi59075907EcuLjeNjVWic3XixFo4c9s0fj3M+JRIQAwAAAAkw6vWsh/ARa9MB2BP3+tq/uYknwubJidVsfmw1wORTPQuZl5xOo/piFRYLIbRaTX1y0H/xIosx4mn5OYZBVj65v7/PegwAgPKQ06l2cJC3MoF8jgpAyah6upyUh/jjcbEu8g1ms3A+L266xI4GFWFfXMjJ5Mfms4X9m0U6PrYcmU7lZKIdHJy/epX1iPCw3O7Fin54oPjjsVoNMuuBrEJADAAAAAAAAAAV9WnWAwAAAAAAAAAAZIOAGAAAAAAAAAAqioAYALClIAjCMNz0JgAAAAAAkB+fZT0AAEBRhWFo23av1zMMY/2btn4uKaUQwjAMTdPCMNQ0LZFHBoBCc11XfSbX6XQSmXKDIHBdVwjR7/d3f7SKY2+FcpBSOo6jPvu3LKvT6WQ9IhQJ75+qCYIgCAIhhDosie8Hi7tbXPGi9iTxA7wnsUgdAGB7YRi22+3BYKDr+vo3bcp1Xd/3LcsKw1DtJjVNI7kAgIht26ZpmqaZ1AO2Wq3hcJjUo1VWu90+Pz/PehTATtQR3fn5uQpEHMfRdd2yrKzHhWLg/VM18RM33/eDIHj//n10a0F3i6tf1F4lfoC3AhXEAIDtaZr2WIKw4qaNeJ4XBEH8SMK27d0fFgCAfaPbEkogCIJerxeVy/V6vXa7TcCHNfH+qRrP86JzQMuyvvjii/itBd0trn5RpUFADADINdd1F4LmTqcTP7aQUqoLcDRNU5XF6hi03W7ruq7ruud5Qghd16ObFNu2o3pkdWt005P3BYC4FfPJmqSU6tMv9Qjihw4Prut6ntfv9z3PU512hBC9Xm/NWpLVA3vsSRVVLxOG4RZzoOd5nudFzxulA57nqf4VhmGo52q322pKV1ecPHZHsfPMrLakerG9Xi/+6WMQBN1uVwgxHA7VAIQQpmlG10GvGNVjW1g9ppSy1Wqpn2iatlA29djDPjkeIE0LlzZLKXe/OAzVwfsnhx47e9p97xMEQfz3G9/x7bJbzPDgYfWLWj1ssfLYQyRx9JiwewAAcsw0zRW3vn//3jTNDx8+PPitpmmDwSC6qdPpxO8b/bP7+/t+vx/9y3XuCwBxq+eTdRiGsTCVRTcNh0PTNIfDYfRcnU7n/fv38bv3er3oH6w/sBVPussceH5+Hv/379+/tywr+vb7779feLTo1tV33GVU5+fnvV4v+rbX6y3f1zTNXq/X7/fVt9H2XD2q1Vt4xS7syRf72HiADKn5J/62B9bH+ycPVp893e+89+l0Ov1+//vvv3/w1q13i5kcPERWvKgVz/7kscc6R4+PHeDtw6cZ59MAAOzAdd2oqZkQwjAMy7LU57Tq2+gTb8Mw1MICEbXYnWojtXzr6vsCQNzq+WQduq77vq++NgxjMBjEbzUMIyoZVsU+juPsPrAVT7rLHOh53sJDmaYZzcy6rodhGF0I4vt+VF+2+o67jMp13XhhTq/Xe/C+qvBHfR1t8NWj2vpX/+SLfWw8QFZUM9lOp8M1VdgC75+cWH32pOyy9xkMBoZhuK7bbrdVyfCad8zhwcM6L2rFsz957LH70WOyaDEBACgwdWFU/CeWZa2Tm4Rh2O12o7tLKdNZHBZA+SQynwwGA9d11cWGmqYtXM658IDqjGL3ga1+0q35vh9dQBqJd5zsdDqO46izJtVAY807JiVqqbHgwS2wYlS7/OrXebH0lEB+hGFo23a/3+d4CVvg/ZMf65w97bj3iRZVW1iicLWcHzw89qLWf/aFY48cno0SEAMAck3lII8dVSxHJFLKdQ5But1ur9eLdsO+70dldACwkd3nEzWVRdU6YRi2Wq34Atlq7ezo24V2eNsN7Mkn3ZppmqvXKDdNU52LRg0Q17zj1hb2C0EQrF+ns2JUu/zq9/digcQFQaByGWo/sQXeP7my9dnTOjzPMwwjOkTRNM0wDCnlOmXIOTx4UFa/qBXPvvrYI4dno7SYAADkWr/ft207fiij6t3U16ZpRl8LIcIwdBxnnU+M1a49+nbhuioAWN/u84laLib+gAv/QJ1dR1+rk4odB/bkk25tuRZpOZC1LMt1Xcdx4mVK69xxO51Op9vtql2JKmRbv05nxaie/NWrS2Kjb+Nf7+/FAsmSUna73Xi6t2aLG0Dw/smfrc+e1iGlXNgVquYJ0bc77hZTPnhQVr+oFc+++tgjh2ejn9zf32c9BgAAVpFSRjtUVU0cb+fkuq7v++pDXSmlunhNXfsjpVRLCgghbNt2XTf61vM83/ejK3pM03Rd17Ksfr//5H0BIG7FfLLmI/i+77puVA4TBEF80XBVURIEgZrrwjDs9Xpq0ovW9Y5X06gVvZ8c2GNPujwHtlotKaVlWQudkVdwHCdaql6dGi3Xjn3xxRemaS5spcfuuPvMrF5vtAHjnQEdx/F9P35158J10I+N6slf/cL+S/12nnzYJ8cDpKnVai1fyzUcDrMaD4qF908OPXj2JNbYGz5JRc/Rb9z3/U6nEw9zt9stxp8i5YOHdV7UimGvOPZYfQix+gBvTwiIAQDFIKUMw9AwjOUytzAM1VoBG62ioO618OEtAGwhkfnksalMBcTbrVH25MC2mz/XpEb+4Ly9pzuur9vtrp93Kw+Oav0t/NjLSeHFAgAQt9e9/+oH399uca/70ye32DrPvnDskbezUQJiAAAAIL92CYjxINXpj4tCAABAOvJ/7MEidQAAAEBOua6rrjHUNG3flxaWm7rOVH1tGEaez9AAAEAJFOvYgwpiAAAAAAAAAKgoKogBAAAAAMAqwWwWzGbh3Z15cqIdHGQ9HBSMNxpptZpRr/PmAfKJgBgAAAAAACwK53P/5sYfj/3xOLi9jX5u1OvmyYnZaJiNRobDQ/7547E3GnmjUXh3p35iNZvmyUnn2bNsBwZgAS0mAAAAAADAR95oJKdT/+ZGTiZCCP3oyGw0jHrdajbDuztvNJKTiT8eq8jPPDlRNxnHxxmPG7khp1OVCwe3t1qt1jk9tZpN7eDAG43c62v1Q6vZNBsNq9nMerAAhCAgBgAAAACg4lSZsJxM/JsbIYRWq5mNhioT1g8PH7yLCpH98Th+F1VcTFhcTcFs5l5fR7nwYxHwQnxsNZud01PeM0C2CIgBAAAAAKicqEw4Kge2ms3tEl6VLy8XHa/Il1EawWymAl/127eaTfXfk3eU06l7daUaUOhHR+peJMVAJgiIAQAAAACohGA2U5XCUVtY1SMiqYbCqm3xQocKlfqxul3JhPO5Nxq5V1dRLmyenKhWEps+lDcaqW7FUVLcefaMjxaANBEQAwAAAABQWstrzUVlwnsNbaMweuF5Wd2u0FQu7N/ceKOREMKo16MWw7s/uKpEjh5Z1RSTFAMpICAGAAAAAKBsVqw1l34lr+pCwOp2habqfN2rK7Hn9HYhg1aFyZm8b4HqICAGAAAAAKAMtlhrLn2sblcsGfZ/COdz1aR49y4WAFYjIAYAAAAAoKgSXGsufaxul1tyOlUthvOwgtzCOnid01Oz0VhnyH8m9AAAFR1JREFUHTwAayIgBgAAAACgSPa91lz6WN0uJ1Qu7I1Gwe2tVqupXDg/b6pgNnOvr+PDIykGEkFADAAAAABA3mW11lz6WN0uffES3UIEr/EgO/MCZ6AECIgBAAAAAMipXK01lz5Wt9urhdYNRWzyu9wimfcGsAUCYgAAAAAAcqQQa82lj9XtkhLO595o5N/ceKORKGYuvCyeFKuPEKxms8p/L8BGCIgBAAAAAMhYodeaSx+r223Hvb6OcuFSpqgL2bd6jZ3T00Jn30AKCIgBAAAAAMhA+daaSx+r261joQ9D59mzkuXCy1RSrMJiUZYqaWB/CIgBAAAAAEhJddaaSx+r2y1QoXDF+/M+2Ge58+xZ1uMC8oWAGAAAAACA/ar4WnPpq/LqdnI6VZFocHur1Wqd09MqvOonqbeEe32tNovVbJqNhtVsZj0uIBcIiAEAAAAASB5rzeVERVa3C2Yz9/o6yoUJQB+zEKCrJsVleicAWyAgBgAAAAAgSe71te15rDWXQ8ur252/elWCX0379Wu1LJvqI0EuvA45nbpXV1ELjve/+AXl/KgsAmIAAAAAAJKkUsjKtr4thGh1u97ZWQliQff6WghBx5LtqA4w/Rcvsh4IkBkCYgAAAAAAAACoqE+zHgAAAAAAAAAAIBsExAAAAAAAAABQUQTEAAAAAAAAAFBRBMQAAAAAACQgCALXddvttu/7WY8FT5BSttvtVqvVarVc1816OFsqx6vIEBsQUAiIAQAAAADYVRiGjuNomqbretZjwRPCMLRtezAYDIfD4XAYhqHneVkPamPleBUZYgMCkU/u7++zHgMAAAAAACVh27ZpmqZpZj0QPEpKGYZh/HfUbrfPz88zHNIWyvEqMsQGBCKfZT0AAAAAAACA9BiGEf9WSlnEuu9yvIoMsQGBCC0mAAAAAABARYVh6Lpur9fLeiA7KceryBAbEBVHQAwAAAAAAKooDMN2u93pdDRNy3os2yvHq8gQGxAgIAYAAAAAAJWj1ijr9/sLrQaKpRyvIkNsQEDQgxgAAAAAAFRNEASO4/T7/UIXjZbjVWSIDQgoVBADAAAAAIAKkVJ2u914LOg4TrZD2kI5XkWG2IBA5JP7+/usxwAAAAAAQLGpNqZCiCAINE3TNE3X9cFgkPW48IBWqxWG4ULR6HA4zGo82ynHq8gQGxCIEBADAAAAAAAAQEXRYgIAAAAAAAAAKoqAGAAAAAAAAAAqioAYAAAAAAAAACrqs6wHAAAAAABASYTzuTca/ae//ut/8tOfWs2mfniY9YgAPCGcz/+v4fD3/8E/6J2dZT0WIBssUgcAAAAAwK680Uj9J4T485/97C9//WshhFGvW80mSXFu2RcXcjIZfv111gPZiT8et7799v5Xv8p6IIXkXl/bnvf3v/3t//jNb/Sjo8HLl2ajkfWggLRRQQwAAAAAwJa80cgfj73RKLy704+Oes+fd5490w8PVSmxf3Nje57teUa93jk9tZpN7eAg6yGjnILZjM8hNhLMZt23b/2bG/PkZPDyZTCb2Z7X+vZbq9nsWxYbE5VCBTEAAAAAAJuR06k3GrlXVyoXVmXCxvHx8r8M53P36sobjeRkIoSwmk3z5ISkOCfKVEE8/OYbSl/XZ19cOO/eabXa4OVLq9mMfu5cXjqXl0KIzulp/8WL7AYIpIqAGAAAAACAtahc2BuNgttbrVZTufCaqVwwm6n7khTnBwFxBfnjcfft2+D2tvf8ee/sbPkPMJzPbc9zr67oOIHqICAGAAAAAGAVle2619dRLmw2GvGqwy0ejaQ4DwiIKyVKfo16vW9ZqzeXPx7bnicnEzpOoAoIiAEAAAAAeMCDSW7n2bM9PX7n9HSX3BlbICCuDrUYndiwdwQdJ1ARBMQAAAAAAPwoWl/OG41EKhW+y50rSIrTUY6AWAjxyVdfERA/ZmExuk1rgek4gSr4LOsBAAAAAACQvYVcWF2EbjWbKVxabhwfG8fH/RcvoqTYvboiKQZ2Fy1Gd/7q1XZ/StrBgVrIzva81rff0nECpUQFMQAAAACg0rzRyB+PvdEovLsz6nW19Fy2ARA1xemggrjEnlyMbgt0nEBZERADAAAAAKpIhcIqF9aPjlQubBwfZz2u36GSYvfqKs+DLK4yBcS958+JLJWNFqPb+sHpOIEyocUEAAAAAKBCFopzO6eneY5co+4TqszZvbpy3r0jKQYeEy1Gt6fEnI4TKCUqiAEAAAAA5RfMZu71dQmaNsQbYpAU74gK4jLZcTG6LdBxAqVBQAwAAAAAKK1gNlP1wnIyEUJ0Tk8LmgsvW06KO8+eUcm4EQLi0ogWo1Plvak9Lx0nUA4ExAAAAACAsgnnc9W6V+XCVrNpnpxYzWYiC1XljUrAvdFICJGTRfaKojQB8c97PavZrGZAvI/F6LYYg+15cjKh4wQKioAYAAAAAFASKhf2b26itFS1GC5lLrxg+bWTFD+pNAFx67vvjHq9agHxXhej2wIdJ1BcBMQAAAAAgMKjijZCUrw+AuLiihajy1UaS8cJFBQBMQAAAACgqOjDu0KV66nXVKaAWD88HLx8mfVA0pD+YnSbouMECoeAGAAAAABQMHI6VS2Go1zYajaN4+Osx5VT4XzuXl1FK/WVuyPzRsoUEAshSvBCnpTVYnRboOMECoSAGAAAAABQJN5o1H79WqvVVC7MRdzrC2Yz1YhDTiZGvf7+F7/IekQZc6+vg9msBOGdfXEhhCjBC1nNvb7uvnmT4WJ0m4o6Tgy+/LLz7FnWwwEeRUAMAAAA4P9v7/5x3EaWAA5z/RYLPGxEA86NduYNe45AHYE6AnkEMtyQhE8gHkF9BHWy8U6H62wadmxgOjMW2AXmBQU3+CiLI2mkoTn8fcFAEv8VW00FNcUiMDPmzz9/8OLBH5z/8iV8/UrNNWbHff48u3k7x5ixNCSIAQAAAAAAAGChXk0dAAAAAAAAAABgGiSIAQAAAAAAAGChfp46AAAAAAAAHueca9s2hJAkSZ7nRVFMHdEsMYwXZ63tv9Vap2k6VTAhBOdcDCOEEIPpus57H0IoikJrfY2jd13nnLv4/mc6aWcaNpaJHsQAAAAAgB9dCGG9Xm+3W8l2tW2rlMrzfOq4ZoZhvDjvfdd1SZKEELz3Wussy7IsmySYruustXmeSzCSHW6apr9OXddXjfDi+5/ppJ1p2FgsEsQAAAAAgB+dcy6E0M86SfJlwpDmiGG8HmuttXaQjX1OxhjnXD+Auq6TJJl7gnimk3amYWOxaDEBAAAAAPjRDe5Yd84ppaYKZr4YxmdjjDHGSA2vUqqqqn7fibquY4WvUqqfw12v10oppZQxJkkSWXpMz4qu63a7Xf+Toiikv8ExDoXkvS/LUmstnxhjuq6Lbwdneo0K2ZlO2pmGjcUiQQwAAAAAmJMQQtd1E5ZqvgwM4/UYY6y1sVzUOVeWZb96tJ8vbtu267rYoHa73b5+/bppGsn2Oufqut5sNmeEcVJG8lBISqndbrdarWRRnud5nse3gzOVHhfXK0+e6aSdadhYFBLEAAAAAIDZkM6eR9ZU4hCG8aqMMf10sDQmNsbEAlt5fJxUlWqt9x9zF/PFWmvvfVwkD5obHO4iLR3GQzpkcKZFUUjh8zXMdNLONGwsDQliAAAAAMA8hBDqum6aZnD7Nk7CMF7ESEtZa22ssY1idjiEUJZlmqaSMXTOHf9FxMTxZZ0d0n4LiytNqplO2pmGjQUiQQwAAAAAmAHvfdu2FOI9EcMYyWPEpOHvGZuPtPfNsmzkcWRlWVZVFTOG8nS7MwIYkBLg877Ws0PaP9zxXY+PN9NJO9OwsUyvpg4AAAAAAIBHSBfXfqqlbdtpQ5ojhrFvvV6v1+uzk3dpmjrn4ltrbawRzvN8MLDe+9gaIk3Tfj3ppXoyNE1T13U/P9t1XV3Xx2x7Ukj9pUqpruviW+fcxVtMzHTSzjRsLNZPDw8PU8cAAAAAAMCY1Wq1Xx0pT/HC8RjGyDl3c3NTVdXZjw7rt2Xw3mdZ1u//0Lat9PNNvhXVxlyhPNgtNnPIsqzrujzPm6aRfrXOuaIoJLC6ruV5ccfEKU+0k1SvfNFxK2OMZHK997GVxGazkQhHQpLNJfUsRcpKqbZt+xEOJpW1tqqqmC5/oplO2pmGjcUiQQwAAAAAAJalbdu2be/u7p54+7+UBmutv7sfadSwv1QeBzeo270UaZ1xKKRDHg1pZLcyCGc36wAwORLEAAAAAABgWdbr9aDmFwAWiwQxAAAAAABYFufcNap3AWCOSBADAAAAAAAAwEK9mjoAAAAAAAAAAMA0SBADAAAAAGYjfP1qP36cOorZc58/u8+fp44COIH/8mWm1779+DF8/Tp1FMAYEsQAAAAAgNlwnz6tPnyYOorZq42pjZk6CuAE3R9/zPTaX3344D59mjoKYAwJYgAAAAAAAABYKBLEAAAAAAAsi377duoQgNNk799PHQLwYpEgBgAAAAAAAICFIkEMAAAAAAAAAAtFghgAAAAAgMWxf/01dQjAyezHj1OHALxAJIgBAAAAAAAAYKFIEAMAAAAAAADAQpEgBgAAAAAAAICFIkEMAAAAAMCyqDdvpg4BOA2TFrgeEsQAAAAAACwLuTbMDpMWuB4SxAAAAAAAAACwUCSIAQAAAABYIv/ly9QhAKdh0gLXQIIYAAAAAIAlIteG2WHSAtdAghgAAAAAAAAAFooEMQAAAAAAAAAsFAliAAAAAAAAAFgoEsQAAAAAACxL9v791CEAJ8t++23qEICX6T+///771DEAAAAAAHCs//7yC/nNp/v733+z9+/TX3+dOhDgWH//849++1a9eTN1IKfhWsOP76eHh4epYwAAAAAAAAAATIAWEwAAAAAAAACwUCSIAQAAAAAAAGChSBADAAAAAAAAwEL9PHUAAAAAAICXKYTgnJPXWus0TZ/nuF3Xee9DCEVRaK3H13TOPbraPu+99z5JEtnw0VPz3nddlyRJ0zQnHQh48fpXk1xK+59MxVorL9I0PfVXApgREsQAAAAAgKvw3kt6xTmnlNpsNs9z3KIokiSp6zqE8Oiax6w20HWdtTbP8xCCJKNvb2/HN1FKNU2zWq1OOhCwBNbaruuyLEuSRP5aa733zrmqquSTSXjvjTHJt/8Axf/xTJuzBq6BBDEAAAAA4Cq01lrrruu01pJneRmMMbvdTl7neX5zczNtPMCsFUXhvS+KQikVP5GU8YTZ4SRJlFJ5niff0tZJkjjn6rp+tv91Ac+GBDEAAAAA4Iq8903TSLGt1PbGz8uyTJJkt9sZYySDnGWZrLNer5VSSin5XCpw+4V7zjmp3k3TNE3T48v65FghhDRNJftz6unENFaSJGmabrfbwQpt28oN8nIKWZbFm9Ol+jiE0D8jY4wUJ263W/nEWtu2bZIkm81GDlfXtfTNSNNUtj1yJAdly1rr/rb90VBKVVUVh3H8KxjZUE4zll5WVWWM8d7HgRpfemjP46eJWZOMcH9mWmvjlBiZ/I/+UFyWtLxwzskVfSiwuq7lhdw8Ef9D1jSN1np8afLYxQVcxQMAAAAAANdxd3dXVZW8Lopif4Usy6qqappG3u52u7goTdPNZiOvb29v+5vf3t5mWXZ/f//dt6Kqqv7exHa77e9ns9lorfdXG1cURdM0d3d3+4vu7+/7kdzf3+d5Hvc/ckb7g7PdbuPKsqv4umma/iJxaCSzLOuvc3t72z/EYFTzPO/v81DA4xtut9v4pT88PFRV1V/50aXjIY1MGMxaf6Le3t7Gr/jhsck/clk93W63G8yx3W4XYxsJLF5rcYbf39/H2EaWPnoJANfwauoENQAAAADgxTLG9Kt0pa52QErk5HX/jnKtdSwO1Vr3t+26LhbbytI8z4/pYmGM6d8eXhTFGaV5klbuum69XpdlGR/EJ4H1y/2kQjZWHI+ckVJK+jWv12upHXbO9UcjTdMQgvRmHWzb38n+SMbzres6lih+dzS01lmW9YfxUMDjGw5KQauqGnx3I0sfDenQaWLu+pfw4G6D8ck/clldSexaPhKYNNiR1/IbmKZpXGFk6TGXAHBxtJgAAAAAAFyLdFSQ1yEEY0xM7UVntAiQthL9T/I8l7zquP3n0fUTpseL3VFDCOv1Omar5bFaZ+w/yzJrrVJKbmBP/r+XRQihLMt41vEO94HvjqTspCzLfk5KWGv3n5t3TNuNkzaU2+QP7Wqw9Jg901PiRcrzvK5refZj/wI/cvI/J5mxVwrs7KsSeAoSxAAAAACAq3DOFUXRT5iuVqv9BPEZ9vO8zrljaoH319nf1ThjjNY65jTTNNVax2pfpdSgSfGRtNaS4M6yzDknvZXj0rIsq6qK6SdrbUy7H0OeE7ifV82ybNBA+UjjGw4G2Xvfr6wcX3p2SJi72GnaWtsvDH/i5L+4+F+uKwXGJYBJ0GICAAAAAHAVg/vEk291sk/fc5Zl8UFPSZKEENq2PabITiklj4MTzrlT793e30TuLpfXRVH0A0u+PUzvmD2naWqtzfNcSin7iVRJQ8e3J8Usj/KLX8R6vY6L9suuB+naQ8Y3LIqiLEvJvIcQ6rruBz++9OyQ8ALkeS73HAyaq5w9+S+u67pY836lwLgEMImfHh4epo4BAAAAAPDS1HVtjFFKFUUhqdu6rq21IQR5LlPbttba/n3ZsUOu9G2QAmTpV1vXtaSbY/taSSRJpsY5F7c1xkhCVopwJc262WxiVW9d13IPewzVWltV1ZE3cUv+N+7BWlsURT8P3g9M0qAS8+CMVquVcy7P836P4BCCvH337t1ms4lpMmOMtTbeyZ5lWdd1eZ7LrkZG0nv/7t27frrNOXd/fx/ftm3rnBtEK51Vx7+CQxvGIe26TiklX/eg7/D40kN7HjlNvBg3Nzd5nvfvMxiZ/Mf8UDyF917+oSJHl9Rw3PNIYPLrF19ba7fbrVLq9evXTdN470eWym/jyMUFXAMJYgAAAADALIUQpF3vqU8qk4o8pdQZvSCOPLQUSmutL5XWkSMOihYv5exoj9ywLMv+c7eOWXrxAcR8XXXyP8WPeVUCZyBBDAAAAAAArkV6sx6q6BxfCgB4BjykDgAAAAAAXJLc+y+vtdaD/O/4UgDAM6OCGAAAAAAAAAAW6tXUAQAAAAAAAAAApkGCGAAAAAAAAAAWigQxAAAAAAAAACwUCWIAAAAAAAAAWKj/AW4wdAptCWIIAAAAAElFTkSuQmCC", + "text/plain": [ + "Tree('S', [Tree('2', [Tree('2', ['The']), Tree('2', ['Rock'])]), Tree('4', [Tree('3', [Tree('2', ['is']), Tree('4', [Tree('2', ['destined']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('2', ['to']), Tree('2', [Tree('2', ['be']), Tree('2', [Tree('2', ['the']), Tree('2', [Tree('2', ['21st']), Tree('2', [Tree('2', [Tree('2', ['Century']), Tree('2', [\"'s\"])]), Tree('2', [Tree('3', ['new']), Tree('2', [Tree('2', ['``']), Tree('2', ['Conan'])])])])])])])]), Tree('2', [\"''\"])]), Tree('2', ['and'])]), Tree('3', [Tree('2', ['that']), Tree('3', [Tree('2', ['he']), Tree('3', [Tree('2', [\"'s\"]), Tree('3', [Tree('2', ['going']), Tree('3', [Tree('2', ['to']), Tree('4', [Tree('3', [Tree('2', ['make']), Tree('3', [Tree('3', [Tree('2', ['a']), Tree('3', ['splash'])]), Tree('2', [Tree('2', ['even']), Tree('3', ['greater'])])])]), Tree('2', [Tree('2', ['than']), Tree('2', [Tree('2', [Tree('2', [Tree('2', [Tree('1', [Tree('2', ['Arnold']), Tree('2', ['Schwarzenegger'])]), Tree('2', [','])]), Tree('2', [Tree('2', ['Jean-Claud']), Tree('2', [Tree('2', ['Van']), Tree('2', ['Damme'])])])]), Tree('2', ['or'])]), Tree('2', [Tree('2', ['Steven']), Tree('2', ['Segal'])])])])])])])])])])])])]), Tree('2', ['.'])])])" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_tree_train[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "X_tree_dev, y_tree_dev = get_tree_dataset(sst.dev_reader)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Pure NumPy TreeNN implementation\n", + "\n", + "`TreeNN` is a pure NumPy implementation of this model. It should be regarded as a baseline for models of this form. The original SST paper includes evaluations of a wide range of these models." + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "tree_nn = TreeNN(\n", + " sst_train_vocab, \n", + " embed_dim=100, \n", + " max_iter=100,\n", + " eta=0.05) " + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Finished epoch 100 of 100; error is 2.416002630512955" + ] + } + ], + "source": [ + "tree_nn.fit(X_tree_train, y_tree_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "tree_dev_predictions = tree_nn.predict(X_tree_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " negative 0.51 0.56 0.53 428\n", + " positive 0.53 0.47 0.50 444\n", + "\n", + "avg / total 0.52 0.52 0.52 872\n", + "\n" + ] + } + ], + "source": [ + "print(classification_report(y_tree_dev, tree_dev_predictions))" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "tree_nn_glove = TreeNN(\n", + " sst_glove_vocab,\n", + " embedding=glove_embedding,\n", + " embed_dim=None, # Ignored when embedding is not `None`\n", + " max_iter=100,\n", + " eta=0.05) " + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Finished epoch 100 of 100; error is 1.337790645988493" + ] + } + ], + "source": [ + "tree_nn_glove.fit(X_tree_train, y_tree_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "tree_glove_dev_predictions = tree_nn_glove.predict(X_tree_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " negative 0.54 0.35 0.43 428\n", + " positive 0.53 0.71 0.61 444\n", + "\n", + "avg / total 0.54 0.54 0.52 872\n", + "\n" + ] + } + ], + "source": [ + "print(classification_report(y_tree_dev, tree_glove_dev_predictions))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exploratory exercises\n", + "\n", + "These are largely meant to give you a feel for the material, but some of them could lead to projects and help you with future work for the course (especially the SST bake-off). These are not for credit.\n", + "\n", + "1. In the [distributed representations as features](#Distributed-representations-as-features) section, we just summed all of the leaf-node GloVe vectors to obtain a fixed-dimensional representation for all sentences. This ignores all of the tree structure. See if you can do better by paying attention to the binary tree structure: write a function `glove_subtree_phi` that obtains a vector representation for each subtree by combining the vectors of its daughters, with the leaf nodes again given by GloVe (any dimension you like) and the full representation of the sentence given by the final vector obtained by this recursive process. You can decide on how you combine the vectors. \n", + "\n", + "1. The default hidden activation function for `TfRNNClassifier` is `tf.nn.tanh`. This value is exposed via the `hidden_activation` keyword argument. See if you can get better results with [a different activation function](https://www.tensorflow.org/api_guides/python/nn). If you're feeling really ambitious, you could subclass `TfRNNClassifier` to define a new cost function (maybe with regularization) or optimizer. Deep learning offers a lot of design choices!\n", + "\n", + "1. Most of the experiments above used random initial representations of the words. Try other initialization schemes by importing or rebuilding vectors from the VSM unit.\n", + "\n", + "1. Tree-structured networks can be defined relatively easily in TensorFlow. [This excellent repository](https://github.com/erickrf/treernn) provides working code and a clear explanation. Get to know this code, and consider adapting it for use in SST experiments.\n", + "\n", + "1. The final hidden state in the RNN Classifier is the basis for the classification decisions. It should also be a good embedding for the entire example. Is it? The function `sst.get_sentence_embedding_from_rnn` extracts this space from a trained model given a set of examples. Study this space to see what insights it can provide about what the model has learned." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "widgets": { + "state": {}, + "version": "1.1.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +}