DiscriminativeClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from preprocess import Parser\n",
    "from classifier import DiscriminativeClassifier, BinaryGenerativeClassifier\n",
    "from evaluation import calc_sd_auc, correlate_AUCs, calc_z_diff\n",
    "import pandas as pd\n",
    "from io import StringIO\n",
    "from nltk.collocations import BigramCollocationFinder, TrigramCollocationFinder\n",
    "from nltk import word_tokenize\n",
    "import nltk\n",
    "import re\n",
    "import operator\n",
    "from topicmodel import LDA\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegressionCV, LogisticRegression\n",
    "from sklearn.metrics import roc_auc_score, auc, accuracy_score, confusion_matrix, roc_curve\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import matplotlib.dates as mdates\n",
    "\n",
    "\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "\n",
    "import h2o\n",
    "from h2o.estimators import H2OGeneralizedLinearEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cfnai = pd.read_excel('cfnai-data-series-xlsx.xlsx')\n",
    "cfnai = cfnai[106:586].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meeting</th>\n",
       "      <th>D_NBER</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>201508</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>201509</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-09-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>201510</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-10-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>201511</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>201512</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-12-16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     meeting  D_NBER       date\n",
       "475   201508       0        NaT\n",
       "476   201509       0 2015-09-17\n",
       "477   201510       0 2015-10-28\n",
       "478   201511       0        NaT\n",
       "479   201512       0 2015-12-16"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nber = pd.read_csv('nber.csv', parse_dates=['date'])\n",
    "nber.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "theta_train = pd.read_csv('theta_train.csv')\n",
    "theta_test = pd.read_csv('theta_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "theta_train_gen = pd.read_csv('theta_train_gen.csv').iloc[:,1:]\n",
    "theta_test_gen = pd.read_csv('theta_test_gen.csv').iloc[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>40</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000076</td>\n",
       "      <td>0.000092</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000071</td>\n",
       "      <td>0.000087</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000201</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.000071</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>0.070641</td>\n",
       "      <td>0.000070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.000071</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000163</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000066</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>0.000057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000242</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.000084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000242</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.000084</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  0.000076  0.000092  0.000075  0.000077  0.000074  0.000075  0.000074   \n",
       "1  0.000071  0.000087  0.000070  0.000073  0.000070  0.000070  0.000070   \n",
       "2  0.000058  0.000071  0.000057  0.000059  0.000057  0.000057  0.000057   \n",
       "3  0.000086  0.000105  0.000085  0.000088  0.000084  0.000085  0.000084   \n",
       "4  0.000086  0.000105  0.000085  0.000088  0.000084  0.000085  0.000084   \n",
       "\n",
       "          7         8         9    ...           40        41        42  \\\n",
       "0  0.000074  0.000074  0.000075    ...     0.000074  0.000213  0.000074   \n",
       "1  0.000070  0.000070  0.000070    ...     0.000070  0.000201  0.000070   \n",
       "2  0.000057  0.000057  0.000057    ...     0.000057  0.000163  0.000057   \n",
       "3  0.000084  0.000084  0.000085    ...     0.000084  0.000242  0.000084   \n",
       "4  0.000084  0.000084  0.000085    ...     0.000084  0.000242  0.000084   \n",
       "\n",
       "         43        44        45        46        47        48        49  \n",
       "0  0.000074  0.000086  0.000075  0.000075  0.000074  0.000091  0.000074  \n",
       "1  0.000070  0.000081  0.000070  0.000071  0.000070  0.070641  0.000070  \n",
       "2  0.000057  0.000066  0.000057  0.000058  0.000057  0.000069  0.000057  \n",
       "3  0.000084  0.000098  0.000085  0.000085  0.000084  0.000103  0.000084  \n",
       "4  0.000084  0.000098  0.000085  0.000085  0.000084  0.000103  0.000084  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_train_gen.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meeting</th>\n",
       "      <th>D_NBER</th>\n",
       "      <th>date</th>\n",
       "      <th>index</th>\n",
       "      <th>Date</th>\n",
       "      <th>P_I</th>\n",
       "      <th>EU_H</th>\n",
       "      <th>C_H</th>\n",
       "      <th>SO_I</th>\n",
       "      <th>CFNAI</th>\n",
       "      <th>CFNAI_MA3</th>\n",
       "      <th>DIFFUSION</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>201508</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "      <td>581</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.07</td>\n",
       "      <td>-0.09</td>\n",
       "      <td>-0.08</td>\n",
       "      <td>-0.33</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>201509</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>582</td>\n",
       "      <td>2015-09-30</td>\n",
       "      <td>-0.27</td>\n",
       "      <td>-0.09</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>0.15</td>\n",
       "      <td>-0.27</td>\n",
       "      <td>-0.09</td>\n",
       "      <td>-0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>201510</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-10-28</td>\n",
       "      <td>583</td>\n",
       "      <td>2015-10-31</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>0.12</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>-0.07</td>\n",
       "      <td>-0.17</td>\n",
       "      <td>-0.26</td>\n",
       "      <td>-0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>201511</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "      <td>584</td>\n",
       "      <td>2015-11-30</td>\n",
       "      <td>-0.29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>-0.08</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.27</td>\n",
       "      <td>-0.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>201512</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-12-16</td>\n",
       "      <td>585</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>-0.33</td>\n",
       "      <td>0.11</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.31</td>\n",
       "      <td>-0.28</td>\n",
       "      <td>-0.22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     meeting  D_NBER       date  index       Date   P_I  EU_H   C_H  SO_I  \\\n",
       "475   201508       0        NaT    581 2015-08-31 -0.10 -0.07 -0.09 -0.08   \n",
       "476   201509       0 2015-09-17    582 2015-09-30 -0.27 -0.09 -0.06  0.15   \n",
       "477   201510       0 2015-10-28    583 2015-10-31 -0.10  0.12 -0.11 -0.07   \n",
       "478   201511       0        NaT    584 2015-11-30 -0.29  0.07 -0.06 -0.08   \n",
       "479   201512       0 2015-12-16    585 2015-12-31 -0.33  0.11 -0.04 -0.04   \n",
       "\n",
       "     CFNAI  CFNAI_MA3  DIFFUSION  \n",
       "475  -0.33      -0.04      -0.07  \n",
       "476  -0.27      -0.09      -0.12  \n",
       "477  -0.17      -0.26      -0.28  \n",
       "478  -0.37      -0.27      -0.27  \n",
       "479  -0.31      -0.28      -0.22  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "macro_data = pd.concat([nber, cfnai], axis=1)\n",
    "macro_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>meeting</th>\n",
       "      <th>D_NBER</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197601</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197602</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197603</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>With Mr. Coldwell, dissenting, the Federal Re...</td>\n",
       "      <td>197605</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  meeting  D_NBER\n",
       "0   By unanimous vote, the Federal Reserve Bank o...   197601       0\n",
       "1   By unanimous vote, the Federal Reserve Bank o...   197602       0\n",
       "2   By unanimous vote, the Federal Reserve Bank o...   197603       0\n",
       "3   By unanimous vote, the Federal Reserve Bank o...   197604       0\n",
       "4   With Mr. Coldwell, dissenting, the Federal Re...   197605       0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_table(StringIO(''.join(l.replace('\\u2028', ' ') for l in open('minutes_data.txt'))))\n",
    "grouped_data = data.groupby('meeting', as_index=False)\n",
    "full_minutes = pd.concat([grouped_data['text'].apply(' '.join), grouped_data.first()], axis=1)\n",
    "full_minutes.drop(['text', 'seq'], axis=1, inplace=True)\n",
    "full_minutes.rename(columns={0: 'text'}, inplace=True)\n",
    "full_minutes.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>meeting</th>\n",
       "      <th>D_NBER</th>\n",
       "      <th>index</th>\n",
       "      <th>Date</th>\n",
       "      <th>P_I</th>\n",
       "      <th>EU_H</th>\n",
       "      <th>C_H</th>\n",
       "      <th>SO_I</th>\n",
       "      <th>CFNAI</th>\n",
       "      <th>CFNAI_MA3</th>\n",
       "      <th>DIFFUSION</th>\n",
       "      <th>D_NBER_1</th>\n",
       "      <th>D_NBER_3</th>\n",
       "      <th>D_NBER_6</th>\n",
       "      <th>D_NBER_12</th>\n",
       "      <th>D_NBER_24</th>\n",
       "      <th>meeting_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197601</td>\n",
       "      <td>0</td>\n",
       "      <td>106</td>\n",
       "      <td>1976-01-31</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.44</td>\n",
       "      <td>2.23</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1976-01-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197602</td>\n",
       "      <td>0</td>\n",
       "      <td>107</td>\n",
       "      <td>1976-02-29</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.62</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>0.19</td>\n",
       "      <td>1.34</td>\n",
       "      <td>1.61</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1976-02-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197603</td>\n",
       "      <td>0</td>\n",
       "      <td>108</td>\n",
       "      <td>1976-03-31</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.59</td>\n",
       "      <td>1.39</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1976-03-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>By unanimous vote, the Federal Reserve Bank o...</td>\n",
       "      <td>197604</td>\n",
       "      <td>0</td>\n",
       "      <td>109</td>\n",
       "      <td>1976-04-30</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1976-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>With Mr. Coldwell, dissenting, the Federal Re...</td>\n",
       "      <td>197605</td>\n",
       "      <td>0</td>\n",
       "      <td>110</td>\n",
       "      <td>1976-05-31</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.15</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1976-05-18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text meeting  D_NBER  index  \\\n",
       "0   By unanimous vote, the Federal Reserve Bank o...  197601       0    106   \n",
       "1   By unanimous vote, the Federal Reserve Bank o...  197602       0    107   \n",
       "2   By unanimous vote, the Federal Reserve Bank o...  197603       0    108   \n",
       "3   By unanimous vote, the Federal Reserve Bank o...  197604       0    109   \n",
       "4   With Mr. Coldwell, dissenting, the Federal Re...  197605       0    110   \n",
       "\n",
       "        Date   P_I  EU_H   C_H  SO_I  CFNAI  CFNAI_MA3  DIFFUSION  D_NBER_1  \\\n",
       "0 1976-01-31  0.72  0.95  0.12  0.44   2.23       1.23       0.62       0.0   \n",
       "1 1976-02-29  0.55  0.62 -0.02  0.19   1.34       1.61       0.77       0.0   \n",
       "2 1976-03-31  0.02  0.36  0.04  0.17   0.59       1.39       0.66       0.0   \n",
       "3 1976-04-30  0.27  0.23  0.08  0.07   0.64       0.86       0.53       0.0   \n",
       "4 1976-05-31  0.24  0.15 -0.10  0.05   0.34       0.52       0.37       0.0   \n",
       "\n",
       "   D_NBER_3  D_NBER_6  D_NBER_12  D_NBER_24 meeting_date  \n",
       "0       0.0       0.0        0.0        0.0   1976-01-20  \n",
       "1       0.0       0.0        0.0        0.0   1976-02-18  \n",
       "2       0.0       0.0        0.0        0.0   1976-03-16  \n",
       "3       0.0       0.0        0.0        0.0   1976-04-20  \n",
       "4       0.0       0.0        0.0        0.0   1976-05-18  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nber['meeting'] = nber['meeting'].astype(str)\n",
    "full_minutes['meeting'] = full_minutes['meeting'].astype(str)\n",
    "\n",
    "cfnai_nber = pd.concat([cfnai, nber['meeting'], nber['D_NBER'].shift(-1), \n",
    "                       nber['D_NBER'].shift(-3), nber['D_NBER'].shift(-6),\n",
    "                        nber['D_NBER'].shift(-12), nber['D_NBER'].shift(-24),\n",
    "                       nber['date']], \n",
    "                      axis=1)\n",
    "cfnai_nber.columns = list(cfnai.columns) + ['meeting', 'D_NBER_1', 'D_NBER_3', 'D_NBER_6',\n",
    "                                            'D_NBER_12', 'D_NBER_24',\n",
    "                                            'meeting_date']\n",
    "\n",
    "macro_data = pd.merge(full_minutes, cfnai_nber, how='inner', on='meeting')\n",
    "\n",
    "macro_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = macro_data[:200]\n",
    "test = macro_data[200:].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_features = pd.concat([train[['P_I', 'EU_H', 'C_H', 'SO_I',\n",
    "#                        'CFNAI', 'CFNAI_MA3', 'DIFFUSION']],\n",
    "#                             theta_train], axis=1)\n",
    "\n",
    "# test_features = pd.concat([test[['P_I', 'EU_H', 'C_H', 'SO_I',\n",
    "#                        'CFNAI', 'CFNAI_MA3', 'DIFFUSION']],\n",
    "#                             theta_test], axis=1)\n",
    "\n",
    "train_features = pd.concat([train[['CFNAI_MA3']],\n",
    "                            theta_train], axis=1)\n",
    "\n",
    "test_features = pd.concat([test[['CFNAI_MA3']],\n",
    "                            theta_test], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.to_csv('train.csv', index=False)\n",
    "test.to_csv('test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Peak</th>\n",
       "      <th>Trough</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1980-01-01</td>\n",
       "      <td>1980-07-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1981-07-01</td>\n",
       "      <td>1982-11-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1990-07-01</td>\n",
       "      <td>1991-03-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2001-03-01</td>\n",
       "      <td>2001-11-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>2009-06-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Peak     Trough\n",
       "1 1980-01-01 1980-07-01\n",
       "2 1981-07-01 1982-11-01\n",
       "3 1990-07-01 1991-03-01\n",
       "4 2001-03-01 2001-11-01\n",
       "5 2007-12-01 2009-06-01"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec = pd.read_csv('recs.csv', parse_dates=['Peak', 'Trough'])[1:]\n",
    "rec.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321. connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>23 hours 43 mins</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.10.4.4</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>21 days, 6 hours and 55 minutes </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_dkn22_618jik</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>3.249 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.0 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  -------------------------------\n",
       "H2O cluster uptime:         23 hours 43 mins\n",
       "H2O cluster version:        3.10.4.4\n",
       "H2O cluster version age:    21 days, 6 hours and 55 minutes\n",
       "H2O cluster name:           H2O_from_python_dkn22_618jik\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    3.249 Gb\n",
       "H2O cluster total cores:    8\n",
       "H2O cluster allowed cores:  8\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "Python version:             3.6.0 final\n",
       "--------------------------  -------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.init()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.concat([theta_train, train], axis=1).to_csv('train_df.csv', index=False)\n",
    "pd.concat([theta_test, test], axis=1).to_csv('test_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "train_df = h2o.import_file('train_df.csv', header=1)\n",
    "test_df = h2o.import_file('test_df.csv', header=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.860656\n",
       "1    0.139344\n",
       "Name: D_NBER, dtype: float64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['D_NBER'].value_counts() / test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "predictions_h2o = logistic_h2o.predict(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_predictions_topics = []\n",
    "all_predictions_topics_CFNAI = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.12165448260895254\n",
      "RMSE: 0.34879002653308844\n",
      "LogLoss: 0.43682285659250886\n",
      "Null degrees of freedom: 121\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 99.51944846220036\n",
      "Residual deviance: 106.58477700857216\n",
      "AIC: 208.58477700857216\n",
      "AUC: 0.8717086834733894\n",
      "Gini: 0.7434173669467787\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.028150852431434385: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>90.0</td>\n",
       "<td>15.0</td>\n",
       "<td>0.1429</td>\n",
       "<td> (15.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>3.0</td>\n",
       "<td>14.0</td>\n",
       "<td>0.1765</td>\n",
       "<td> (3.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>93.0</td>\n",
       "<td>29.0</td>\n",
       "<td>0.1475</td>\n",
       "<td> (18.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      90   15   0.1429   (15.0/105.0)\n",
       "1      3    14   0.1765   (3.0/17.0)\n",
       "Total  93   29   0.1475   (18.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0281509</td>\n",
       "<td>0.6086957</td>\n",
       "<td>28.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0220170</td>\n",
       "<td>0.7352941</td>\n",
       "<td>33.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0765551</td>\n",
       "<td>0.6140351</td>\n",
       "<td>9.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.1142612</td>\n",
       "<td>0.8934426</td>\n",
       "<td>5.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.1885177</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0066876</td>\n",
       "<td>1.0</td>\n",
       "<td>78.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.1885177</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0281509</td>\n",
       "<td>0.5537538</td>\n",
       "<td>28.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0281509</td>\n",
       "<td>0.8235294</td>\n",
       "<td>28.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0220170</td>\n",
       "<td>0.8507003</td>\n",
       "<td>33.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.0281509    0.608696  28\n",
       "max f2                       0.022017     0.735294  33\n",
       "max f0point5                 0.0765551    0.614035  9\n",
       "max accuracy                 0.114261     0.893443  5\n",
       "max precision                0.188518     1         0\n",
       "max recall                   0.00668763   1         78\n",
       "max specificity              0.188518     1         0\n",
       "max absolute_mcc             0.0281509    0.553754  28\n",
       "max min_per_class_accuracy   0.0281509    0.823529  28\n",
       "max mean_per_class_accuracy  0.022017     0.8507    33"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.1453500</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.5882353</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>258.8235294</td>\n",
       "<td>258.8235294</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.1392682</td>\n",
       "<td>7.1764706</td>\n",
       "<td>4.7843137</td>\n",
       "<td>1.0</td>\n",
       "<td>0.6666667</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>378.4313725</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.1282271</td>\n",
       "<td>7.1764706</td>\n",
       "<td>5.3823529</td>\n",
       "<td>1.0</td>\n",
       "<td>0.75</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>617.6470588</td>\n",
       "<td>438.2352941</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.1157765</td>\n",
       "<td>7.1764706</td>\n",
       "<td>5.7411765</td>\n",
       "<td>1.0</td>\n",
       "<td>0.8</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.2352941</td>\n",
       "<td>617.6470588</td>\n",
       "<td>474.1176471</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.1101569</td>\n",
       "<td>3.5882353</td>\n",
       "<td>5.1260504</td>\n",
       "<td>0.5</td>\n",
       "<td>0.7142857</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.2941176</td>\n",
       "<td>258.8235294</td>\n",
       "<td>412.6050420</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.0705443</td>\n",
       "<td>2.3921569</td>\n",
       "<td>3.8642534</td>\n",
       "<td>0.3333333</td>\n",
       "<td>0.5384615</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.4117647</td>\n",
       "<td>139.2156863</td>\n",
       "<td>286.4253394</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.0402725</td>\n",
       "<td>2.3921569</td>\n",
       "<td>3.3993808</td>\n",
       "<td>0.3333333</td>\n",
       "<td>0.4736842</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.5294118</td>\n",
       "<td>139.2156863</td>\n",
       "<td>239.9380805</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.0344458</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.4447059</td>\n",
       "<td>0.5</td>\n",
       "<td>0.48</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.7058824</td>\n",
       "<td>258.8235294</td>\n",
       "<td>244.4705882</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.0209657</td>\n",
       "<td>1.7941176</td>\n",
       "<td>2.9093800</td>\n",
       "<td>0.25</td>\n",
       "<td>0.4054054</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.8823529</td>\n",
       "<td>79.4117647</td>\n",
       "<td>190.9379968</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.0149995</td>\n",
       "<td>0.0</td>\n",
       "<td>2.1968788</td>\n",
       "<td>0.0</td>\n",
       "<td>0.3061224</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>119.6878752</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0104333</td>\n",
       "<td>0.0</td>\n",
       "<td>1.7647059</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2459016</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>76.4705882</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.0071735</td>\n",
       "<td>0.0</td>\n",
       "<td>1.4746172</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2054795</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>47.4617244</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.0056425</td>\n",
       "<td>1.1960784</td>\n",
       "<td>1.4352941</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.2</td>\n",
       "<td>0.1176471</td>\n",
       "<td>1.0</td>\n",
       "<td>19.6078431</td>\n",
       "<td>43.5294118</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.0034034</td>\n",
       "<td>0.0</td>\n",
       "<td>1.2577320</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1752577</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>25.7731959</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.0018885</td>\n",
       "<td>0.0</td>\n",
       "<td>1.1192661</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1559633</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>11.9266055</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0002580</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift     cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain     cumulative_gain\n",
       "--  -------  --------------------------  -----------------  -------  -----------------  ---------------  --------------------------  --------------  -------------------------  -------  -----------------\n",
       "    1        0.0163934                   0.14535            3.58824  3.58824            0.5              0.5                         0.0588235       0.0588235                  258.824  258.824\n",
       "    2        0.0245902                   0.139268           7.17647  4.78431            1                0.666667                    0.0588235       0.117647                   617.647  378.431\n",
       "    3        0.0327869                   0.128227           7.17647  5.38235            1                0.75                        0.0588235       0.176471                   617.647  438.235\n",
       "    4        0.0409836                   0.115776           7.17647  5.74118            1                0.8                         0.0588235       0.235294                   617.647  474.118\n",
       "    5        0.057377                    0.110157           3.58824  5.12605            0.5              0.714286                    0.0588235       0.294118                   258.824  412.605\n",
       "    6        0.106557                    0.0705443          2.39216  3.86425            0.333333         0.538462                    0.117647        0.411765                   139.216  286.425\n",
       "    7        0.155738                    0.0402725          2.39216  3.39938            0.333333         0.473684                    0.117647        0.529412                   139.216  239.938\n",
       "    8        0.204918                    0.0344458          3.58824  3.44471            0.5              0.48                        0.176471        0.705882                   258.824  244.471\n",
       "    9        0.303279                    0.0209657          1.79412  2.90938            0.25             0.405405                    0.176471        0.882353                   79.4118  190.938\n",
       "    10       0.401639                    0.0149995          0        2.19688            0                0.306122                    0               0.882353                   -100     119.688\n",
       "    11       0.5                         0.0104333          0        1.76471            0                0.245902                    0               0.882353                   -100     76.4706\n",
       "    12       0.598361                    0.00717349         0        1.47462            0                0.205479                    0               0.882353                   -100     47.4617\n",
       "    13       0.696721                    0.00564253         1.19608  1.43529            0.166667         0.2                         0.117647        1                          19.6078  43.5294\n",
       "    14       0.795082                    0.00340337         0        1.25773            0                0.175258                    0               1                          -100     25.7732\n",
       "    15       0.893443                    0.00188846         0        1.11927            0                0.155963                    0               1                          -100     11.9266\n",
       "    16       1                           0.000258044        0        1                  0                0.139344                    0               1                          -100     0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.1346095656076651\n",
      "RMSE: 0.3668917627961482\n",
      "LogLoss: 0.6301223109297249\n",
      "Null degrees of freedom: 121\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 99.51944846220036\n",
      "Residual deviance: 153.74984386685298\n",
      "AIC: 255.74984386685298\n",
      "AUC: 0.8039215686274509\n",
      "Gini: 0.6078431372549018\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.008493366752675648: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>90.0</td>\n",
       "<td>15.0</td>\n",
       "<td>0.1429</td>\n",
       "<td> (15.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>5.0</td>\n",
       "<td>12.0</td>\n",
       "<td>0.2941</td>\n",
       "<td> (5.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>95.0</td>\n",
       "<td>27.0</td>\n",
       "<td>0.1639</td>\n",
       "<td> (20.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      90   15   0.1429   (15.0/105.0)\n",
       "1      5    12   0.2941   (5.0/17.0)\n",
       "Total  95   27   0.1639   (20.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0084934</td>\n",
       "<td>0.5454545</td>\n",
       "<td>26.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0084934</td>\n",
       "<td>0.6315789</td>\n",
       "<td>26.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0145206</td>\n",
       "<td>0.5294118</td>\n",
       "<td>16.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0697301</td>\n",
       "<td>0.8688525</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.0697301</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0012472</td>\n",
       "<td>1.0</td>\n",
       "<td>99.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.0697301</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0084934</td>\n",
       "<td>0.4696813</td>\n",
       "<td>26.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0084934</td>\n",
       "<td>0.7058824</td>\n",
       "<td>26.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0084934</td>\n",
       "<td>0.7815126</td>\n",
       "<td>26.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.00849337   0.545455  26\n",
       "max f2                       0.00849337   0.631579  26\n",
       "max f0point5                 0.0145206    0.529412  16\n",
       "max accuracy                 0.0697301    0.868852  0\n",
       "max precision                0.0697301    1         0\n",
       "max recall                   0.00124717   1         99\n",
       "max specificity              0.0697301    1         0\n",
       "max absolute_mcc             0.00849337   0.469681  26\n",
       "max min_per_class_accuracy   0.00849337   0.705882  26\n",
       "max mean_per_class_accuracy  0.00849337   0.781513  26"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.0509028</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.5882353</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>258.8235294</td>\n",
       "<td>258.8235294</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.0430494</td>\n",
       "<td>0.0</td>\n",
       "<td>2.3921569</td>\n",
       "<td>0.0</td>\n",
       "<td>0.3333333</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>-100.0</td>\n",
       "<td>139.2156863</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.0356540</td>\n",
       "<td>0.0</td>\n",
       "<td>1.7941176</td>\n",
       "<td>0.0</td>\n",
       "<td>0.25</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>-100.0</td>\n",
       "<td>79.4117647</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.0307715</td>\n",
       "<td>7.1764706</td>\n",
       "<td>2.8705882</td>\n",
       "<td>1.0</td>\n",
       "<td>0.4</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>187.0588235</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.0301962</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.0756303</td>\n",
       "<td>0.5</td>\n",
       "<td>0.4285714</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>258.8235294</td>\n",
       "<td>207.5630252</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.0207137</td>\n",
       "<td>2.3921569</td>\n",
       "<td>2.7601810</td>\n",
       "<td>0.3333333</td>\n",
       "<td>0.3846154</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.2941176</td>\n",
       "<td>139.2156863</td>\n",
       "<td>176.0180995</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.0135141</td>\n",
       "<td>4.7843137</td>\n",
       "<td>3.3993808</td>\n",
       "<td>0.6666667</td>\n",
       "<td>0.4736842</td>\n",
       "<td>0.2352941</td>\n",
       "<td>0.5294118</td>\n",
       "<td>378.4313725</td>\n",
       "<td>239.9380805</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.0107260</td>\n",
       "<td>1.1960784</td>\n",
       "<td>2.8705882</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.4</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.5882353</td>\n",
       "<td>19.6078431</td>\n",
       "<td>187.0588235</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.0061479</td>\n",
       "<td>1.1960784</td>\n",
       "<td>2.3275040</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.3243243</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.7058824</td>\n",
       "<td>19.6078431</td>\n",
       "<td>132.7503975</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.0049585</td>\n",
       "<td>0.0</td>\n",
       "<td>1.7575030</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2448980</td>\n",
       "<td>0.0</td>\n",
       "<td>0.7058824</td>\n",
       "<td>-100.0</td>\n",
       "<td>75.7503001</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0033637</td>\n",
       "<td>1.7941176</td>\n",
       "<td>1.7647059</td>\n",
       "<td>0.25</td>\n",
       "<td>0.2459016</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.8823529</td>\n",
       "<td>79.4117647</td>\n",
       "<td>76.4705882</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.0023626</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.5729251</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2191781</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>57.2925060</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.0017881</td>\n",
       "<td>0.0</td>\n",
       "<td>1.3508651</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1882353</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>35.0865052</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.0013376</td>\n",
       "<td>0.0</td>\n",
       "<td>1.1837477</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1649485</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>18.3747726</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.0007254</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.1192661</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1559633</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>11.9266055</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0001024</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.0509028          3.58824   3.58824            0.5              0.5                         0.0588235       0.0588235                  258.824   258.824\n",
       "    2        0.0245902                   0.0430494          0         2.39216            0                0.333333                    0               0.0588235                  -100      139.216\n",
       "    3        0.0327869                   0.035654           0         1.79412            0                0.25                        0               0.0588235                  -100      79.4118\n",
       "    4        0.0409836                   0.0307715          7.17647   2.87059            1                0.4                         0.0588235       0.117647                   617.647   187.059\n",
       "    5        0.057377                    0.0301962          3.58824   3.07563            0.5              0.428571                    0.0588235       0.176471                   258.824   207.563\n",
       "    6        0.106557                    0.0207137          2.39216   2.76018            0.333333         0.384615                    0.117647        0.294118                   139.216   176.018\n",
       "    7        0.155738                    0.0135141          4.78431   3.39938            0.666667         0.473684                    0.235294        0.529412                   378.431   239.938\n",
       "    8        0.204918                    0.010726           1.19608   2.87059            0.166667         0.4                         0.0588235       0.588235                   19.6078   187.059\n",
       "    9        0.303279                    0.00614789         1.19608   2.3275             0.166667         0.324324                    0.117647        0.705882                   19.6078   132.75\n",
       "    10       0.401639                    0.00495847         0         1.7575             0                0.244898                    0               0.705882                   -100      75.7503\n",
       "    11       0.5                         0.00336369         1.79412   1.76471            0.25             0.245902                    0.176471        0.882353                   79.4118   76.4706\n",
       "    12       0.598361                    0.00236261         0.598039  1.57293            0.0833333        0.219178                    0.0588235       0.941176                   -40.1961  57.2925\n",
       "    13       0.696721                    0.00178812         0         1.35087            0                0.188235                    0               0.941176                   -100      35.0865\n",
       "    14       0.795082                    0.00133759         0         1.18375            0                0.164948                    0               0.941176                   -100      18.3748\n",
       "    15       0.893443                    0.000725409        0.598039  1.11927            0.0833333        0.155963                    0.0588235       1                          -40.1961  11.9266\n",
       "    16       1                           0.000102401        0         1                  0                0.139344                    0               1                          -100      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.13177394708943704\n",
      "RMSE: 0.36300681410882224\n",
      "LogLoss: 0.5532316900479544\n",
      "Null degrees of freedom: 121\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 99.51944846220036\n",
      "Residual deviance: 134.9885323717009\n",
      "AIC: 236.9885323717009\n",
      "AUC: 0.8005602240896359\n",
      "Gini: 0.6011204481792718\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.019851951614165215: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>96.0</td>\n",
       "<td>9.0</td>\n",
       "<td>0.0857</td>\n",
       "<td> (9.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>6.0</td>\n",
       "<td>11.0</td>\n",
       "<td>0.3529</td>\n",
       "<td> (6.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>102.0</td>\n",
       "<td>20.0</td>\n",
       "<td>0.123</td>\n",
       "<td> (15.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      96   9    0.0857   (9.0/105.0)\n",
       "1      6    11   0.3529   (6.0/17.0)\n",
       "Total  102  20   0.123    (15.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0198520</td>\n",
       "<td>0.5945946</td>\n",
       "<td>19.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0182599</td>\n",
       "<td>0.6382979</td>\n",
       "<td>25.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0198520</td>\n",
       "<td>0.5670103</td>\n",
       "<td>19.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0198520</td>\n",
       "<td>0.8770492</td>\n",
       "<td>19.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.0476399</td>\n",
       "<td>0.5714286</td>\n",
       "<td>6.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0023205</td>\n",
       "<td>1.0</td>\n",
       "<td>103.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.0921451</td>\n",
       "<td>0.9904762</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0198520</td>\n",
       "<td>0.5250897</td>\n",
       "<td>19.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0115142</td>\n",
       "<td>0.7523810</td>\n",
       "<td>38.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0182599</td>\n",
       "<td>0.7862745</td>\n",
       "<td>25.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.019852     0.594595  19\n",
       "max f2                       0.0182599    0.638298  25\n",
       "max f0point5                 0.019852     0.56701   19\n",
       "max accuracy                 0.019852     0.877049  19\n",
       "max precision                0.0476399    0.571429  6\n",
       "max recall                   0.00232049   1         103\n",
       "max specificity              0.0921451    0.990476  0\n",
       "max absolute_mcc             0.019852     0.52509   19\n",
       "max min_per_class_accuracy   0.0115142    0.752381  38\n",
       "max mean_per_class_accuracy  0.0182599    0.786275  25"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.0820848</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.0669557</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.0653544</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.7941176</td>\n",
       "<td>1.0</td>\n",
       "<td>0.25</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>617.6470588</td>\n",
       "<td>79.4117647</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.0596437</td>\n",
       "<td>7.1764706</td>\n",
       "<td>2.8705882</td>\n",
       "<td>1.0</td>\n",
       "<td>0.4</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>187.0588235</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.0474456</td>\n",
       "<td>7.1764706</td>\n",
       "<td>4.1008403</td>\n",
       "<td>1.0</td>\n",
       "<td>0.5714286</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.2352941</td>\n",
       "<td>617.6470588</td>\n",
       "<td>310.0840336</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.0321176</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.8642534</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5384615</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.4117647</td>\n",
       "<td>258.8235294</td>\n",
       "<td>286.4253394</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.0205112</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.7770898</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5263158</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.5882353</td>\n",
       "<td>258.8235294</td>\n",
       "<td>277.7089783</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.0187841</td>\n",
       "<td>1.1960784</td>\n",
       "<td>3.1576471</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.44</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.6470588</td>\n",
       "<td>19.6078431</td>\n",
       "<td>215.7647059</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.0122340</td>\n",
       "<td>0.5980392</td>\n",
       "<td>2.3275040</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.3243243</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.7058824</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>132.7503975</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.0083828</td>\n",
       "<td>1.1960784</td>\n",
       "<td>2.0504202</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.2857143</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.8235294</td>\n",
       "<td>19.6078431</td>\n",
       "<td>105.0420168</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0067959</td>\n",
       "<td>0.0</td>\n",
       "<td>1.6470588</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2295082</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8235294</td>\n",
       "<td>-100.0</td>\n",
       "<td>64.7058824</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.0053635</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.4746172</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2054795</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>47.4617244</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.0039040</td>\n",
       "<td>0.0</td>\n",
       "<td>1.2664360</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>26.6435986</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.0026649</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.1837477</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1649485</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>18.3747726</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.0019653</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.1192661</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1559633</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>11.9266055</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0003948</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.0820848          0         0                  0                0                           0               0                          -100      -100\n",
       "    2        0.0245902                   0.0669557          0         0                  0                0                           0               0                          -100      -100\n",
       "    3        0.0327869                   0.0653544          7.17647   1.79412            1                0.25                        0.0588235       0.0588235                  617.647   79.4118\n",
       "    4        0.0409836                   0.0596437          7.17647   2.87059            1                0.4                         0.0588235       0.117647                   617.647   187.059\n",
       "    5        0.057377                    0.0474456          7.17647   4.10084            1                0.571429                    0.117647        0.235294                   617.647   310.084\n",
       "    6        0.106557                    0.0321176          3.58824   3.86425            0.5              0.538462                    0.176471        0.411765                   258.824   286.425\n",
       "    7        0.155738                    0.0205112          3.58824   3.77709            0.5              0.526316                    0.176471        0.588235                   258.824   277.709\n",
       "    8        0.204918                    0.0187841          1.19608   3.15765            0.166667         0.44                        0.0588235       0.647059                   19.6078   215.765\n",
       "    9        0.303279                    0.012234           0.598039  2.3275             0.0833333        0.324324                    0.0588235       0.705882                   -40.1961  132.75\n",
       "    10       0.401639                    0.00838285         1.19608   2.05042            0.166667         0.285714                    0.117647        0.823529                   19.6078   105.042\n",
       "    11       0.5                         0.00679594         0         1.64706            0                0.229508                    0               0.823529                   -100      64.7059\n",
       "    12       0.598361                    0.00536351         0.598039  1.47462            0.0833333        0.205479                    0.0588235       0.882353                   -40.1961  47.4617\n",
       "    13       0.696721                    0.00390399         0         1.26644            0                0.176471                    0               0.882353                   -100      26.6436\n",
       "    14       0.795082                    0.00266493         0.598039  1.18375            0.0833333        0.164948                    0.0588235       0.941176                   -40.1961  18.3748\n",
       "    15       0.893443                    0.00196534         0.598039  1.11927            0.0833333        0.155963                    0.0588235       1                          -40.1961  11.9266\n",
       "    16       1                           0.000394834        0         1                  0                0.139344                    0               1                          -100      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.14358412263559645\n",
      "RMSE: 0.3789249564697428\n",
      "LogLoss: 0.7613540753990617\n",
      "Null degrees of freedom: 121\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 104.21038206480068\n",
      "Residual deviance: 185.7703943973712\n",
      "AIC: 287.7703943973712\n",
      "AUC: 0.42735042735042733\n",
      "Gini: -0.14529914529914534\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.0004225935916309775: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>0.0</td>\n",
       "<td>104.0</td>\n",
       "<td>1.0</td>\n",
       "<td> (104.0/104.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>0.0</td>\n",
       "<td>18.0</td>\n",
       "<td>0.0</td>\n",
       "<td> (0.0/18.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>0.0</td>\n",
       "<td>122.0</td>\n",
       "<td>0.8525</td>\n",
       "<td> (104.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  -------------\n",
       "0      0    104  1        (104.0/104.0)\n",
       "1      0    18   0        (0.0/18.0)\n",
       "Total  0    122  0.8525   (104.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0004226</td>\n",
       "<td>0.2571429</td>\n",
       "<td>121.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0004226</td>\n",
       "<td>0.4639175</td>\n",
       "<td>121.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0926508</td>\n",
       "<td>0.2</td>\n",
       "<td>7.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.2448453</td>\n",
       "<td>0.8442623</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.1160391</td>\n",
       "<td>0.3333333</td>\n",
       "<td>2.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0004226</td>\n",
       "<td>1.0</td>\n",
       "<td>121.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.2448453</td>\n",
       "<td>0.9903846</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0006198</td>\n",
       "<td>0.2185183</td>\n",
       "<td>120.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0074979</td>\n",
       "<td>0.5</td>\n",
       "<td>60.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0926508</td>\n",
       "<td>0.5267094</td>\n",
       "<td>7.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.000422594  0.257143  121\n",
       "max f2                       0.000422594  0.463918  121\n",
       "max f0point5                 0.0926508    0.2       7\n",
       "max accuracy                 0.244845     0.844262  0\n",
       "max precision                0.116039     0.333333  2\n",
       "max recall                   0.000422594  1         121\n",
       "max specificity              0.244845     0.990385  0\n",
       "max absolute_mcc             0.000619801  0.218518  120\n",
       "max min_per_class_accuracy   0.0074979    0.5       60\n",
       "max mean_per_class_accuracy  0.0926508    0.526709  7"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 14.75 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.1195830</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.1153612</td>\n",
       "<td>6.7777778</td>\n",
       "<td>2.2592593</td>\n",
       "<td>1.0</td>\n",
       "<td>0.3333333</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.0555556</td>\n",
       "<td>577.7777778</td>\n",
       "<td>125.9259259</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.1100819</td>\n",
       "<td>0.0</td>\n",
       "<td>1.6944444</td>\n",
       "<td>0.0</td>\n",
       "<td>0.25</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0555556</td>\n",
       "<td>-100.0</td>\n",
       "<td>69.4444444</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.1001109</td>\n",
       "<td>0.0</td>\n",
       "<td>1.3555556</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0555556</td>\n",
       "<td>-100.0</td>\n",
       "<td>35.5555556</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.0982902</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9682540</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1428571</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0555556</td>\n",
       "<td>-100.0</td>\n",
       "<td>-3.1746032</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.0697638</td>\n",
       "<td>1.1296296</td>\n",
       "<td>1.0427350</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1538462</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.1111111</td>\n",
       "<td>12.9629630</td>\n",
       "<td>4.2735043</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.0508401</td>\n",
       "<td>0.0</td>\n",
       "<td>0.7134503</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1052632</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1111111</td>\n",
       "<td>-100.0</td>\n",
       "<td>-28.6549708</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.0283575</td>\n",
       "<td>1.1296296</td>\n",
       "<td>0.8133333</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.12</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.1666667</td>\n",
       "<td>12.9629630</td>\n",
       "<td>-18.6666667</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.0196922</td>\n",
       "<td>0.5648148</td>\n",
       "<td>0.7327327</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1081081</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.2222222</td>\n",
       "<td>-43.5185185</td>\n",
       "<td>-26.7267267</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.0120333</td>\n",
       "<td>1.1296296</td>\n",
       "<td>0.8299320</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1224490</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.3333333</td>\n",
       "<td>12.9629630</td>\n",
       "<td>-17.0068027</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0074650</td>\n",
       "<td>1.6944444</td>\n",
       "<td>1.0</td>\n",
       "<td>0.25</td>\n",
       "<td>0.1475410</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.5</td>\n",
       "<td>69.4444444</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.0058435</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8356164</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1232877</td>\n",
       "<td>0.0</td>\n",
       "<td>0.5</td>\n",
       "<td>-100.0</td>\n",
       "<td>-16.4383562</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.0039856</td>\n",
       "<td>0.5648148</td>\n",
       "<td>0.7973856</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.5555556</td>\n",
       "<td>-43.5185185</td>\n",
       "<td>-20.2614379</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.0025743</td>\n",
       "<td>1.1296296</td>\n",
       "<td>0.8384880</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1237113</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.6666667</td>\n",
       "<td>12.9629630</td>\n",
       "<td>-16.1512027</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.0017875</td>\n",
       "<td>1.1296296</td>\n",
       "<td>0.8705403</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1284404</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.7777778</td>\n",
       "<td>12.9629630</td>\n",
       "<td>-12.9459735</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0004226</td>\n",
       "<td>2.0854701</td>\n",
       "<td>1.0</td>\n",
       "<td>0.3076923</td>\n",
       "<td>0.1475410</td>\n",
       "<td>0.2222222</td>\n",
       "<td>1.0</td>\n",
       "<td>108.5470085</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.119583           0         0                  0                0                           0               0                          -100      -100\n",
       "    2        0.0245902                   0.115361           6.77778   2.25926            1                0.333333                    0.0555556       0.0555556                  577.778   125.926\n",
       "    3        0.0327869                   0.110082           0         1.69444            0                0.25                        0               0.0555556                  -100      69.4444\n",
       "    4        0.0409836                   0.100111           0         1.35556            0                0.2                         0               0.0555556                  -100      35.5556\n",
       "    5        0.057377                    0.0982902          0         0.968254           0                0.142857                    0               0.0555556                  -100      -3.1746\n",
       "    6        0.106557                    0.0697638          1.12963   1.04274            0.166667         0.153846                    0.0555556       0.111111                   12.963    4.2735\n",
       "    7        0.155738                    0.0508401          0         0.71345            0                0.105263                    0               0.111111                   -100      -28.655\n",
       "    8        0.204918                    0.0283575          1.12963   0.813333           0.166667         0.12                        0.0555556       0.166667                   12.963    -18.6667\n",
       "    9        0.303279                    0.0196922          0.564815  0.732733           0.0833333        0.108108                    0.0555556       0.222222                   -43.5185  -26.7267\n",
       "    10       0.401639                    0.0120333          1.12963   0.829932           0.166667         0.122449                    0.111111        0.333333                   12.963    -17.0068\n",
       "    11       0.5                         0.00746501         1.69444   1                  0.25             0.147541                    0.166667        0.5                        69.4444   0\n",
       "    12       0.598361                    0.0058435          0         0.835616           0                0.123288                    0               0.5                        -100      -16.4384\n",
       "    13       0.696721                    0.00398561         0.564815  0.797386           0.0833333        0.117647                    0.0555556       0.555556                   -43.5185  -20.2614\n",
       "    14       0.795082                    0.00257431         1.12963   0.838488           0.166667         0.123711                    0.111111        0.666667                   12.963    -16.1512\n",
       "    15       0.893443                    0.00178751         1.12963   0.87054            0.166667         0.12844                     0.111111        0.777778                   12.963    -12.946\n",
       "    16       1                           0.000422594        2.08547   1                  0.307692         0.147541                    0.222222        1                          108.547   0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.1384221783694127\n",
      "RMSE: 0.3720513114738513\n",
      "LogLoss: 0.864785351927186\n",
      "Null degrees of freedom: 121\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 99.19119025562574\n",
      "Residual deviance: 211.0076258702334\n",
      "AIC: 313.00762587023337\n",
      "AUC: 0.4358543417366947\n",
      "Gini: -0.12829131652661063\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.0004942806748936005: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>11.0</td>\n",
       "<td>94.0</td>\n",
       "<td>0.8952</td>\n",
       "<td> (94.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>0.0</td>\n",
       "<td>17.0</td>\n",
       "<td>0.0</td>\n",
       "<td> (0.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>11.0</td>\n",
       "<td>111.0</td>\n",
       "<td>0.7705</td>\n",
       "<td> (94.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      11   94   0.8952   (94.0/105.0)\n",
       "1      0    17   0        (0.0/17.0)\n",
       "Total  11   111  0.7705   (94.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0004943</td>\n",
       "<td>0.265625</td>\n",
       "<td>110.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0004943</td>\n",
       "<td>0.4748603</td>\n",
       "<td>110.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0013575</td>\n",
       "<td>0.1862464</td>\n",
       "<td>82.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0466987</td>\n",
       "<td>0.8524590</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.0193966</td>\n",
       "<td>0.2</td>\n",
       "<td>4.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0004943</td>\n",
       "<td>1.0</td>\n",
       "<td>110.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.0466987</td>\n",
       "<td>0.9904762</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0022207</td>\n",
       "<td>0.1796783</td>\n",
       "<td>69.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0022018</td>\n",
       "<td>0.3904762</td>\n",
       "<td>70.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0004943</td>\n",
       "<td>0.5523810</td>\n",
       "<td>110.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.000494281  0.265625  110\n",
       "max f2                       0.000494281  0.47486   110\n",
       "max f0point5                 0.00135754   0.186246  82\n",
       "max accuracy                 0.0466987    0.852459  0\n",
       "max precision                0.0193966    0.2       4\n",
       "max recall                   0.000494281  1         110\n",
       "max specificity              0.0466987    0.990476  0\n",
       "max absolute_mcc             0.00222072   0.179678  69\n",
       "max min_per_class_accuracy   0.00220178   0.390476  70\n",
       "max mean_per_class_accuracy  0.000494281  0.552381  110"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.0322753</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.0238673</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.0201160</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.0191163</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.4352941</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>617.6470588</td>\n",
       "<td>43.5294118</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.0177770</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0252101</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1428571</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>-100.0</td>\n",
       "<td>2.5210084</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.0135747</td>\n",
       "<td>0.0</td>\n",
       "<td>0.5520362</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0769231</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>-100.0</td>\n",
       "<td>-44.7963801</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.0104773</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.7554180</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1052632</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1176471</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-24.4582043</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.0075941</td>\n",
       "<td>0.0</td>\n",
       "<td>0.5741176</td>\n",
       "<td>0.0</td>\n",
       "<td>0.08</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1176471</td>\n",
       "<td>-100.0</td>\n",
       "<td>-42.5882353</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.0044016</td>\n",
       "<td>0.5980392</td>\n",
       "<td>0.5818760</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.0810811</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>-41.8124006</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.0034905</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.7322929</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1020408</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.2941176</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-26.7707083</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0028004</td>\n",
       "<td>0.5980392</td>\n",
       "<td>0.7058824</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.0983607</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.3529412</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>-29.4117647</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.0018633</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.7864625</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1095890</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.4705882</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-21.3537470</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.0012251</td>\n",
       "<td>2.9901961</td>\n",
       "<td>1.0975779</td>\n",
       "<td>0.4166667</td>\n",
       "<td>0.1529412</td>\n",
       "<td>0.2941176</td>\n",
       "<td>0.7647059</td>\n",
       "<td>199.0196078</td>\n",
       "<td>9.7577855</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.0007777</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.0357793</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1443299</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.8235294</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>3.5779260</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.0005074</td>\n",
       "<td>1.1960784</td>\n",
       "<td>1.0534269</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1467890</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.9411765</td>\n",
       "<td>19.6078431</td>\n",
       "<td>5.3426875</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0000799</td>\n",
       "<td>0.5520362</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0769231</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-44.7963801</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.0322753          0         0                  0                0                           0               0                          -100      -100\n",
       "    2        0.0245902                   0.0238673          0         0                  0                0                           0               0                          -100      -100\n",
       "    3        0.0327869                   0.020116           0         0                  0                0                           0               0                          -100      -100\n",
       "    4        0.0409836                   0.0191163          7.17647   1.43529            1                0.2                         0.0588235       0.0588235                  617.647   43.5294\n",
       "    5        0.057377                    0.017777           0         1.02521            0                0.142857                    0               0.0588235                  -100      2.52101\n",
       "    6        0.106557                    0.0135747          0         0.552036           0                0.0769231                   0               0.0588235                  -100      -44.7964\n",
       "    7        0.155738                    0.0104773          1.19608   0.755418           0.166667         0.105263                    0.0588235       0.117647                   19.6078   -24.4582\n",
       "    8        0.204918                    0.00759414         0         0.574118           0                0.08                        0               0.117647                   -100      -42.5882\n",
       "    9        0.303279                    0.00440163         0.598039  0.581876           0.0833333        0.0810811                   0.0588235       0.176471                   -40.1961  -41.8124\n",
       "    10       0.401639                    0.00349051         1.19608   0.732293           0.166667         0.102041                    0.117647        0.294118                   19.6078   -26.7707\n",
       "    11       0.5                         0.0028004          0.598039  0.705882           0.0833333        0.0983607                   0.0588235       0.352941                   -40.1961  -29.4118\n",
       "    12       0.598361                    0.0018633          1.19608   0.786463           0.166667         0.109589                    0.117647        0.470588                   19.6078   -21.3537\n",
       "    13       0.696721                    0.0012251          2.9902    1.09758            0.416667         0.152941                    0.294118        0.764706                   199.02    9.75779\n",
       "    14       0.795082                    0.000777669        0.598039  1.03578            0.0833333        0.14433                     0.0588235       0.823529                   -40.1961  3.57793\n",
       "    15       0.893443                    0.000507358        1.19608   1.05343            0.166667         0.146789                    0.117647        0.941176                   19.6078   5.34269\n",
       "    16       1                           7.99295e-05        0.552036  1                  0.0769231        0.139344                    0.0588235       1                          -44.7964  0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for ycol in ['D_NBER','D_NBER_1', 'D_NBER_3', 'D_NBER_6',\n",
    "            'D_NBER_12']:\n",
    "    logistic_h2o = H2OGeneralizedLinearEstimator(alpha=0, family = \"binomial\"\n",
    "                                                )\n",
    "    logistic_h2o.train(y = ycol, x = list(range(50)), training_frame = train_df)\n",
    "    predictions = logistic_h2o.predict(test_df)\n",
    "    all_predictions_topics.append(predictions)\n",
    "    name = \"prediction \" + ycol + \"TopicsOnly.csv\"\n",
    "    h2o.export_file(predictions, name, force = True)\n",
    "    \n",
    "    print(logistic_h2o.model_performance(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.34185202296287015\n",
      "RMSE: 0.5846811293028621\n",
      "LogLoss: 0.8888861392551901\n",
      "Null degrees of freedom: 122\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 169.12791205662666\n",
      "Residual deviance: 216.88821797826637\n",
      "AIC: 318.88821797826637\n",
      "AUC: 0.9675070028011205\n",
      "Gini: 0.9350140056022409\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.8760196316809216: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>104.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0095</td>\n",
       "<td> (1.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>1.0</td>\n",
       "<td>16.0</td>\n",
       "<td>0.0588</td>\n",
       "<td> (1.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>105.0</td>\n",
       "<td>17.0</td>\n",
       "<td>0.0164</td>\n",
       "<td> (2.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  -----------\n",
       "0      104  1    0.0095   (1.0/105.0)\n",
       "1      1    16   0.0588   (1.0/17.0)\n",
       "Total  105  17   0.0164   (2.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.8760196</td>\n",
       "<td>0.9411765</td>\n",
       "<td>16.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.8760196</td>\n",
       "<td>0.9411765</td>\n",
       "<td>16.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.9193271</td>\n",
       "<td>0.9740260</td>\n",
       "<td>14.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.9193271</td>\n",
       "<td>0.9836066</td>\n",
       "<td>14.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.9981284</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.5969177</td>\n",
       "<td>1.0</td>\n",
       "<td>73.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.9981284</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.8760196</td>\n",
       "<td>0.9316527</td>\n",
       "<td>16.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.8760196</td>\n",
       "<td>0.9411765</td>\n",
       "<td>16.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.8760196</td>\n",
       "<td>0.9658263</td>\n",
       "<td>16.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.87602      0.941176  16\n",
       "max f2                       0.87602      0.941176  16\n",
       "max f0point5                 0.919327     0.974026  14\n",
       "max accuracy                 0.919327     0.983607  14\n",
       "max precision                0.998128     1         0\n",
       "max recall                   0.596918     1         73\n",
       "max specificity              0.998128     1         0\n",
       "max absolute_mcc             0.87602      0.931653  16\n",
       "max min_per_class_accuracy   0.87602      0.941176  16\n",
       "max mean_per_class_accuracy  0.87602      0.965826  16"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.9962525</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.9921617</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.9865735</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.2352941</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.9859439</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.2941176</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.9810696</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.4117647</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.9242296</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.3529412</td>\n",
       "<td>0.7647059</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.8346530</td>\n",
       "<td>3.5882353</td>\n",
       "<td>6.0433437</td>\n",
       "<td>0.5</td>\n",
       "<td>0.8421053</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.9411765</td>\n",
       "<td>258.8235294</td>\n",
       "<td>504.3343653</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.8079452</td>\n",
       "<td>0.0</td>\n",
       "<td>4.5929412</td>\n",
       "<td>0.0</td>\n",
       "<td>0.64</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>359.2941176</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.7382200</td>\n",
       "<td>0.0</td>\n",
       "<td>3.1033386</td>\n",
       "<td>0.0</td>\n",
       "<td>0.4324324</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>210.3338633</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.7021566</td>\n",
       "<td>0.0</td>\n",
       "<td>2.3433373</td>\n",
       "<td>0.0</td>\n",
       "<td>0.3265306</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>134.3337335</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.6575355</td>\n",
       "<td>0.0</td>\n",
       "<td>1.8823529</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2622951</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>88.2352941</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.5988099</td>\n",
       "<td>0.0</td>\n",
       "<td>1.5729251</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2191781</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>57.2925060</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.5628774</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.4352941</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>43.5294118</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.5235630</td>\n",
       "<td>0.0</td>\n",
       "<td>1.2577320</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1752577</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>25.7731959</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.4127334</td>\n",
       "<td>0.0</td>\n",
       "<td>1.1192661</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1559633</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>11.9266055</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2732738</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.996252           7.17647   7.17647            1                1                           0.117647        0.117647                   617.647   617.647\n",
       "    2        0.0245902                   0.992162           7.17647   7.17647            1                1                           0.0588235       0.176471                   617.647   617.647\n",
       "    3        0.0327869                   0.986574           7.17647   7.17647            1                1                           0.0588235       0.235294                   617.647   617.647\n",
       "    4        0.0409836                   0.985944           7.17647   7.17647            1                1                           0.0588235       0.294118                   617.647   617.647\n",
       "    5        0.057377                    0.98107            7.17647   7.17647            1                1                           0.117647        0.411765                   617.647   617.647\n",
       "    6        0.106557                    0.92423            7.17647   7.17647            1                1                           0.352941        0.764706                   617.647   617.647\n",
       "    7        0.155738                    0.834653           3.58824   6.04334            0.5              0.842105                    0.176471        0.941176                   258.824   504.334\n",
       "    8        0.204918                    0.807945           0         4.59294            0                0.64                        0               0.941176                   -100      359.294\n",
       "    9        0.303279                    0.73822            0         3.10334            0                0.432432                    0               0.941176                   -100      210.334\n",
       "    10       0.401639                    0.702157           0         2.34334            0                0.326531                    0               0.941176                   -100      134.334\n",
       "    11       0.5                         0.657535           0         1.88235            0                0.262295                    0               0.941176                   -100      88.2353\n",
       "    12       0.598361                    0.59881            0         1.57293            0                0.219178                    0               0.941176                   -100      57.2925\n",
       "    13       0.696721                    0.562877           0.598039  1.43529            0.0833333        0.2                         0.0588235       1                          -40.1961  43.5294\n",
       "    14       0.795082                    0.523563           0         1.25773            0                0.175258                    0               1                          -100      25.7732\n",
       "    15       0.893443                    0.412733           0         1.11927            0                0.155963                    0               1                          -100      11.9266\n",
       "    16       1                           0.273274           0         1                  0                0.139344                    0               1                          -100      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.2871462965265335\n",
      "RMSE: 0.5358603330407407\n",
      "LogLoss: 0.7776414595713738\n",
      "Null degrees of freedom: 122\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 169.12791205662666\n",
      "Residual deviance: 189.7445161354152\n",
      "AIC: 291.7445161354152\n",
      "AUC: 0.8974789915966387\n",
      "Gini: 0.7949579831932774\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.8001080409574585: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>100.0</td>\n",
       "<td>5.0</td>\n",
       "<td>0.0476</td>\n",
       "<td> (5.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>3.0</td>\n",
       "<td>14.0</td>\n",
       "<td>0.1765</td>\n",
       "<td> (3.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>103.0</td>\n",
       "<td>19.0</td>\n",
       "<td>0.0656</td>\n",
       "<td> (8.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  -----------\n",
       "0      100  5    0.0476   (5.0/105.0)\n",
       "1      3    14   0.1765   (3.0/17.0)\n",
       "Total  103  19   0.0656   (8.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.8001080</td>\n",
       "<td>0.7777778</td>\n",
       "<td>18.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.8001080</td>\n",
       "<td>0.8045977</td>\n",
       "<td>18.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.8598994</td>\n",
       "<td>0.7971014</td>\n",
       "<td>12.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.8598994</td>\n",
       "<td>0.9344262</td>\n",
       "<td>12.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.9851849</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.3480142</td>\n",
       "<td>1.0</td>\n",
       "<td>110.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.9851849</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.8001080</td>\n",
       "<td>0.7410291</td>\n",
       "<td>18.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.7188547</td>\n",
       "<td>0.8666667</td>\n",
       "<td>28.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.8001080</td>\n",
       "<td>0.8879552</td>\n",
       "<td>18.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.800108     0.777778  18\n",
       "max f2                       0.800108     0.804598  18\n",
       "max f0point5                 0.859899     0.797101  12\n",
       "max accuracy                 0.859899     0.934426  12\n",
       "max precision                0.985185     1         0\n",
       "max recall                   0.348014     1         110\n",
       "max specificity              0.985185     1         0\n",
       "max absolute_mcc             0.800108     0.741029  18\n",
       "max min_per_class_accuracy   0.718855     0.866667  28\n",
       "max mean_per_class_accuracy  0.800108     0.887955  18"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.9642131</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.9538549</td>\n",
       "<td>7.1764706</td>\n",
       "<td>7.1764706</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>617.6470588</td>\n",
       "<td>617.6470588</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.9458173</td>\n",
       "<td>0.0</td>\n",
       "<td>5.3823529</td>\n",
       "<td>0.0</td>\n",
       "<td>0.75</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1764706</td>\n",
       "<td>-100.0</td>\n",
       "<td>438.2352941</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.9349160</td>\n",
       "<td>7.1764706</td>\n",
       "<td>5.7411765</td>\n",
       "<td>1.0</td>\n",
       "<td>0.8</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.2352941</td>\n",
       "<td>617.6470588</td>\n",
       "<td>474.1176471</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.9246771</td>\n",
       "<td>7.1764706</td>\n",
       "<td>6.1512605</td>\n",
       "<td>1.0</td>\n",
       "<td>0.8571429</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.3529412</td>\n",
       "<td>617.6470588</td>\n",
       "<td>515.1260504</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.8590615</td>\n",
       "<td>5.9803922</td>\n",
       "<td>6.0723982</td>\n",
       "<td>0.8333333</td>\n",
       "<td>0.8461538</td>\n",
       "<td>0.2941176</td>\n",
       "<td>0.6470588</td>\n",
       "<td>498.0392157</td>\n",
       "<td>507.2398190</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.7989520</td>\n",
       "<td>3.5882353</td>\n",
       "<td>5.2879257</td>\n",
       "<td>0.5</td>\n",
       "<td>0.7368421</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.8235294</td>\n",
       "<td>258.8235294</td>\n",
       "<td>428.7925697</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.7359626</td>\n",
       "<td>0.0</td>\n",
       "<td>4.0188235</td>\n",
       "<td>0.0</td>\n",
       "<td>0.56</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8235294</td>\n",
       "<td>-100.0</td>\n",
       "<td>301.8823529</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.6795627</td>\n",
       "<td>0.5980392</td>\n",
       "<td>2.9093800</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.4054054</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>190.9379968</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.6353327</td>\n",
       "<td>0.0</td>\n",
       "<td>2.1968788</td>\n",
       "<td>0.0</td>\n",
       "<td>0.3061224</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>119.6878752</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5762255</td>\n",
       "<td>0.0</td>\n",
       "<td>1.7647059</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2459016</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>76.4705882</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.5225955</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.5729251</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2191781</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>57.2925060</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.4652707</td>\n",
       "<td>0.0</td>\n",
       "<td>1.3508651</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1882353</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>35.0865052</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.4190086</td>\n",
       "<td>0.0</td>\n",
       "<td>1.1837477</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1649485</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>18.3747726</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.3601470</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0534269</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1467890</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>5.3426875</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2639957</td>\n",
       "<td>0.5520362</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0769231</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-44.7963801</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.964213           7.17647   7.17647            1                1                           0.117647        0.117647                   617.647   617.647\n",
       "    2        0.0245902                   0.953855           7.17647   7.17647            1                1                           0.0588235       0.176471                   617.647   617.647\n",
       "    3        0.0327869                   0.945817           0         5.38235            0                0.75                        0               0.176471                   -100      438.235\n",
       "    4        0.0409836                   0.934916           7.17647   5.74118            1                0.8                         0.0588235       0.235294                   617.647   474.118\n",
       "    5        0.057377                    0.924677           7.17647   6.15126            1                0.857143                    0.117647        0.352941                   617.647   515.126\n",
       "    6        0.106557                    0.859062           5.98039   6.0724             0.833333         0.846154                    0.294118        0.647059                   498.039   507.24\n",
       "    7        0.155738                    0.798952           3.58824   5.28793            0.5              0.736842                    0.176471        0.823529                   258.824   428.793\n",
       "    8        0.204918                    0.735963           0         4.01882            0                0.56                        0               0.823529                   -100      301.882\n",
       "    9        0.303279                    0.679563           0.598039  2.90938            0.0833333        0.405405                    0.0588235       0.882353                   -40.1961  190.938\n",
       "    10       0.401639                    0.635333           0         2.19688            0                0.306122                    0               0.882353                   -100      119.688\n",
       "    11       0.5                         0.576225           0         1.76471            0                0.245902                    0               0.882353                   -100      76.4706\n",
       "    12       0.598361                    0.522595           0.598039  1.57293            0.0833333        0.219178                    0.0588235       0.941176                   -40.1961  57.2925\n",
       "    13       0.696721                    0.465271           0         1.35087            0                0.188235                    0               0.941176                   -100      35.0865\n",
       "    14       0.795082                    0.419009           0         1.18375            0                0.164948                    0               0.941176                   -100      18.3748\n",
       "    15       0.893443                    0.360147           0         1.05343            0                0.146789                    0               0.941176                   -100      5.34269\n",
       "    16       1                           0.263996           0.552036  1                  0.0769231        0.139344                    0.0588235       1                          -44.7964  0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.35754008590601183\n",
      "RMSE: 0.5979465577340602\n",
      "LogLoss: 0.9531463575132457\n",
      "Null degrees of freedom: 122\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 169.12791205662666\n",
      "Residual deviance: 232.56771123323196\n",
      "AIC: 334.567711233232\n",
      "AUC: 0.7803921568627451\n",
      "Gini: 0.5607843137254902\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.8741424067923311: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>101.0</td>\n",
       "<td>4.0</td>\n",
       "<td>0.0381</td>\n",
       "<td> (4.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>9.0</td>\n",
       "<td>8.0</td>\n",
       "<td>0.5294</td>\n",
       "<td> (9.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>110.0</td>\n",
       "<td>12.0</td>\n",
       "<td>0.1066</td>\n",
       "<td> (13.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      101  4    0.0381   (4.0/105.0)\n",
       "1      9    8    0.5294   (9.0/17.0)\n",
       "Total  110  12   0.1066   (13.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.8741424</td>\n",
       "<td>0.5517241</td>\n",
       "<td>11.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.7890091</td>\n",
       "<td>0.5789474</td>\n",
       "<td>26.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.8741424</td>\n",
       "<td>0.6153846</td>\n",
       "<td>11.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.9047569</td>\n",
       "<td>0.8934426</td>\n",
       "<td>7.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.9509314</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.3234967</td>\n",
       "<td>1.0</td>\n",
       "<td>119.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.9509314</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.8741424</td>\n",
       "<td>0.5029343</td>\n",
       "<td>11.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.7521823</td>\n",
       "<td>0.7058824</td>\n",
       "<td>40.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.7890091</td>\n",
       "<td>0.7473389</td>\n",
       "<td>26.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.874142     0.551724  11\n",
       "max f2                       0.789009     0.578947  26\n",
       "max f0point5                 0.874142     0.615385  11\n",
       "max accuracy                 0.904757     0.893443  7\n",
       "max precision                0.950931     1         0\n",
       "max recall                   0.323497     1         119\n",
       "max specificity              0.950931     1         0\n",
       "max absolute_mcc             0.874142     0.502934  11\n",
       "max min_per_class_accuracy   0.752182     0.705882  40\n",
       "max mean_per_class_accuracy  0.789009     0.747339  26"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.9316028</td>\n",
       "<td>3.5882353</td>\n",
       "<td>3.5882353</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>258.8235294</td>\n",
       "<td>258.8235294</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.9239489</td>\n",
       "<td>7.1764706</td>\n",
       "<td>4.7843137</td>\n",
       "<td>1.0</td>\n",
       "<td>0.6666667</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1176471</td>\n",
       "<td>617.6470588</td>\n",
       "<td>378.4313725</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.9130865</td>\n",
       "<td>7.1764706</td>\n",
       "<td>5.3823529</td>\n",
       "<td>1.0</td>\n",
       "<td>0.75</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.1764706</td>\n",
       "<td>617.6470588</td>\n",
       "<td>438.2352941</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.9066608</td>\n",
       "<td>0.0</td>\n",
       "<td>4.3058824</td>\n",
       "<td>0.0</td>\n",
       "<td>0.6</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1764706</td>\n",
       "<td>-100.0</td>\n",
       "<td>330.5882353</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.9059889</td>\n",
       "<td>7.1764706</td>\n",
       "<td>5.1260504</td>\n",
       "<td>1.0</td>\n",
       "<td>0.7142857</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.2941176</td>\n",
       "<td>617.6470588</td>\n",
       "<td>412.6050420</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.8713059</td>\n",
       "<td>3.5882353</td>\n",
       "<td>4.4162896</td>\n",
       "<td>0.5</td>\n",
       "<td>0.6153846</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.4705882</td>\n",
       "<td>258.8235294</td>\n",
       "<td>341.6289593</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.8342621</td>\n",
       "<td>1.1960784</td>\n",
       "<td>3.3993808</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.4736842</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.5294118</td>\n",
       "<td>19.6078431</td>\n",
       "<td>239.9380805</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.8036995</td>\n",
       "<td>1.1960784</td>\n",
       "<td>2.8705882</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.4</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.5882353</td>\n",
       "<td>19.6078431</td>\n",
       "<td>187.0588235</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.7585863</td>\n",
       "<td>0.5980392</td>\n",
       "<td>2.1335453</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2972973</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.6470588</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>113.3545310</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.7109346</td>\n",
       "<td>1.1960784</td>\n",
       "<td>1.9039616</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.2653061</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.7647059</td>\n",
       "<td>19.6078431</td>\n",
       "<td>90.3961585</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.6270391</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.6470588</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2295082</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.8235294</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>64.7058824</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.5878698</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.4746172</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.2054795</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>47.4617244</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.5337288</td>\n",
       "<td>0.0</td>\n",
       "<td>1.2664360</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.0</td>\n",
       "<td>0.8823529</td>\n",
       "<td>-100.0</td>\n",
       "<td>26.6435986</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.4638206</td>\n",
       "<td>0.5980392</td>\n",
       "<td>1.1837477</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1649485</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>18.3747726</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.4024161</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0534269</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1467890</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9411765</td>\n",
       "<td>-100.0</td>\n",
       "<td>5.3426875</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.3067066</td>\n",
       "<td>0.5520362</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0769231</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.0588235</td>\n",
       "<td>1.0</td>\n",
       "<td>-44.7963801</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.931603           3.58824   3.58824            0.5              0.5                         0.0588235       0.0588235                  258.824   258.824\n",
       "    2        0.0245902                   0.923949           7.17647   4.78431            1                0.666667                    0.0588235       0.117647                   617.647   378.431\n",
       "    3        0.0327869                   0.913086           7.17647   5.38235            1                0.75                        0.0588235       0.176471                   617.647   438.235\n",
       "    4        0.0409836                   0.906661           0         4.30588            0                0.6                         0               0.176471                   -100      330.588\n",
       "    5        0.057377                    0.905989           7.17647   5.12605            1                0.714286                    0.117647        0.294118                   617.647   412.605\n",
       "    6        0.106557                    0.871306           3.58824   4.41629            0.5              0.615385                    0.176471        0.470588                   258.824   341.629\n",
       "    7        0.155738                    0.834262           1.19608   3.39938            0.166667         0.473684                    0.0588235       0.529412                   19.6078   239.938\n",
       "    8        0.204918                    0.8037             1.19608   2.87059            0.166667         0.4                         0.0588235       0.588235                   19.6078   187.059\n",
       "    9        0.303279                    0.758586           0.598039  2.13355            0.0833333        0.297297                    0.0588235       0.647059                   -40.1961  113.355\n",
       "    10       0.401639                    0.710935           1.19608   1.90396            0.166667         0.265306                    0.117647        0.764706                   19.6078   90.3962\n",
       "    11       0.5                         0.627039           0.598039  1.64706            0.0833333        0.229508                    0.0588235       0.823529                   -40.1961  64.7059\n",
       "    12       0.598361                    0.58787            0.598039  1.47462            0.0833333        0.205479                    0.0588235       0.882353                   -40.1961  47.4617\n",
       "    13       0.696721                    0.533729           0         1.26644            0                0.176471                    0               0.882353                   -100      26.6436\n",
       "    14       0.795082                    0.463821           0.598039  1.18375            0.0833333        0.164948                    0.0588235       0.941176                   -40.1961  18.3748\n",
       "    15       0.893443                    0.402416           0         1.05343            0                0.146789                    0               0.941176                   -100      5.34269\n",
       "    16       1                           0.306707           0.552036  1                  0.0769231        0.139344                    0.0588235       1                          -44.7964  0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.35692354901655476\n",
      "RMSE: 0.5974307901477415\n",
      "LogLoss: 0.9496554479141908\n",
      "Null degrees of freedom: 122\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 169.12791205662666\n",
      "Residual deviance: 231.71592929106254\n",
      "AIC: 333.71592929106254\n",
      "AUC: 0.5400641025641025\n",
      "Gini: 0.08012820512820507\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.5945821882315772: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>54.0</td>\n",
       "<td>50.0</td>\n",
       "<td>0.4808</td>\n",
       "<td> (50.0/104.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>6.0</td>\n",
       "<td>12.0</td>\n",
       "<td>0.3333</td>\n",
       "<td> (6.0/18.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>60.0</td>\n",
       "<td>62.0</td>\n",
       "<td>0.459</td>\n",
       "<td> (56.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      54   50   0.4808   (50.0/104.0)\n",
       "1      6    12   0.3333   (6.0/18.0)\n",
       "Total  60   62   0.459    (56.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.5945822</td>\n",
       "<td>0.3000000</td>\n",
       "<td>61.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.2561037</td>\n",
       "<td>0.4712042</td>\n",
       "<td>118.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.6793196</td>\n",
       "<td>0.2272727</td>\n",
       "<td>44.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.9088161</td>\n",
       "<td>0.8442623</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.7853709</td>\n",
       "<td>0.2105263</td>\n",
       "<td>18.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.2561037</td>\n",
       "<td>1.0</td>\n",
       "<td>118.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.9088161</td>\n",
       "<td>0.9903846</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.5945822</td>\n",
       "<td>0.1318727</td>\n",
       "<td>61.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.6259587</td>\n",
       "<td>0.5555556</td>\n",
       "<td>55.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.5945822</td>\n",
       "<td>0.5929487</td>\n",
       "<td>61.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.594582     0.3       61\n",
       "max f2                       0.256104     0.471204  118\n",
       "max f0point5                 0.67932      0.227273  44\n",
       "max accuracy                 0.908816     0.844262  0\n",
       "max precision                0.785371     0.210526  18\n",
       "max recall                   0.256104     1         118\n",
       "max specificity              0.908816     0.990385  0\n",
       "max absolute_mcc             0.594582     0.131873  61\n",
       "max min_per_class_accuracy   0.625959     0.555556  55\n",
       "max mean_per_class_accuracy  0.594582     0.592949  61"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 14.75 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.8910201</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.8808206</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.8682138</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.8601314</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.8536543</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.8353210</td>\n",
       "<td>1.1296296</td>\n",
       "<td>0.5213675</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.0769231</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.0555556</td>\n",
       "<td>12.9629630</td>\n",
       "<td>-47.8632479</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.7847170</td>\n",
       "<td>3.3888889</td>\n",
       "<td>1.4269006</td>\n",
       "<td>0.5</td>\n",
       "<td>0.2105263</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.2222222</td>\n",
       "<td>238.8888889</td>\n",
       "<td>42.6900585</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.7592724</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0844444</td>\n",
       "<td>0.0</td>\n",
       "<td>0.16</td>\n",
       "<td>0.0</td>\n",
       "<td>0.2222222</td>\n",
       "<td>-100.0</td>\n",
       "<td>8.4444444</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.7159258</td>\n",
       "<td>1.1296296</td>\n",
       "<td>1.0990991</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1621622</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.3333333</td>\n",
       "<td>12.9629630</td>\n",
       "<td>9.9099099</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.6635905</td>\n",
       "<td>1.6944444</td>\n",
       "<td>1.2448980</td>\n",
       "<td>0.25</td>\n",
       "<td>0.1836735</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.5</td>\n",
       "<td>69.4444444</td>\n",
       "<td>24.4897959</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5978334</td>\n",
       "<td>1.1296296</td>\n",
       "<td>1.2222222</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1803279</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.6111111</td>\n",
       "<td>12.9629630</td>\n",
       "<td>22.2222222</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.5520296</td>\n",
       "<td>0.5648148</td>\n",
       "<td>1.1141553</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1643836</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.6666667</td>\n",
       "<td>-43.5185185</td>\n",
       "<td>11.4155251</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.5176784</td>\n",
       "<td>1.1296296</td>\n",
       "<td>1.1163399</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1647059</td>\n",
       "<td>0.1111111</td>\n",
       "<td>0.7777778</td>\n",
       "<td>12.9629630</td>\n",
       "<td>11.6339869</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.4668372</td>\n",
       "<td>0.0</td>\n",
       "<td>0.9782360</td>\n",
       "<td>0.0</td>\n",
       "<td>0.1443299</td>\n",
       "<td>0.0</td>\n",
       "<td>0.7777778</td>\n",
       "<td>-100.0</td>\n",
       "<td>-2.1764032</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.3724822</td>\n",
       "<td>0.5648148</td>\n",
       "<td>0.9327217</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.1376147</td>\n",
       "<td>0.0555556</td>\n",
       "<td>0.8333333</td>\n",
       "<td>-43.5185185</td>\n",
       "<td>-6.7278287</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1656389</td>\n",
       "<td>1.5641026</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2307692</td>\n",
       "<td>0.1475410</td>\n",
       "<td>0.1666667</td>\n",
       "<td>1.0</td>\n",
       "<td>56.4102564</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.89102            0         0                  0                0                           0               0                          -100      -100\n",
       "    2        0.0245902                   0.880821           0         0                  0                0                           0               0                          -100      -100\n",
       "    3        0.0327869                   0.868214           0         0                  0                0                           0               0                          -100      -100\n",
       "    4        0.0409836                   0.860131           0         0                  0                0                           0               0                          -100      -100\n",
       "    5        0.057377                    0.853654           0         0                  0                0                           0               0                          -100      -100\n",
       "    6        0.106557                    0.835321           1.12963   0.521368           0.166667         0.0769231                   0.0555556       0.0555556                  12.963    -47.8632\n",
       "    7        0.155738                    0.784717           3.38889   1.4269             0.5              0.210526                    0.166667        0.222222                   238.889   42.6901\n",
       "    8        0.204918                    0.759272           0         1.08444            0                0.16                        0               0.222222                   -100      8.44444\n",
       "    9        0.303279                    0.715926           1.12963   1.0991             0.166667         0.162162                    0.111111        0.333333                   12.963    9.90991\n",
       "    10       0.401639                    0.663591           1.69444   1.2449             0.25             0.183673                    0.166667        0.5                        69.4444   24.4898\n",
       "    11       0.5                         0.597833           1.12963   1.22222            0.166667         0.180328                    0.111111        0.611111                   12.963    22.2222\n",
       "    12       0.598361                    0.55203            0.564815  1.11416            0.0833333        0.164384                    0.0555556       0.666667                   -43.5185  11.4155\n",
       "    13       0.696721                    0.517678           1.12963   1.11634            0.166667         0.164706                    0.111111        0.777778                   12.963    11.634\n",
       "    14       0.795082                    0.466837           0         0.978236           0                0.14433                     0               0.777778                   -100      -2.1764\n",
       "    15       0.893443                    0.372482           0.564815  0.932722           0.0833333        0.137615                    0.0555556       0.833333                   -43.5185  -6.72783\n",
       "    16       1                           0.165639           1.5641    1                  0.230769         0.147541                    0.166667        1                          56.4103   0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "Export File progress: |███████████████████████████████████████████████████| 100%\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.34628751186611373\n",
      "RMSE: 0.5884619884632428\n",
      "LogLoss: 0.9249750096409297\n",
      "Null degrees of freedom: 122\n",
      "Residual degrees of freedom: 71\n",
      "Null deviance: 169.12791205662666\n",
      "Residual deviance: 225.69390235238689\n",
      "AIC: 327.6939023523869\n",
      "AUC: 0.32717086834733894\n",
      "Gini: -0.3456582633053221\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.29500152747168584: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>18.0</td>\n",
       "<td>87.0</td>\n",
       "<td>0.8286</td>\n",
       "<td> (87.0/105.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>2.0</td>\n",
       "<td>15.0</td>\n",
       "<td>0.1176</td>\n",
       "<td> (2.0/17.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>20.0</td>\n",
       "<td>102.0</td>\n",
       "<td>0.7295</td>\n",
       "<td> (89.0/122.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0    1    Error    Rate\n",
       "-----  ---  ---  -------  ------------\n",
       "0      18   87   0.8286   (87.0/105.0)\n",
       "1      2    15   0.1176   (2.0/17.0)\n",
       "Total  20   102  0.7295   (89.0/122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.2950015</td>\n",
       "<td>0.2521008</td>\n",
       "<td>101.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.1888796</td>\n",
       "<td>0.4569892</td>\n",
       "<td>117.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.2950015</td>\n",
       "<td>0.1764706</td>\n",
       "<td>101.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.9003986</td>\n",
       "<td>0.8524590</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.2950015</td>\n",
       "<td>0.1470588</td>\n",
       "<td>101.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.1888796</td>\n",
       "<td>1.0</td>\n",
       "<td>117.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>0.9003986</td>\n",
       "<td>0.9904762</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.6319809</td>\n",
       "<td>0.3185284</td>\n",
       "<td>46.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.4226038</td>\n",
       "<td>0.3809524</td>\n",
       "<td>71.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.2950015</td>\n",
       "<td>0.5268908</td>\n",
       "<td>101.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.295002     0.252101  101\n",
       "max f2                       0.18888      0.456989  117\n",
       "max f0point5                 0.295002     0.176471  101\n",
       "max accuracy                 0.900399     0.852459  0\n",
       "max precision                0.295002     0.147059  101\n",
       "max recall                   0.18888      1         117\n",
       "max specificity              0.900399     0.990476  0\n",
       "max absolute_mcc             0.631981     0.318528  46\n",
       "max min_per_class_accuracy   0.422604     0.380952  71\n",
       "max mean_per_class_accuracy  0.295002     0.526891  101"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate: 13.93 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0163934</td>\n",
       "<td>0.8916902</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0245902</td>\n",
       "<td>0.8596553</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0327869</td>\n",
       "<td>0.8541325</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0409836</td>\n",
       "<td>0.8448870</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0573770</td>\n",
       "<td>0.8428315</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1065574</td>\n",
       "<td>0.8111173</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1557377</td>\n",
       "<td>0.7875651</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2049180</td>\n",
       "<td>0.7474443</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3032787</td>\n",
       "<td>0.6939005</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>-100.0</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.4016393</td>\n",
       "<td>0.6062516</td>\n",
       "<td>0.5980392</td>\n",
       "<td>0.1464586</td>\n",
       "<td>0.0833333</td>\n",
       "<td>0.0204082</td>\n",
       "<td>0.0588235</td>\n",
       "<td>0.0588235</td>\n",
       "<td>-40.1960784</td>\n",
       "<td>-85.3541417</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>0.5</td>\n",
       "<td>0.5174991</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.3529412</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.0491803</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.1764706</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-64.7058824</td></tr>\n",
       "<tr><td></td>\n",
       "<td>12</td>\n",
       "<td>0.5983607</td>\n",
       "<td>0.4213439</td>\n",
       "<td>2.9901961</td>\n",
       "<td>0.7864625</td>\n",
       "<td>0.4166667</td>\n",
       "<td>0.1095890</td>\n",
       "<td>0.2941176</td>\n",
       "<td>0.4705882</td>\n",
       "<td>199.0196078</td>\n",
       "<td>-21.3537470</td></tr>\n",
       "<tr><td></td>\n",
       "<td>13</td>\n",
       "<td>0.6967213</td>\n",
       "<td>0.3757807</td>\n",
       "<td>1.7941176</td>\n",
       "<td>0.9287197</td>\n",
       "<td>0.25</td>\n",
       "<td>0.1294118</td>\n",
       "<td>0.1764706</td>\n",
       "<td>0.6470588</td>\n",
       "<td>79.4117647</td>\n",
       "<td>-7.1280277</td></tr>\n",
       "<tr><td></td>\n",
       "<td>14</td>\n",
       "<td>0.7950820</td>\n",
       "<td>0.3182502</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.9617950</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1340206</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.7647059</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-3.8204973</td></tr>\n",
       "<tr><td></td>\n",
       "<td>15</td>\n",
       "<td>0.8934426</td>\n",
       "<td>0.2598947</td>\n",
       "<td>1.1960784</td>\n",
       "<td>0.9875877</td>\n",
       "<td>0.1666667</td>\n",
       "<td>0.1376147</td>\n",
       "<td>0.1176471</td>\n",
       "<td>0.8823529</td>\n",
       "<td>19.6078431</td>\n",
       "<td>-1.2412304</td></tr>\n",
       "<tr><td></td>\n",
       "<td>16</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0848338</td>\n",
       "<td>1.1040724</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1538462</td>\n",
       "<td>0.1393443</td>\n",
       "<td>0.1176471</td>\n",
       "<td>1.0</td>\n",
       "<td>10.4072398</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift      cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain      cumulative_gain\n",
       "--  -------  --------------------------  -----------------  --------  -----------------  ---------------  --------------------------  --------------  -------------------------  --------  -----------------\n",
       "    1        0.0163934                   0.89169            0         0                  0                0                           0               0                          -100      -100\n",
       "    2        0.0245902                   0.859655           0         0                  0                0                           0               0                          -100      -100\n",
       "    3        0.0327869                   0.854133           0         0                  0                0                           0               0                          -100      -100\n",
       "    4        0.0409836                   0.844887           0         0                  0                0                           0               0                          -100      -100\n",
       "    5        0.057377                    0.842831           0         0                  0                0                           0               0                          -100      -100\n",
       "    6        0.106557                    0.811117           0         0                  0                0                           0               0                          -100      -100\n",
       "    7        0.155738                    0.787565           0         0                  0                0                           0               0                          -100      -100\n",
       "    8        0.204918                    0.747444           0         0                  0                0                           0               0                          -100      -100\n",
       "    9        0.303279                    0.693901           0         0                  0                0                           0               0                          -100      -100\n",
       "    10       0.401639                    0.606252           0.598039  0.146459           0.0833333        0.0204082                   0.0588235       0.0588235                  -40.1961  -85.3541\n",
       "    11       0.5                         0.517499           1.19608   0.352941           0.166667         0.0491803                   0.117647        0.176471                   19.6078   -64.7059\n",
       "    12       0.598361                    0.421344           2.9902    0.786463           0.416667         0.109589                    0.294118        0.470588                   199.02    -21.3537\n",
       "    13       0.696721                    0.375781           1.79412   0.92872            0.25             0.129412                    0.176471        0.647059                   79.4118   -7.12803\n",
       "    14       0.795082                    0.31825            1.19608   0.961795           0.166667         0.134021                    0.117647        0.764706                   19.6078   -3.8205\n",
       "    15       0.893443                    0.259895           1.19608   0.987588           0.166667         0.137615                    0.117647        0.882353                   19.6078   -1.24123\n",
       "    16       1                           0.0848338          1.10407   1                  0.153846         0.139344                    0.117647        1                          10.4072   0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for ycol in ['D_NBER','D_NBER_1', 'D_NBER_3', 'D_NBER_6',\n",
    "            'D_NBER_12']:\n",
    "    logistic_h2o = H2OGeneralizedLinearEstimator(alpha=0, family = \"binomial\", intercept=False)\n",
    "    logistic_h2o.train(y = ycol, x = list(train_features.columns), training_frame = train_df)\n",
    "    \n",
    "    predictions = logistic_h2o.predict(test_df)\n",
    "    all_predictions_topics_CFNAI.append(predictions)\n",
    "    name = \"prediction \" + ycol + \"Topics+CFNAI.csv\"\n",
    "    h2o.export_file(predictions_h2o, name, force = True)\n",
    "    \n",
    "    print(logistic_h2o.model_performance(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(test['D_NBER'], np.zeros(test.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUC inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "AUC_topics = [0.8717086834733894,\n",
    "             0.8039215686274509,\n",
    "             0.8005602240896359,\n",
    "             1-0.42735042735042733,\n",
    "             1-0.4358543417366947]\n",
    "\n",
    "AUC_topics_CFNAI = [0.9675070028011205,\n",
    "                   0.8974789915966387,\n",
    "                   0.7803921568627451,\n",
    "                   0.5400641025641025,\n",
    "                   1-0.32717086834733894]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-2.47539236083\n",
      "-2.24058895811\n",
      "0.464798707955\n",
      "0.831845788246\n",
      "-3.09068470254\n"
     ]
    }
   ],
   "source": [
    "for i, ycol in enumerate(['D_NBER','D_NBER_1', 'D_NBER_3', \n",
    "                'D_NBER_6', 'D_NBER_12']):\n",
    "    n0, n1 = test[ycol].value_counts()\n",
    "    AUC_A = AUC_topics[i]\n",
    "    AUC_B = AUC_topics_CFNAI[i]\n",
    "    \n",
    "    sd_A = calc_sd_auc(AUC_A, n0, n1)\n",
    "    sd_B = calc_sd_auc(AUC_B, n0, n1)\n",
    "    \n",
    "    recession_idx = np.where(test[ycol]==1)[0]\n",
    "    \n",
    "    recession_probs_A = all_predictions_topics[0]['p1'] \n",
    "    recession_probs_B = all_predictions_topics_CFNAI[0]['p1']\n",
    "    \n",
    "    corr_A_B = correlate_AUCs(AUC_A, AUC_B,\n",
    "                             recession_probs_A,\n",
    "                             recession_probs_B,\n",
    "                             recession_idx)\n",
    "    \n",
    "    z_stat = calc_z_diff(AUC_A, AUC_B, sd_A, sd_B, corr_A_B)\n",
    "    \n",
    "    print(z_stat[0])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAETCAYAAADnFbcdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu81VWd//HXGxAURlLk1qhAeSUYr4jaRVBDzRwNK7Nk\nlMxpyn6Zl34yVtMwXZRyarQsS/2VTKSmZWreMikUZ2QUHE1HR3CMhEIR0PGCgpfP74+1Dn7Z7HPO\n5nDOWXL2+/l47Mc5e33X9/v9fPf3nM9ee32/ey1FBGZmVkav0gGYmTUzJ2Ezs4KchM3MCnISNjMr\nyEnYzKwgJ2Ezs4KchM3MCnISNjMryEnYzKygPqUDsDIGDx4co0aNKh2GWY+1YMGCFRExpL16TsJN\natSoUcyfP790GGY9lqQ/NlLP3RFmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOw\nmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQk\nbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5\nCZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlB\nTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5Cdtm68ILL2Ts2LGMGTOGCy64YF35d7/7XXbffXfG\njBnD2WefXTBCa0tr56/Z9CkdgFlHPPTQQ1x66aXcc8899O3blyOOOIKjjjqKJUuWcP311/PAAw/Q\nr18/li9fXjpUq6O187fzzjuXDq3bvWlbwpKmSopWHs+Wjq8zSJqYj2di6Vg2N4888gj7778//fv3\np0+fPkyYMIFrr72Wiy++mL//+7+nX79+AAwdOrRwpFZPa+evGb1pk3DFh4EDax7vLRpR57mPdDz3\nlQ5kczN27Fjmzp3LypUrWb16NTfffDNLlixh4cKFzJ07l/33358JEyZw7733lg7V6mjt/DWjzaE7\n4v6IeKx0EF0hIp4D5pWOY3M0evRopk2bxmGHHcaAAQPYa6+96N27N6+++iqrVq1i3rx53HvvvRx3\n3HE8/vjjSCodslW0dv6a0ebQEq5LUi9JcyQtlvSWSvlfSXpJ0vmVssWSZkn6W0mPSXpZ0n2SDq7Z\n5n6Sfi5pad7Go5LOlbRVTb05ku6S9N68ndWSHpI0uaberpJ+KWl53ucTkq6R1Ccv36A7QskZed9r\nJS2TdJGkgTXbDklfk3SapD9Iel7SHZLGdMoLvBn4xCc+wYIFC7jzzjvZdttt2XXXXdlhhx049thj\nkcT48ePp1asXK1asKB2q1VHv/DWjzaEl3LslaVW8HhGvS5oCPAD8EDg+J8urgP8CvlizzkRg31y+\nBpgG3CJpz4h4NNcZCTwI/AR4FhgDfBl4O3B8zfZ2Ai4EzgNWAGcB10javdJyvwl4Bvh0rrM9cCRt\nv/l9HTgH+B7wK+AdwFeBPSVNiIjXK3WnAI8CnwP6AucD1+cYXm1jHz3C8uXLGTp0KE888QTXXnst\n8+bNo1evXvzud7/j4IMPZuHChaxdu5bBgweXDtXqqHf+mlJEvCkfwFQgWnncWKk3OZd9HLgEeB7Y\npWZbi4G1wI6Vsq2BVcBPWtm/SG9SU4DXge0qy+YAr1T3AwwFXgO+kJ8PznEd3cYxTsx1Jubng0hv\nEJfX1JtSu638fBGwRaXsQ7n8ne29vvvuu29s7t797nfH6NGjY4899ojbb789IiLWrFkTJ5xwQowZ\nMyb23nvvmD17duEorTX1zl9PAsyPBnLd5tASngwsrSlbd3dERPxS0g+Bi4F+wMkRsajOduZFxJLK\nes9Luol0YQyA/JH/i6RktiOwRWX9XYCVleeLqvuJiOWSlgMjctFK4HFghqRhwJxW4qo6gNSinVVT\nfhXwY2ACcEOl/DcR8Url+YP55wjg32s3LumTwCcBRowYUbt4szN37twNyvr27cusWbUvn70Z1Tt/\nzWhz6BN+KCLm1zxqL9TNJCXg5cAVrWznqVbKtq88/zHwKeA7wCRgP+AzedmWNeuuqrO9NS318jvh\nJGA+qctioaTHJX26lfggtYQBllULI3UtrKwsby2GNa3E2rKdSyJiXESMGzJkSBthmFl32RyScJsk\n9Qd+BDwEvAWY0UrVYa2U/SlvZ0vgGOD8iLgwIu6IiPnASx2NLSIej4gTgSHA3sBvge9Lel8rq7Qk\n1eHVwtwnvh31E7+ZbcY2+yRMuji2PSmBng18TtLhdeodIGnHlieStgbeD9ydi/oBvUl9vVVTNzXA\n3EV0P3BmLhrbStV5pL7r2ouAHyH1T8/Z1FjM7M1lc+gT3ktSvcvb80mJ9xTgbyLiceA7kg4DZkra\nIyKq31l9CrhN0nTeuDtiAOnOAyLifyXNA86StIx0N8PJrN9d0TBJe5DeIH4GPEZK8FOBV0kt4g1E\nxCpJ3wLOkfQicDMwGvgacBfpbgsz60E2hyR8TSvlI4BLgZ9GRPVKzMeB3wOXS3p/7psFuIPUkjwX\n2AF4GHhfRCysrPtR0gW+75G6Ia4m3f51YwfifhJ4gtT63QF4mXTh7KiIWNDGel8Enib1TZ9K6gv+\nV+CcWP/2NDPrAfRGjuq5JC0G7oqIKaVjebMYN25czJ8/v3QYZj2WpAURMa69ej2hT9jMbLPlJGxm\nVtDm0Ce8ySJiVOkYzMzqcUvYzKwgJ2Ezs4KchM3MCnISNjMryEnYzKwgJ2Ezs4KchM3MCnISNjMr\nyEnYzKygjUrCksZKOiYPpI6kPvJc4mZmHdZQEpY0WNIc0hCR1/LGzA8/AL7dNaGZmfV8jbaE/wVY\nDfxl/tniaqDeLBZmZtaARgfwmQQcFhFP1vQ+LOKN2YXNzGwjNdoSHsD6LeAW25HmRDMzsw5oNAnf\nBVRnpYh8Qe7zePJJM7MOa7Q74mzgDkn7An2B84AxwFuBd3ZRbGZmPV5DLeGIeBDYgzQ55lxgKPBr\nYO+aiTLNzGwjNNQSljQ0IpaSpomvt2x5ndXMzKwdjfYJL5M0tLZQ0nbAss4NycyseTSahFv7VtwA\n4OVOisXMrOm02R0h6Zv51wC+LKl6m1pv4ADgwS6Kzcysx2uvT/g9+adICfeVyrK1wGPAjC6Iy8ys\nKbSZhCPiQABJVwJ/FxHPdUtUZmZNoqG7IyLio10diJlZM2r0yxpIeidwPGmsiL7VZRFxZCfHZWbW\nFBodyvJjpK8n7wi8j9QfPIr0bbk/dVFsZmY9XqO3qJ0DfC4iJpMS8Jmkry1fDTzZRbGZmfV4jSbh\ntwO35N/XAgMiIkjjDJ/SFYGZmTWDRpPwM8DW+fc/Ae/Iv7+F9IUNMzPrgEYvzN0FHEL6YsYvgAsl\nTSTNqvHbrgnNzKznazQJnwZslX//Wv75LuBm4B87Oygzs2bR6H3Cyyu/vwb8U5dFZGbWRBq9RW21\npCF1ygfVjCdhZmYbodELc1tSfyS1LTdiG2ZmVqO9UdROzb8GMFXSC5XFvYEJgGfWMDProPb6hP8h\n/xRwFvB6ZdlaYDFwKmZm1iHtjaL2VgBJdwNHRsQz3RKVmVmTaHSizwNrE7CkHSQ1PACQmZltqNG7\nI6ZLmlJ5fhPwBPCkpHFdFZyZWU/X6J0NU4H/AZB0OGmWjYnANXhmDTOzDmu0O2E4sDT/fiRwTUTc\nKWkZcE+XRGZm1gQabQmvAnbIvx8OzK6s37uzgzIzaxaNtoSvA2ZJegQYCtyay/ckd1OYmdnGa7Ql\nfDrwI9IwlkdExPO5fCRwSVcEZmbWDBodwGct8PU65ed3ekRmZk3E4z6YmRXkJGxmVpCTsJlZQU7C\nZmYFbVQSlvQXkvaUtEVXBWRm1kwaHTtigKR/BZ4DFgA75vKLJH2xC+MzM+vRGm0JnwfsBrwTeLlS\nfhvw4c4OysysWTT6jbljgOMi4j8kRaX8YeDtnR+WmVlzaLQlPARYXqd8QCfGYmbWdBpNwgtIo6e1\naGkNnwzc3akRmZk1kUa7I74I3Cxp97zOZySNIY0pPKGLYjMz6/Eand7oTlKyHUoaxOdY4EXgXRHh\n8YTNzDqo4TniImIB8JEujMXMrOk0lIQl9W9reUSs7pxwzMyaS6Mt4Rd442JcPZ5dw8ysAxpNwu+r\neb4FsDdwCvAPnRqRmVkTaXRQ91/XKb5R0kJgCvCvnRqVmVmT2NRR1OYDh3RGIGZmzajDSVhSX+Az\npFvWzMysAxq9O+Jp1r8wJ2AbYC1wYhfEZWbWFBq9MPelmuevA08D/x4R9caUMDOzBrSbhCX1AV4B\nbo6IJ7s+JDOz5tFun3BEvApcBPTr+nDMzJpLoxfm7gH27MpAzMyaUaN9whcB35L0l6RhLV+sLoyI\nhzs7MDOzZtBoEr46//x+/tlyp4Ty7/7asplZBzSahEd3aRRmZk2qzSQs6UfA5yLi0W6Kx8ysqbR3\nYe4kYKvuCMTMrBm1l4TVLVGYmTWpRm5Ra2scYTMz2wSNXJh7Umq7QRwRvjvCzKwDGknCnwSe7epA\nzMyaUSNJ+FcepMfMrGu01yfs/mAzsy7kuyPMzApqszsiIjZ1+iMzM2uDk6yZWUFOwmZmBTkJm5kV\n5CRsZlaQk7CZWUFOwmZmBTkJm5kV5CRsZlaQk7CZWUFOwtYjPProo+y1117rHgMHDuSCCy4oHZbV\nOPnkkxk6dChjx45dV7Zq1SomTZrELrvswqRJk3jmmWcKRtj9nIStR9htt924//77uf/++1mwYAH9\n+/dn8uTJpcOyGlOnTuXWW29dr2zGjBkceuihLFq0iEMPPZQZM2YUiq6MIklY0oGSrpb0Z0lrJa2U\n9BtJJ0rq9AHiJU2UNF1SqeM9XdKxdcqnS/JIdZ1s9uzZ7LTTTowcObJ0KFbjoIMOYtCgQeuVXX/9\n9Zx00kkAnHTSSVx33XUlQium25OSpNOBfwMGAdOA9wInAwuBHwBHdcFuJwL/SLmW/+nABkkYuAw4\nsJtj6fGuuuoqPvrRj5YOwxr01FNP8da3vhWA4cOH89RTTxWOqHs1Mqh7p5F0EPBt4KKIOK1m8fWS\nvgX8RXfGVEvSFsCrEdHlLdSIWAos7er9NJO1a9dyww03cN5555UOxTpAEu1Np9bTdHfLcBqwCji7\n3sKIeDwifg8gabyk2yW9IOlFSbMlja/Wl3S5pKWS9pY0V9JqSYskfapSZzqpFQzwiqRo6QKQNCo/\nP1XSNyX9GVgDbCNpiKQfSlqYt7tE0hWStq+NW9Kekn6Zu1VekvSopHPyssXASOCEln1Lurwlttru\nCEkDJV2Uu2rW5G2docpfZu5eCUlH57or8mOWpG024nz0OLfccgv77LMPw4YNKx2KNWjYsGEsW7YM\ngGXLljF06NDCEXWvbkvCua/3YOC2iHi5nbp7AHcA2wJTgROBgcAdkvasqT4QuAKYBRwD3AtcLOng\nvPwy4P/l399N+vhf2wXwRWBX0nx6k4GXSd0la4EvAe8D/i+wC/BvkrasxDoeuBvYCTgDeD+ptb9D\nrjIZeBL4dWXfX23luHsBNwEfB74F/DVwa97e1+usciFp9pOPAf8EfDCXNa0rr7zSXRGbmaOPPpqZ\nM2cCMHPmTI455pjCEXWziOiWBzCMlDDOa6Duz0mTi25TKRtIakVfWym7PG/z4EpZP2AlcEmlbHqu\n16dmP6Ny+X2A2ompN7Bjrj+5Un4nsATo38a6i4FZdcqnp1Ow7vlReftTa+pdRmqhD87PJ+Z6M2vq\nXUR6A6l7LKQ3mfnA/BEjRkRP88ILL8SgQYPi2WefLR2KteL444+P4cOHR58+fWL77bePyy67LFas\nWBGHHHJI7LzzznHooYfGypUrS4fZKYD50UBu7NY+4Y1wEHBjRKyb5TkinpN0A6l1WLU6In5XqbdG\n0kJgxEbs77r8oq1H0qeBT5FauQMqi3bLy/sD7wLOj4jVG7G/1hwEvE5q2VfNAj5BakX/qlJ+U029\nB0lvQsNIre/1RMQlwCUA48aN63F3ZQwYMICVK1eWDsPacOWVV9Ytnz17djdH8ubRnX3CK4GXSP2j\n7RkELKtT/iSpi6Kq3p3da4At65S3ZoN9Sfos8H3gdtKdDeOBA/Lilm1vS3oNO+vi2iBgVUSsrSl/\nsrK8alXN8zU18ZnZm1y3tYQj4lVJc4BJkvpFxJo2qq8ChtcpH079pLvJ4dUpOx6YHRFntRRIeltN\nnWdILdcNLtZ10CpgkKS+NYl4eGW5mfUg3X13xAxgO+Cb9RZKelvlotyRkrauLNua1BUxpwP7bUn4\nW23EOv2BV2rKPl59krsg7gKmSGpr22sa3PcdpHPy4ZryE0gXCe9uYBtmthnp1j7hiLhT0pnAtyW9\ng3Rh7QnSx/pDgVNIV/q/SrpINVvSN0gt1WmkxPiVDuz64fzzLEm3AK9FxPx21rkVmCbpC8A9wCHA\nh+rU+zwped6d73NeCrwd2CsiPlvZ/3skHUXqWlgREYvrbOsWUlL/gaQhwH8BR5Jel/MiYkVDR2tm\nm41uvzAXERdIuod0O9c/A4OB50lX7f8O+FVEvC5pIum2rJmAgHnAhIh4oAO7vZHUv3sq8OW8vfbu\nCP8KsE2Oc0tSoj0ceLzmeO6V9K5c/7ukC2N/BH5cqXYOcClwNalFPJN069168nG/HziX9KazHenO\nijMBj0Zj1gOpzk0B1gTGjRsX8+e392HAzDpK0oKIGNdePY+iZmZWkJOwmVlBTsJmZgU5CZuZFeQk\nbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5\nCZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlB\nTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZW\nkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgU5CZuZFeQkbGZWkJOwmVlBTsJmZgUpIkrH\nYAVIehr4Y+k4usBgYEXpIGyj9NRzNjIihrRXyUnYehRJ8yNiXOk4rHHNfs7cHWFmVpCTsJlZQU7C\n1tNcUjoA22hNfc7cJ2xmVpBbwmZmBTkJ20aRNFVStPJ4tnR8nUHSxHw8E0vH0lGSDpR0taQ/S1or\naaWk30g6UVLvLtjfREnTJRXJKZJOl3RsnfLpkt7UH/f7lA7ANlsfBpbWlL1aIpAucB9wIPBw6UA6\nQtLpwLeB3wLTSPeDbwscBvwA+F/g+k7e7UTgH4GvAa938rYbcTpwF3BtTfllwK3dH07jnISto+6P\niMdKB9EVIuI5YF7pODpC0kGkBHxRRJxWs/h6Sd8C/qL7I3uDpC2AV6MbLkhFxFI2bCy8qbg7wjqV\npF6S5khaLOktlfK/kvSSpPMrZYslzZL0t5Iek/SypPskHVyzzf0k/VzS0ryNRyWdK2mrmnpzJN0l\n6b15O6slPSRpck29XSX9UtLyvM8nJF0jqU9evkF3hJIz8r7XSlom6SJJA2u2HZK+Juk0SX+Q9Lyk\nOySN6ZQXuH3TgFXA2fUWRsTjEfH7HOt4SbdLekHSi5JmSxpfrS/p8vy67y1pbn5NF0n6VKXOdFIr\nGOCVlu6pvGxUfn6qpG9K+jOwBthG0hBJP5S0MG93iaQrJG1fG7ekPfM5W1n5GzgnL1sMjAROqHSN\nXd4SW213hKSB+dz9WdKavK0zJKlSp+Vv4Ohcd0V+zJK0zUacj/ZFhB9+NPwApgIB7Eb6JFV99Mp1\ndgBWAlfl51sB/wXMB/pWtrWY1Ep5BPgI8AHgbuBlYLdKvQ+R/smPASYApwJPtmy/Um8OsCzvawpw\nBPAbUjfJzpV6i4B7gA/m7X0MmNUSG+mjdQATK+ucm8suAg4HzgBeAOa2HHeuF/m4fg0cnWP/A/AY\n0KeLz01vYDVwRQN19wBeAhbkGD8I3JvL9qzUuxx4Lp+jvwMmAVfk4zy4cr4vy2XvAg4ADsjLRuXy\nPwHXAUfl87hV/hv6LnBcPg/H5xgWA1tWYhifj+v3wInAITmW7+Xle+fzfmvLvoGd8rLpQFS21Suf\nsxeBs0hdNBfmGM+t1Gv5G/hDjvEw4LP59ZnZqeet9D+1H5vXgzeScL3HjZV6k3PZx0n3gT4P7FKz\nrcXAWmDHStnWpJbcT1rZv0gJfwqp73G7yrI5wCvV/QBDgdeAL+Tng3NcR7dxjC3/gBPz80Gk1tvl\nNfWm1G4rP18EbFEp+1Auf2cXn5theT/nNVD358CzwDaVsoH5tb+2UnY5lYSby/qR3mQvqZRNz/X6\n1OxnVC6/j3xLbBsx9QZ2zPUnV8rvBJYA/dtYdzEwq075dNZPwkfl7U+tqXdZPseDa/4GZtbUu4jU\nSGjzWDbm4e4I66jJwH41j9NbFkbEL4EfAhcDfwucFhGL6mxnXkQsqaz3PHAT6cIYsO7j4zck/Q/p\nH+UV4CekhLxLzfYWVfcTEcuB5cCIXLQSeByYkbtBatev5wCgL6m1XHUVqZU9oab8NxHxSuX5g/nn\nCN48DiK9aa67oyVSX/gNbHg8qyPid5V6a4CFbNzxXBc5i1VJ+rSkByS9QHotn8iLdsvL+5Na1z+N\niNUbsb/WHER6876ipnwW6RwfWFN+U83zB0lvQsM6IRbAfcLWcQ9FxPyaR+2FupmkP9jlbPhH3+Kp\nVsqq/YI/Bj4FfIf0cXg/4DN52ZY1666qs701LfVyIphE6ho5D1go6XFJn24lPkgtYUgfedeJiFdJ\nSX1QTf3aGNa0EmtnW0n6uDyygbqDqDme7EnSnRRVz9Spt+41bdAG+5L0WeD7wO3AsaRuhwPy4pZt\nb0vKU511cW0QsCoi1taUP1lZXtXl59JJ2LpEbsH8CHgIeAswo5Wq9VoUw0h9iEjaktSHeH5EXBgR\nd0TEfFKy6ZBIF6dOBIaQ+hN/C3xf0vtaWaXlH3F4tTBfyNuO+om/2+U3hTnAJEn92qm+iprjyYZT\nP+luqnp3QhwPzI6IsyLitoi4l/SGXfUMqeW6wcW6DloFDJLUt6Z8eGV5t3IStq5yIekf5xjSlfrP\nSTq8Tr0DJO3Y8kTS1sD7SRfoILWke5O6IKqmbmqAkdwPnJmLxrZSdR6p7/r4mvKPkPqn52xqLJ1o\nBumN4Zv1Fkp6m6Q9gDuAI/Pr3bJsa+Cv6djxtLQQt2qz1vr6s+F5/Xj1Se6CuAuYUns3TJ39N7Lv\nO0h578M15SeQzvHdG6zRxXyfsHXUXpIG1ymfT0q8pwB/ExGPA9+RdBgwU9IeuZ+2xVPAbfk2pzWk\nW6wGAF8FiIj/lTQPOEvSMtLg3yfTwZZRTkAXAj8j3bHQm5TQXyW1iDcQEauU7q89R9KLwM3AaNIX\nE+5iw37DYiLiTklnAt+W9A7ShbUnSB/rDyWdl4+RXt+jgNmSvkFqqU4jJcavdGDXLV9sOUvSLcBr\n+RNLW24Fpkn6AululUNIFzFrfZ6UPO/O52Ep8HZgr4j4bGX/75F0FKlrYUVELK6zrVtI5+wHkoaQ\n7qQ5kvS6nBcR3T+4fGdd4fOjOR60fXdEkK5ur6LmSjXpo/8yUgJrGThqMemCyClAy0W3/wQOqVl3\nFOmf53nSx9WLSK3l2tvI5gB31Yl5MfnOBtLdEjNJF5ZW51jvAA6v1J9YZ9si3Zb2KKnFtAz4HjCw\nZl8BfK1O/Btcke/i8/RO4Joc5yv5OG8j3dHRcivh/qT+2BdIt2zNBsbXbOdyYGmd7c8B5lSe986v\nx3JS90HUHPspdbaxFenC7dP53N4IvC3Xn15Td2/gV6Q7Ol4C/huYVlm+O+nWs9V5/ZbzPb0llkrd\ngflvaFk+lwvzuVWlTsvfwHtb+fsf1VnnyqOoWTH5Jvu7ImJK6VjMSnGfsJlZQU7CZmYFuTvCzKwg\nt4TNzApyEjYzK8hJ2MysICdhs06gNG7x9MrzxZI+XyCOcXkc3FHdvW/rGCdh65HyYOQtA3y/kgfp\n+WdJA7ophP1Ig9O0S2nevhe6OJ6G5EHQH+rAem+aY9jc+GvL1pPdDvwNsAXwHtKYsf1Jg8JvQNIW\nsf4QlB0WEU93xnas53NL2HqyNRHxZEQsiYgrSF+R/gCsN33NkZLukbSWNGMGkv5a0gKlqY/+IOnr\n1VG3JA2VdH2eZuePkk6u3XFtd4Skt0i6WGlapJclPSLpI0pTKP0YGFBpuU/P6/TN4ygvVZr+597a\nQZAkHSHpv/M25wK7tveiSDpW0u9z/KuUpl8aJmkqaQaTMZVYpuZ1zszrvCjpT5Iua5nmZ1OPodm5\nJWzN5GXSqGxV3yBNc/MY8HxOED8FPkea0WEEaYbifqSBZCCNpzASeC9prIJ/IY2RUJckkcbM2JY0\nStijpMHo+wP/ThoM/1xgp7xKy8f6H+eyj5EGrTkS+JWk/SLigTz63HXApaRxG/YgTfLZKknDSYPR\nnwP8gjTpZ8sYvj8jjSR3FGnsBEgzM0MaD+J00oD4I0lT/nyX9Emjw8fQVqxNo7sGFPHDj+58kBJl\ndbql8aRBz3+Wn08kDcTywZr17gT+oabsA6SkIlJLM4B3VZaPJE2hNL1Sthj4fP59EimJjW4l1qnA\nCzVlO+V1RtSUXwd8P/9+LmnwmerAM1+ijQFmgH3y8pGtLJ9OGrC/vdf3CNKAS7025Rj8CLeErUc7\nIl8s6kPqF76eNFljVe1wi/sC4yVNq5T1Io34NZw0hOXrpKEXAYiIPyrNItyavYFlEfHIRsS+Dynp\nP6w3JgGG1CJvGXJzNGl6qOrXXtsbD/cBUl/5Q5Juy7//PNrpw5Z0CKn1PJo0SH9v0nRAw4HWjr2R\nY2h6TsLWk90JfJI0lOOfo/5FtxdrnvcC/ok0DGStaqLq6u/798r72I8NBz7flFlFXstjOx9AmkH4\nE8B5kiZEK90DkkaSxky+FPgy6RPFPsCVpETcrcfQ0zgJW0+2Ojac96499wG7t7aepP8mJZfxpL5Q\nJI0A/rLyl4dtAAABn0lEQVSNbf4n8FZJo1tpDa8ltSxr1xEwPCqTbNZ4BPigJFVawwe0UnedXPdu\n0iDpXyENbP4RUiu5XizjSMn2jIh4DSAPnt4Zx9D0fHeE2fq+AnxM0lckjZW0u6QPSfomQEQ8SpoR\n4oeSDpS0F6n/ua2W3WzgP4BfSDpcaYqhSZI+kJcvBrbMZYMl9Y+IhaQLhJfn/b89fxHj85KOzev9\ngHRB8AJJu0n6EGlC1FZJOkDSlyTtl988jiYNxN8yM8ZiYKSkfXIs/YBFpFxxeo79o1Rm1t7EY7DS\nndJ++NEVD2ouzNVZPpH0UXlwnWWH8cYsDc+R+o3/T2X5MNLU8C8BS0gzgzxEKxfm8vNtSB/nnybd\npfEwcFxl+cWkqZvWzSpB6seeTrojYS1p2p4bgH0r672fdLfFy8C/keZKa+vC3GjSLCVPkS6sPQac\nXVneD/g5aYLNdbOBAKeRJl99ifSmclztfjp6DM3+8FCWZmYFuTvCzKwgJ2Ezs4KchM3MCnISNjMr\nyEnYzKwgJ2Ezs4KchM3MCnISNjMryEnYzKyg/w9oZZFBjbm1NwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1184c9eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_confusion_matrix(cnf_matrix, class_labels, title='Confusion Matrix', cmap=plt.cm.Blues):\n",
    "    import itertools\n",
    "        \n",
    "    fig, ax = plt.subplots(nrows=1)\n",
    "    ax.imshow(cnf_matrix, interpolation='nearest', cmap=cmap, alpha=0)\n",
    "    #plt.axis('off')\n",
    "    tick_marks = np.arange(len(class_labels))\n",
    "    ax.set_xticks(tick_marks)\n",
    "    ax.set_yticks(tick_marks)\n",
    "    \n",
    "    ax.set_xticklabels(class_labels, fontsize=16)\n",
    "    ax.set_yticklabels(class_labels, fontsize=16)\n",
    "    \n",
    "    for i, j in itertools.product(range(cnf_matrix.shape[0]), range(cnf_matrix.shape[1])):\n",
    "        ax.annotate(str(cnf_matrix[i, j]), xy=(j, i), horizontalalignment='center')\n",
    "        \n",
    "        \n",
    "    ax.set_ylabel('True state', fontsize=14)\n",
    "    ax.set_xlabel('Predicted state', fontsize=14)\n",
    "    # ax.set_title(title, fontsize=20)\n",
    "    ax.grid(False)\n",
    "    \n",
    "cnf_matrix = confusion_matrix(test['D_NBER_3'], \n",
    "                              all_predictions_topics[2]['p1'].as_data_frame() > 0.02)\n",
    "\n",
    "plot_confusion_matrix(cnf_matrix, ['Expansion', 'Contraction'])\n",
    "plt.savefig('confusion_matrix.png', format='png', dpi=600, transparent=False,\n",
    "           bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmczfX+wPHX29jXhIQhE7KPse8i2V0UlRKp5FaW0nVL\ndYu2X3W79yoiWQol2mSJiLIkSZR9KctgkGUwdmZ5//74HqczzHKGOXPmnPN+Ph7nYb7f7+ec7/t7\nZpz3+X4/n+/7I6qKMcYYA5DD3wEYY4zJPiwpGGOMcbOkYIwxxs2SgjHGGDdLCsYYY9wsKRhjjHGz\npGCMMcbNkoIJKiISLSLnROS0iPwpIpNFpOBlbZqIyPcickpE4kRkrohUu6xNYRF5W0T2ul5rp2u5\neCr7FREZLCKbROSMiMSIyOciUtOXx2tMZrOkYILR31S1IBAF1AaevbRBRBoD3wKzgdJABLAe+FFE\nbna1yQ18B1QH2gOFgcbAUaBBKvt8B3gCGAxcD9wCzAI6ZTR4EcmZ0ecYk1nE7mg2wUREooF+qrrY\ntfxvoLqqdnIt/wBsVNXHL3veN8ARVe0jIv2A14AKqnrai31WArYBjVV1dSptlgIfq+pE13JfV5zN\nXMsKDASeBHICC4AzqjrU4zVmA8tU9X8iUhoYDbQATgMjVXWUF2+RMWmyMwUTtEQkHOgA7HAt5wea\nAJ+n0PwzoI3r59uBBd4kBJfWQExqCSEDugENgWrAdOAeEREAESkKtAVmiEgOYC7OGU4Z1/6fFJF2\n17h/YywpmKA0S0ROAfuAw8Bw1/rrcf7mD6bwnIPApf6CYqm0SU1G26fmdVU9pqrngB8ABZq7tvUA\nflLVA0B9oISqvqyqF1V1FzAB6JkJMZgQZ0nBBKNuqloIaAlU4a8P++NAElAqheeUwukzAIhNpU1q\nMto+Nfsu/aDOdd0ZwL2uVfcB01w/3wSUFpETlx7Ac0DJTIjBhDhLCiZoqeoyYDLwH9fyGeAn4K4U\nmt+N07kMsBhoJyIFvNzVd0C4iNRLo80ZIL/H8o0phXzZ8nSgh4jchHNZ6UvX+n3AblW9zuNRSFU7\nehmvMamypGCC3dtAGxGp5VoeBjzgGj5aSESKisirOKOLXnK1+Qjng/dLEakiIjlEpJiIPCciV3zw\nquofwFhguoi0FJHcIpJXRHqKyDBXs3XAnSKSX0QqAg+nF7iq/oZz9jIRWKiqJ1ybVgOnROQZEckn\nImEiUkNE6l/NG2SMJ0sKJqip6hFgKvCia3kF0A64E6cfYA/OsNVmrg93VPUCTmfzNmARcBLng7g4\n8HMquxoMvAuMAU4AO4E7cDqEAUYCF4FDwBT+uhSUnk9csXzicUyJQGecIbe7+StxFPHyNY1JlQ1J\nNcYY42ZnCsYYY9wsKRhjjHGzpGCMMcbNkoIxxhi3gCu8Vbx4cS1fvry/wzDGmICydu3ao6paIr12\nAZcUypcvz5o1a/wdhjHGBBQR2eNNO7t8ZIwxxs2SgjHGGDdLCsYYY9wsKRhjjHGzpGCMMcbNZ0lB\nRD4QkcMisimV7SIio0Rkh4hsEJE6vorFGGOMd3x5pjAZZ9Lz1HQAKrke/YH3fBiLMcYYL/gsKajq\ncuBYGk26AlPVsQq4TkQyY/YqY4wJLonxcHRHluzKnzevlcFj+kEgxrXuirluRaQ/ztkE5cqVy5Lg\njMnuar30LXHn4v0dhvGx6hLNm7nGU1ziaHXhv2x9o7tP9xcQdzSr6nhgPEC9evVsAghjgLhz8US/\n0cnfYRhfSkqCsY3g3FnoNIqt1br4fJf+TAr7gbIey+GudcYYE9r2rYaS1SF3AbhrMhQuBfmKZsmu\n/TkkdQ7QxzUKqREQp6pXXDoyxpiQceEUzBsKk9rAytHOupLVsiwhgA/PFERkOtASKC4iMcBwIBeA\nqo4D5gMdgR3AWeBBX8VijDHZ3o7FMPdJiIuBho9C44F+CcNnSUFV701nuwIDfLV/Y4wJGMvegiWv\nQvFb4KGFUK6h30IJiI5mY4wJSonxEJYLKrWBhPPQ4p+QK69fQ7KkYIwxWe3UnzB/KOS9Drq+C6Wj\nnEc2YLWPjDEmq6jCbx/DmAbw+7dQrIKzLhuxMwVjjMkKJ/bCnMGwawmUawJdRkPxiv6O6gqWFIwx\nJivEn4c/N0Cn/0LdhyBH9rxQY0nBGGN85ch22DQTWj0LJW6BIZshVz5/R5UmSwrGBBDPekdF8uXy\nczQmVYnx8OPbsOzfzl3JdR+AwqWzfUIASwrGBBSrdxQADvwGswfCoU1Q/U7o8G8oWMLfUXnNkoIx\nxmSWC6dhajfImRd6fgJVAi+BW1IwxphrdXAD3FgT8hSEez6CGyMh33X+juqqZM/ub2OMCQTnT8LX\nT8H7zWHjF866iBYBmxDAzhSMMebq/P4tfD0ETu6HRo9DlY7+jihTWFIwxpiM+uYZ+HkclKgCDy+C\nsvX9HVGmsaRgjDHeuFSOQgTC60PeItD8H5Azj3/jymSWFIwxJj0nD8K8p5z+gkaPQc0e/o7IZ6yj\n2RhjUqMKa6fAmIaw83uQMH9H5HN2pmCMMSk5thvmDobdy+GmZtBllFPVNMhZUggSnuUPTPCy0hZZ\n6OjvcGAddH4b6jyQbQvYZTZLCkHCyh8YkwkOb3XKVETdB7e0gyfWQ/7r/R1VlrKkYIwxCRdhxUhY\n/hYUKA7VukHu/CGXEMCSgjEm1O1fC7MHweHNUKMHdHjTSQghypKCMSZ0Hd8Dk9pCgRvg3hlQuYO/\nI/I7SwrGmNBzbDdcHwFFb4Ju4+CWts7NaMbuUzDGhJDzcTD3CRhdF2LWOusi77KE4MHOFIwxoWH7\nAqeA3ek/ofEAuKGqvyPKliwpGGOCmyp89ShsmAE3VIN7Pobwuv6OKtuypGCMCW4iTt9By+eg2RDI\nmdvfEWVrlhSMMcEnbj/M+wc0/DtUaAWtnvN3RAHDOpqNMcEjKQnWfAhjG8GupXDygL8jCjh2ppCN\nZaSekdXEMSEvdqczsij6B6fE9d9GOcNOTYZYUsjGrJ6RMRmwdQ4cXO8kgzp9nL4Ek2E+vXwkIu1F\nZLuI7BCRYSlsLyIic0VkvYhsFpEHfRmPMSbIHNrsXCYCaDwQBqyGug9YQrgGPksKIhIGjAE6ANWA\ne0Wk2mXNBgBbVLUW0BL4r4jY0ABjTNoSLsCS/4P3W8DC551hp2G5oHApf0cW8Hx5+agBsENVdwGI\nyAygK7DFo40ChUREgILAMSDBhzEZYwLdvl9gzkA4sg0i74H2b9iZQSbyZVIoA+zzWI4BGl7W5l1g\nDnAAKATco6pJl7+QiPQH+gOUK1fOJ8EaYwLA3lXwQXsoXBru+9ypWWQylb+HpLYD1gGlgSjgXREp\nfHkjVR2vqvVUtV6JEiWyOkZjjL+dOer8G94A2r4Cj6+yhOAjvkwK+4GyHsvhrnWeHgRmqmMHsBuo\n4sOYjDGB5NwJmDPIKWB36k9nSswmgyDvFd8dTSbxZVL4BagkIhGuzuOeOJeKPO0FWgOISEmgMrDL\nhzEZYwLFtnkwpiH89rEzosgqmWYJn/UpqGqCiAwEFgJhwAequllEHnVtHwe8AkwWkY2AAM+o6lFf\nxWSMCQDx52HWY7B5JpSsAfdOhzJ1/B1VyPDpzWuqOh+Yf9m6cR4/HwDswqAx5i8580BSPLT6FzR7\n0hlqarKMvzuajTEG4mLg097OjGgicPdHcOs/LSH4gZW5yGY86x1ZPSMT9JKSYO0HsGg4aJJz38H1\nEXbfgR9ZUshmrN6RCRlHdzgji/auhJtbwd/ehqLl/R1VyLOkYIzxjx/+C4c3Q9exEHWfnR1kE5YU\njDFZ58+NEJYbSlSGdq/B7cOh0I3+jsp4sI5mY4zvJVyA71+F8S1h0YvOuvzXW0LIhuxMwRjjW/tW\nw+yBcHQ71LoX2v2fvyMyabCkYIzxnc2z4PO+UCQcen0JlW73d0QmHZYUjDGZ78JpyFMQKtwGzZ+C\nZkMgTyF/R2W84FWfgojkFpGKvg7GGBPgzh2HWQNg4u1OP0LewtD6RUsIASTdpCAinYCNwCLXcpSI\nfOXrwIwxAWbLHKeA3frpULmDv6MxV8mby0cv40yOswRAVdfZWYMxxu3ccZgzGLbOgRtrQq/PoVQt\nf0dlrpI3SSFeVU9I8htL1EfxhAzPchaerLSFCTg588KxXc5loiaDrV5RgPMmKWwVkbuBHCISAQwG\nVvk2rOBn5SxMQDuxF5a9CR3+DbkLQP9lEGbjVoKBNx3NA4G6QBIwE7gAPOHLoIwx2VRSEvw8HsY0\ngk1fwYF1znpLCEHDm99kO1V9Bnjm0goRuRMnQRhjQsWR350CdvtWQYXWTgG768r5OyqTybw5U/hX\nCuuez+xAjDHZmCrMGQhHtkG39+D+Ly0hBKlUzxREpB3QHigjIv/z2FQY51KSMSbYHVwPRco6dYq6\njnXuNyhU0t9RGR9K60zhMLAJOA9s9nh8C9ggZGOCWfx5WDwCxrdyOpQBile0hBACUj1TUNXfgN9E\nZJqqns/CmIwx/rTnJ+dSUewOiLofWg7zd0QmC3nT0VxGRF4DqgF5L61U1Vt8FpUxxj9WT4D5/4Tr\nykLvr5zaRSakeNPRPBn4EBCcy0afAZ/6MCZjTFZLdN1IWeE2aPQ4PPaTJYQQ5U1SyK+qCwFUdaeq\n/gvrUzAmOJw9Bl89Cl886CwXqwDt/8+pcGpCkjeXjy6ISA5gp4g8CuwHrOShMYFMFbbMhvlDndpF\nzYZAUiLkCPN3ZMbPvEkKQ4ACOOUtXgOKAA/5Mqhg5VnvyGocGb85fRi+HgLbvoZSUU7fwY01/R2V\nySbSTQqq+rPrx1NAbwARKePLoIKV1Tsy2UJivDNFZpuXodEAK1FhkkmzT0FE6otINxEp7lquLiJT\ngZ/Tep4xJps5Hg3fvexcNipSBp7cAE2fsIRgrpBqUhCR14FpQC9ggYiMwJlTYT1gw1GNCQRJibDq\nPRjb2Clkd/QPZ32ufP6Ny2RbaX1N6ArUUtVzInI9sA+oqaq7siY0Y8w1ObzNKWAXsxoqtnEK2BUJ\n93dUJptLKymcV9VzAKp6TER+t4RgTIBIjIeP74T4c3DnBKh5FySfKMuYFKWVFG4WkUvlsQWI8FhG\nVe9M78VFpD3wDhAGTFTVN1Jo0xJ4G8gFHFXVW70P3xiTzKHNUKKKM/tZ90lQrCIULOHvqEwASSsp\ndL9s+d2MvLCIhAFjgDZADPCLiMxR1S0eba4DxgLtVXWviNyQkX0YY1ziz8HS12Hlu9D+DWjYH25q\n7O+oTABKqyDed9f42g2AHZcuOYnIDJx+ii0ebe4DZqrqXtc+D1/jPo0JPdErYM5gOLYT6vSByLv9\nHZEJYN6UubhaZXA6py+Jca3zdAtQVESWishaEemT0guJSH8RWSMia44cOeKjcI0JQEteh8mdQBOh\nz2zoMhryXefvqEwA8/cg5Zw48z+3BvIBP4nIKlX93bORqo4HxgPUq1dPszxKY7IbVafjOLyecwPa\nbc9D7gL+jsoEAa+TgojkUdULGXjt/UBZj+Vw1zpPMUCsqp4BzojIcqAW8DvGmCudiYUFw+D6CGj1\nHFRq4zyMySTpJgURaQBMwql5VE5EagH9VHVQOk/9BagkIhE4yaAnTh+Cp9nAuyKSE8gNNARGZuwQ\nsjerd2QyhSpsngnzn4bzJ2ziG+Mz3pwpjAI6A7MAVHW9iLRK70mqmiAiA4GFOENSP1DVza5Kq6jq\nOFXdKiILgA048z5PVNVNV3ks2ZLVOzLX7ORBmPcUbJ8PpWtD1zlQsrq/ozJBypukkENV90jyG18S\nvXlxVZ0PzL9s3bjLlt8C3vLm9YwJScejYddSaPsqNHzM6hUZn/Lmr2uf6xKSuu49GIRd8zfGt47t\nhl1LoN5Dzv0GQzZD/uv9HZUJAd4khcdwLiGVAw4Bi13rjDGZLSkRfh4H370COXNDtW5OMrCEYLKI\nN0khQVV7+jwSY0LdoS0wZyDsXwu3tIdO/7NkYLKcN0nhFxHZDnyKc/fxKR/HZEzoORMLE1s7Ja27\nT4Ia3a2AnfELb2ZeqyAiTXCGlL4kIuuAGao6w+fRGRPsjkdD0fJQoBh0ew/KN3d+NsZPvCpzoaor\nVXUwUAc4iTP5jjHmal08Cwufh1G1Yef3zrrq3SwhGL/z5ua1gjiF7HoCVXFuOGvi47iMCV67f3Am\nvzm+G+o+CGXq+jsiY9y86VPYBMwF/q2qP/g4HmOC2zfD4Of3oGgEPPA1RDT3d0TGJONNUrhZVZN8\nHkkQ8CxpcYmVtjDJFAmHJoOg5XOQO7+/ozHmCqkmBRH5r6r+A/hSRK6oTOrNzGuhxkpamCucOQrf\nPANVO0P1O6DJQH9HZEya0jpT+NT1b4ZmXDPG4BSw2/gFfPM0XDjllLg2JgCkNfPaatePVVU1WWJw\nFbq71pnZjAlOcfudAna/L4Ay9aDru3BDVX9HZYxXvBmS+lAK6x7O7ECMCRo7FsHu5dDudXj4W0sI\nJqCk1adwD84w1AgRmemxqRBwwteBGRNQYnc6j1vaQu0+UPF2p1PZmACTVp/CaiAWZ8a0MR7rTwG/\n+TIoYwJGYgKsGgtLXoMCN0CFXyEslyUEE7DS6lPYDezGqYpqjLncn5ucAnYHfoPKnaDTf52EYEwA\nS+vy0TJVvVVEjgOeQ1IFUFW18o0mdB3eCuNvhbzXQY8PneGmVsDOBIG0Lh9dmnKzeFYEYkxAOBPr\n1CcqUQXavAKR91i9IhNUUh195HEXc1kgTFUTgcbA34ECWRCbMdnHxTOw4Dl4J9LpUBaBxo9bQjBB\nx5shqbNwpuKsAHwIVAI+8WlUxmQnu5bC2MawaozrzKCEvyMyxme8qX2UpKrxInInMFpVR4mIjT4y\nwS8pEeY+Ab99BNdXgL7zoXxTf0dljE95NR2niNwF9Aa6udbZEAsT/HKEOeUqmj4JLYc5s6IZE+S8\nvaO5FU7p7F0iEgFM921YxvjJ6cPwxcPOcFNwSlS0eckSggkZ6SYFVd0EDAbWiEgVYJ+qvubzyIzJ\nSqqw/lMY0wC2zoE/NzjrbZipCTHezLzWHPgI2I9zj8KNItJbVX/0dXDGZIkT++DrIU7NovAGztlB\nicr+jsoYv/CmT2Ek0FFVtwCISFWcJGG1gE1wWPUe7FkJHf4N9fs5fQnGhChvkkLuSwkBQFW3ikhu\nH8ZkjO8d/QPiz0KpWtDqWWjYH4qW93dUxvidN0nhVxEZB3zsWu6FFcQzgSoxAVaOgqVvQOkop7R1\nnkLOwxjjVVJ4FKej+WnX8g/AaJ9FZIyvHNzgFLA7uB6q/g06/sffERmT7aSZFESkJlAB+EpV/501\nIRnjAzuXwLQekO96uHsqVOvq74iMyZZSHZIqIs/hlLjoBSwSkZRmYDMme7t4xvm3XGNoMggG/GwJ\nwZg0pHWfQi8gUlXvAuoDj2X0xUWkvYhsF5EdIjIsjXb1RSRBRHpkdB/GpOjCaZj/tFOz6MIpyJUX\nbh8B+a3iuzFpSevy0QVVPQOgqkdExJu7n91EJAxnxrY2QAzwi4jM8RzJ5NHuTeDbDEXuI7Ve+pa4\nc/FX9dwi+az6R7aw4zuY+yTE7YMGj+DcXmOM8UZaSeFmj7mZBajgOVezqt6Zzms3AHao6i4AEZkB\ndAW2XNZuEPAlztmI38Wdiyf6jU7+DsNcjYtnYP4/Yd00KFYJHvwGbmrs76iMCShpJYXuly2/m8HX\nLgPs81iOARp6NhCRMsAdOLWVUk0KItIf6A9Qrly5DIZhQkZYHjj6OzR7Cm59xrlkZIzJkLTmaP4u\nC/b/NvCMqiZJGjVmVHU8MB6gXr16mmpDE3pOHYLvX4E2Lzv9BQ8ugDBvRlobY1Liy/89+3Fmbbsk\n3LXOUz1ghishFAc6ikiCqs7yYVwmGKjCuk9g4XMQf8657+CWdpYQjLlGvvwf9AtQyVVqez/QE7jP\ns4GqRlz6WUQmA19bQjDpOr4Hvn4Sdn7vDDXtMhqKV/J3VMYEBa+TgojkUdUL3rZX1QQRGQgsBMKA\nD1R1s4g86to+LsPRGgPwzdOwb7VzR3K9hyFHhgbGGWPS4E3p7AbAJKAIUE5EagH9VHVQes9V1fnA\n/MvWpZgMVLWvNwGbEHXkd6c+UeFSTjVTEbjOBh0Yk9m8+Yo1CugMxAKo6nqc0ULG+F5iPCz/D4xr\nCouHO+uK3mQJwRgf8ebyUQ5V3XPZ6KBEH8VjzF8OrHMK2P250SlN0fZVf0dkTNDzJinsc11CUtfd\nx4OA330blgl5Gz6Drx6FAsXh7o+gWhd/R2RMSPAmKTyGcwmpHHAIWMxV1EHKzjxLW1ipCj9LjIew\nXFC+GdTtC61fgHxF/R2VMSEj3aSgqodxhpMGLSttkQ1cOAWLRzh3JPeZA4VLQ+f/+TsqY0KON6OP\nJgBX3EWsqv19EpEJPX8scgrYndwPjR5zzhZy2oyvxviDN5ePFnv8nBenVtG+VNoa471zx+GbYbBh\nBhSv7EyNWbaBv6MyJqR5c/noU89lEfkIWOGziEzoUIXdy6HF09BiKOTM4++IjAl5V1PmIgIomdmB\nmBBx6k9YNRZue9EpYDdoLeTO7++ojDEu3vQpHOevPoUcwDEg1VnUjEmRKvz2MXz7PCRcgKpdIbyu\nJQRjspk0k4I4d6zV4q/qpkmqaqWrTcYcj4a5T8CupXBTU/jbKChe0d9RGWNSkGZSUFUVkfmqWiOr\nAjJBJikJPrkH4vZDp/9B3QetgJ0x2Zg3fQrrRKS2qv7m82hM8Djyu1OfKFde6DoWCpWEIuH+jsoY\nk45Uv7KJyKWEURv4RUS2i8ivIvKbiPyaNeGZgJNwEZb92ylg9+M7zrrwupYQjAkQaZ0prAbqAFZ0\nxnhn/68wZxAc2gQ1ukP9h/0dkTEmg9JKCgKgqjuzKJYsZfWOMtmqcbDwWShYEnpOhyod/R2RMeYq\npJUUSojIU6ltVNWALkxj9Y4yiaoz4U2ZOlC7N7R5GfJd5++ojDFXKa2kEAYUxHXGYEwy5086k97k\nyAUd/+2Up7ASFcYEvLSSwkFVfTnLIjGB4/eF8PUQOHUQGj3+19mCMSbgpdunYIzbmVhYMAw2fgYl\nqsLdUyG8nr+jMsZkorSSQussi8IEhlMHYdvXcOswaP4PK29tTBBKNSmo6rGsDMRkUycPwJY50OhR\nuLEGDNnsFLIzxgSlq6mSakKBKvw6Bb59wZn0pkonuK6sJQRjgpwlBXOlY7tgzmCI/gHKN4cuo5yE\nYIwJepYUTHIXz8LE252zg7+9A3UesJFFxoQQSwrGcXyPU8Aud37oMhpKRUGRMv6OyhiTxayGcahL\nuAhL34DRdWHLLGddlU6WEIwJUXamEMpi1sKcgXB4C9S8C8q38HdExhg/s6QQqpa+AcvehII3wr2f\nQuX2/o7IGJMNWFIIVYVLO53IbV6CvEX8HY0xJpvwaZ+CiLR3Tc6zQ0SGpbC9l4hsEJGNIrJSRGr5\nMp6Qdj7OGWa65kNnuU4f+NvblhCMMcn47ExBRMKAMUAbIAZn9rY5qrrFo9lu4FZVPS4iHYDxQENf\nxRSytn/jFLA7fQgKWweyMSZ1vrx81ADYoaq7AERkBtAVcCcFVV3p0X4VYHM2ZqbTR2DBM7DpS7ih\nOvT8xJn3wBhjUuHLpFAG2OexHEPaZwEPA9+ktEFE+gP9AcqVK5dZ8QW/vSudukWtnoemT1oBO2NM\nurJFR7OItMJJCs1S2q6q43EuLVGvXj3NwtACT1wMHPgNqv4NqnaBwb86N6UZY4wXfJkU9gOeBXPC\nXeuSEZFIYCLQQVVjfRhPcEtKgl8nw7cvQlguqHAb5C5gCcEYkyG+TAq/AJVEJAInGfQE7vNsICLl\ngJlAb1X93YexBLfYnc7Ioj0rIOJWp2ZR7gL+jsoYE4B8lhRUNUFEBgILceZ7/kBVN4vIo67t44AX\ngWLAWHGKriWoqk3llRFxMfBeUwjLDV3ehdr3WwE7Y8xV82mfgqrOB+Zftm6cx8/9gH6+jCFonYmF\nAsWgSDi0fQWqdIbCpfwdlTEmwFlBvECTcAG+fw1GVoeD6511DR6xhGCMyRTZYvSR8dK+1TB7IBzd\nDrXuhSI28Y0xJnNZUggEqrDweVg11rkjudcXUKmNv6MyxgQhSwqBQARQqN8Pbh8OeQr5OyJjTJCy\npJBdnTsB3/4LonrBTY2h3f/ZqCJjjM9ZR3N2tPVrGNMQ1n3yV2eyJQRjTBawM4Xs5PRhmP9PZ1rM\nkjXhvhlQura/owo68fHxxMTEcP78eX+HYkymy5s3L+Hh4eTKleuqnm9JITtZOwW2z4fbXoCmTzjl\nKkymi4mJoVChQpQvXx6xMzATRFSV2NhYYmJiiIiIuKrXsKTgbyf2wak/oWx9aDoYqneD4pX8HVVQ\nO3/+vCUEE5REhGLFinHkyJGrfg3rU/CXpCRYPQHGNoLZA5zlnHksIWQRSwgmWF3r37adKfjD0T9g\nziDY+xPc3MopYJfD8rMxxv/skyir7V/rFLA7vBW6vQe9v4KiN/k7KpPFRIT777/fvZyQkECJEiXo\n3Llzhl6nfPnyHD169KrbrFu3DhFhwYIFqT5fVbnttts4efKke92sWbMQEbZt2+Zet3Tp0ivi79u3\nL1988QXgdPAPGzaMSpUqUadOHRo3bsw336Q4r1aGvP7661SsWJHKlSuzcOHCFNusW7eORo0aERUV\nRb169Vi9ejUA06ZNIyoqyv3IkSMH69atA+D222/n+PHj1xxfoLGkkFUunnH+LRUFjQfAgNUQdZ8N\nNQ1RBQoUYNOmTZw7dw6ARYsWUaZM1s+fPX36dJo1a8b06dNTbTN//nxq1apF4cKFM/S8y73wwgsc\nPHiQTZs28euvvzJr1ixOnTp1TfFv2bKFGTNmsHnzZhYsWMDjjz9OYmLiFe2efvpphg8fzrp163j5\n5Zd5+umnAejVqxfr1q1j3bp1fPTRR0RERBAVFQVA7969GTt27DXFF4gsKfha/Hn47mUYXdepbJoj\nzLkruVBJf0dm/Kxjx47MmzcPcD5k7733Xve2Y8eO0a1bNyIjI2nUqBEbNmwAIDY2lrZt21K9enX6\n9euH6l9nT5bGAAAYG0lEQVQTEX788cc0aNCAqKgo/v73v6f44ehJVfn888+ZPHkyixYtSnWI7rRp\n0+jatat7+fTp06xYsYJJkyYxY8YMr4717NmzTJgwgdGjR5MnTx4ASpYsyd133+3V81Mze/Zsevbs\nSZ48eYiIiKBixYruswBPIuI+04mLi6N06dJXtJk+fTo9e/Z0L3fp0iVDSS9YWJ+CL+39GeYMhKO/\nQ637rN8gmyo/bF6mv2b0G53SbdOzZ09efvllOnfuzIYNG3jooYf44YcfABg+fDi1a9dm1qxZfP/9\n9/Tp04d169bx0ksv0axZM1588UXmzZvHpEmTANi6dSuffvopP/74I7ly5eLxxx9n2rRp9OnTJ9X9\nr1y5koiICCpUqEDLli2ZN28e3bt3v6Ldjz/+yPvvv+9enj17Nu3bt+eWW26hWLFirF27lrp166Z5\nrDt27KBcuXLJzjZSM2TIEJYsWZLi+zVs2LBk6/bv30+jRo3cy+Hh4ezff8UEj7z99tu0a9eOoUOH\nkpSUxMqVK69o8+mnnzJ79mz3ctGiRblw4QKxsbEUK1Ys3biDhSUFX0i46JSoWD3eme/g/i+h4u3+\njsqkwpsPcF+IjIwkOjqa6dOn07Fjx2TbVqxYwZdffgnAbbfdRmxsLCdPnmT58uXMnDkTgE6dOlG0\naFEAvvvuO9auXUv9+vUBOHfuHDfccEOa+/f8ZtyzZ0+mTp2aYlI4duwYhQoVSva8J554wv286dOn\nU7du3VRHvWR0NMzIkSMz1N4b7733HiNHjqR79+589tlnPPzwwyxevNi9/eeffyZ//vzUqFEj2fNu\nuOEGDhw4YEnBXKOwXM7ZQYNHoPWLVsDOpKpLly4MHTqUpUuXEht79VOUqyoPPPAAr7/+ulftExMT\n+fLLL5k9ezavvfaa+6anU6dOJUsAADlz5iQpKYkcOXJw7Ngxvv/+ezZu3IiIkJiYiIjw1ltvUaxY\nsSs6Zo8dO0bx4sWpWLEie/fu5eTJk+meLWTkTKFMmTLs27fPvRwTE5Ni38yUKVN45513ALjrrrvo\n1y/53F4zZsxIdvnukvPnz5MvX7404w06qhpQj7p162pmuOmZrzPlddzOxKrOeUL1RIyznBCfua9v\nMs2WLVv8HYIWKFBAVVX37dun77zzjqqqLlmyRDt16qSqqoMGDdKXX37ZvT4qKsq9/pVXXlFV1fnz\n5yugR44c0c2bN2vFihX10KFDqqoaGxur0dHRqqp600036ZEjR5Ltf+HChdq2bdtk6/r06aNTpky5\nItaGDRvqH3/8oaqq77//vvbv3z/Z9hYtWuiyZcv0/PnzWr58eff7Gx0dreXKldMTJ06oquo///lP\n7du3r164cEFVVQ8fPqyfffZZxt64y2zatEkjIyP1/PnzumvXLo2IiNCEhIQr2lWpUkWXLFmiqqqL\nFy/WOnXquLclJiZq6dKldefOncmek5SUpKVLl9b4+MD7v5zS3ziwRr34jLWL3Jlhy2yngN2vU2HP\nj866MDsJM+kLDw9n8ODBV6wfMWIEa9euJTIykmHDhjFlyhTA6WtYvnw51atXZ+bMmZQrVw6AatWq\n8eqrr9K2bVsiIyNp06YNBw8eTHW/06dP54477ki2rnv37il2rHbq1ImlS5em+7w8efLw8ccf8+CD\nDxIVFUWPHj2YOHEiRYoUAeDVV1+lRIkSVKtWjRo1atC5c2ev+hjSUr16de6++26qVatG+/btGTNm\nDGFhYQD069ePNWvWADBhwgT+8Y9/UKtWLZ577jnGjx/vfo3ly5dTtmxZbr755mSvvXbtWho1akTO\nnKH1f1nUY/RCIKhXr55e+kVfi/LD5l37teRTf8L8obB1LtwYCV3HQKnIa47N+NbWrVupWrWqv8MI\nGAcPHqRPnz4sWrTI36FkqSeeeIIuXbrQunVrf4eSYSn9jYvIWlWtl95zQysFZrbFI+D3b+H2EdB4\nkJ0dmKBUqlQpHnnkEa/6A4JJjRo1AjIhXCv7FMuo43sAhaLlnWTQ/B9Wr8gEvWu9nyAQPfLII/4O\nwS9CKil4jkcvki+DZamTkuCXCbD4JSjfFHp9DoVudB7GGBMkQiopXHUfwpHfnQJ2+1Y59xt0+m/m\nBmaMMdlESCWFq/L7Qvi0N+TOD3e8D5H3WL0iY0zQsqSQmsR45ya08PpQqyfc9i8omPYdosYYE+js\nPoXLxZ+DRcNhUltITID810OXUZYQTKYqX7480dHRtGzZMtn6J598kjJlypCUlOReN3nyZAYOHJjF\nEf5lxIgRlClTJlmJ6RMnTvgtnjlz5vDGG29c1XMvvedLly6lb9++KbZZvXo1LVq0oHLlytSuXZt+\n/fpx9uzZTP89dOzY0f0+jho1iqpVq9KrV69rOr7MYGcKnvasdPoOYndA7d6QeMGGmZosk5SUxFdf\nfUXZsmVZtmwZrVq18ndIbkOGDGHo0KH+DgNwSoN06dLFJ6996NAh7rrrLmbMmEHjxo0B+OKLL665\nxHdK5s+f7/557NixLF68mPDwcIAMHV9CQkKm3mBnZwoAF07DvH/Ahx2cy0Z9ZkPXdyF3AX9HZrLC\nh52ufPw46uq3e6FEiRKEhYVx/fXXu9ctXbqU6tWr89hjj6V4Z3FiYiIRERGoKidOnCAsLIzly5cD\n0KJFC/744w9Wr15N48aNqV27Nk2aNGH79u3u7ZcmjwFo1qwZ69evZ9myZe5v/7Vr187Qh9/IkSN5\n6KGHANi4cSM1atTg7NmzjBgxgt69e9O4cWMqVarEhAkTAKfkduvWralTpw41a9Z0VySNjo6matWq\nPPLII1SvXp22bdu655kYNWoU1apVIzIy0l28z/Mbe3R0NLfddhuRkZG0bt2avXv3As7kPoMHD6ZJ\nkybcfPPN7ol+Lr3nuXPndt9p7WnMmDE88MAD7oQA0KNHD0qWTF7qfu7cuTRs2JDatWtz++23c+jQ\nIYAU38+DBw/SokULoqKiqFGjhrsS7qXJjx599FF27dpFhw4dGDlyZLLjO3LkCN27d6d+/frUr1+f\nH390KiZceo+bNm1K7969vf6decWbWhjZ6ZFZtY+SOX9K9e1I1W+GqV44nfmvb7KVK+rCfNDxyseK\nd65++1Xq16+fTp06VePi4rR06dJ68eJFVVX98MMPdcCAAaqq2q5dO920aZPOnTtX69Wrp6+++qq7\n5pCqalxcnLtWz6JFi/TOO+9UVdXJkyfrE088oaqq27dv10v/jzp37qwrVqxQVdVTp06lWOdn+PDh\nWrp0aa1Vq5bWqlVLW7ZsqapOzaDmzZvrzJkztW7duu7XGT58uEZGRurZs2f1yJEjGh4ervv379f4\n+HiNi4tTVdUjR45ohQoVNCkpSXfv3q1hYWH622+/qarqXXfdpR999JGqqpYqVUrPnz+vqqrHjx+/\n4v3o3LmzTp48WVVVJ02apF27dlVV1QceeEB79OihiYmJunnzZq1QoYJXv4M77rhDZ82aleI2z/0e\nO3ZMk5KSVFV1woQJ+tRTT6X6fv7nP//RV199VVVVExIS9OTJk6qavCaV58+e+7n33nv1hx9+UFXV\nPXv2aJUqVdzvcZ06dfTs2bMpxnottY9C99rI2WPw49vQ8jnIUxAe+8kZYWRCz4PpzKdwrdu9cPHi\nRebPn8///vc/ChUqRMOGDVm4cOEV01s2b96c5cuXs3v3bp599lkmTJjArbfe6i6ZHRcXxwMPPMAf\nf/yBiBAfHw84lUFfeeUV3nrrLT744AP39fSmTZvy1FNP0atXL+6880735YvLpXT5KEeOHEyePJnI\nyEj+/ve/07RpU/e2rl27ki9fPvLly0erVq1YvXo1nTp14rnnnmP58uXkyJGD/fv3u79he854Vrdu\nXaKjowGnvHivXr3o1q0b3bp1uyKun376yV1KvHfv3u4Z1QC6detGjhw5qFatmns/mSUmJoZ77rmH\ngwcPcvHiRSIiIoCU38/69evz0EMPER8fT7du3dzH6Y3FixezZcsW9/LJkyc5ffo04Fxi8kUFV59e\nPhKR9iKyXUR2iMiwFLaLiIxybd8gInV8GQ8AqrD5KxjTAH4aA3tdk21YQjB+tHDhQk6cOEHNmjUp\nX748K1asSPESUosWLfjhhx9YvXq1u6Ny6dKlNG/eHHCmvGzVqhWbNm1i7ty57tnU8ufPT5s2bZg9\nezafffYZvXr1AmDYsGFMnDiRc+fO0bRpU7Zt28bzzz/vvgSSnj/++IOCBQty4MCBZOsvn0NBRJg2\nbRpHjhxh7dq1rFu3jpIlS7rjuzQbGziXeBISEgCYN28eAwYM4Ndff6V+/fru9d7wfE31ssZb9erV\nWbt2bbrtBg0axMCBA9m4cSPvv/+++zhSej9btGjB8uXLKVOmDH379mXq1KleH0NSUhKrVq1yTxm6\nf/9+ChYsCDhTuvqCz5KCiIQBY4AOQDXgXhGpdlmzDkAl16M/8J6v4gHg5EH49H74vC8ULgP9l0KF\n23y6S2O8MX36dCZOnEh0dDTR0dHs3r2bRYsWcfbs2WTtGjRowMqVK8mRIwd58+YlKiqK999/nxYt\nWgDOmcKl+QQmT56c7Ln9+vVj8ODB1K9f3z05z86dO6lZsybPPPMM9evXZ9u2bbz22mvuD6G0xMXF\nMXjwYJYvX05sbKz7uj04s7OdP3+e2NhYli5dSv369YmLi+OGG24gV65cLFmyhD179qT5+klJSezb\nt49WrVrx5ptvEhcX5/6WfEmTJk3cU4JOmzbNnRyv1sCBA5kyZQo///yze93MmTOvONPwfJ8vVbCF\nlN/PPXv2ULJkSR555BH69evHr7/+6nU8bdu2ZfTo0e7l9H4nmcGXZwoNgB2quktVLwIzgK6XtekK\nTHVd8loFXCcipXwW0ed9YcdiaPMy9PsObqzps10Z462zZ8+yYMECOnX6q5O6QIECNGvWjLlz5yZr\nmydPHsqWLeuegrJ58+acOnWKmjWdv+Wnn36aZ599ltq1a1/xrbpu3boULlyYBx980L3u7bffpkaN\nGkRGRpIrVy46dOiQYowjR45MNiQ1OjqaIUOGMGDAAG655RYmTZrEsGHDOHz4MOBc9mnVqhWNGjXi\nhRdeoHTp0vTq1Ys1a9ZQs2ZNpk6dSpUqVdJ8XxITE7n//vupWbMmtWvXZvDgwVx33XXJ2owePZoP\nP/yQyMhIPvroI/dEOlerZMmSzJgxg6FDh1K5cmWqVq3KwoULr5h4aMSIEdx1113UrVuX4sWLu9en\n9H4uXbqUWrVqUbt2bT799FP3rHXeGDVqFGvWrCEyMpJq1aoxbty4azo+b/isdLaI9ADaq2o/13Jv\noKGqDvRo8zXwhqqucC1/Bzyjqmsue63+OGcSlCtXrm563zBS9edGyJkPile8uueboBCqpbMPHDhA\ny5Yt2bZtGzl8OF/4iBEjKFiwYLYZwhqKrqV0dkAMSVXV8apaT1XrlShR4upf6MaalhBMSJo6dSoN\nGzbktdde82lCMIHPl6OP9gNlPZbDXesy2sYYc4369OlDnz59smRfI0aMyJL9GN/w5VeGX4BKIhIh\nIrmBnsCcy9rMAfq4RiE1AuJUNfU5BI3JJL66bGqMv13r37bPzhRUNUFEBgILgTDgA1XdLCKPuraP\nA+YDHYEdwFngwdRez5jMkjdvXmJjYylWrNgVQyeNCWSqSmxsLHnz5r3q1wjZOZpN6IqPjycmJsY9\nttyYYJI3b17Cw8PJlSv5RGI2R7MxqciVK5f7DlRjTHI2DMEYY4ybJQVjjDFulhSMMca4BVxHs4gc\nAa7ylmaKA0czMZxAYMccGuyYQ8O1HPNNqpru3b8BlxSuhYis8ab3PZjYMYcGO+bQkBXHbJePjDHG\nuFlSMMYY4xZqSWG8vwPwAzvm0GDHHBp8fswh1adgjDEmbaF2pmCMMSYNlhSMMca4BWVSEJH2IrJd\nRHaIyLAUtouIjHJt3yAidfwRZ2by4ph7uY51o4isFJFa/ogzM6V3zB7t6otIgms2wIDmzTGLSEsR\nWScim0VkWVbHmNm8+NsuIiJzRWS965gDutqyiHwgIodFZFMq2337+aWqQfXAKdO9E7gZyA2sB6pd\n1qYj8A0gQCPgZ3/HnQXH3AQo6vq5Qygcs0e773HKtPfwd9xZ8Hu+DtgClHMt3+DvuLPgmJ8D3nT9\nXAI4BuT2d+zXcMwtgDrAplS2+/TzKxjPFBoAO1R1l6peBGYAXS9r0xWYqo5VwHUiUiqrA81E6R6z\nqq5U1eOuxVU4s9wFMm9+zwCDgC+Bw1kZnI94c8z3ATNVdS+Aqgb6cXtzzAoUEmdyjII4SSEha8PM\nPKq6HOcYUuPTz69gTAplgH0eyzGudRltE0gyejwP43zTCGTpHrOIlAHuAN7Lwrh8yZvf8y1AURFZ\nKiJrRSRr5uD0HW+O+V2gKnAA2Ag8oapJWROeX/j088vmUwgxItIKJyk083csWeBt4BlVTQqhGdZy\nAnWB1kA+4CcRWaWqv/s3LJ9qB6wDbgMqAItE5AdVPenfsAJTMCaF/UBZj+Vw17qMtgkkXh2PiEQC\nE4EOqhqbRbH5ijfHXA+Y4UoIxYGOIpKgqrOyJsRM580xxwCxqnoGOCMiy4FaQKAmBW+O+UHgDXUu\nuO8Qkd1AFWB11oSY5Xz6+RWMl49+ASqJSISI5AZ6AnMuazMH6OPqxW8ExKnqwawONBOle8wiUg6Y\nCfQOkm+N6R6zqkaoanlVLQ98ATwewAkBvPvbng00E5GcIpIfaAhszeI4M5M3x7wX58wIESkJVAZ2\nZWmUWcunn19Bd6agqgkiMhBYiDNy4QNV3Swij7q2j8MZidIR2AGcxfmmEbC8POYXgWLAWNc35wQN\n4AqTXh5zUPHmmFV1q4gsADYAScBEVU1xaGMg8PL3/AowWUQ24ozIeUZVA7aktohMB1oCxUUkBhgO\n5IKs+fyyMhfGGGPcgvHykTHGmKtkScEYY4ybJQVjjDFulhSMMca4WVIwxhjjZknBZDsikuiq8nnp\nUT6NtuVTqyaZwX0udVXiXC8iP4pI5at4jUcvlZUQkb4iUtpj20QRqZbJcf4iIlFePOdJ1z0LxqTL\nkoLJjs6papTHIzqL9ttLVWsBU4C3Mvpk130CU12LfYHSHtv6qeqWTInyrzjH4l2cTwKWFIxXLCmY\ngOA6I/hBRH51PZqk0Ka6iKx2nV1sEJFKrvX3e6x/X0TC0tndcqCi67mtReQ3ceah+EBE8rjWvyEi\nW1z7+Y9r3QgRGSrOvA31gGmufeZzfcOv5zqbcH+Qu84o3r3KOH/CoxCaiLwnImvEmVPgJde6wTjJ\naYmILHGtaysiP7nex89FpGA6+zEhxJKCyY7yeVw6+sq17jDQRlXrAPcAo1J43qPAO6oahfOhHCMi\nVV3tm7rWJwK90tn/34CNIpIXmAzco6o1cSoAPCYixXCqr1ZX1UjgVc8nq+oXwBqcb/RRqnrOY/OX\nrudecg9OfaaribM94Fm243nXXeqRwK0iEqmqo3Cqh7ZS1VYiUhz4F3C7671cAzyVzn5MCAm6Mhcm\nKJxzfTB6ygW867qGnohTIvpyPwHPi0g4zpwCf4hIa5yqob+4ynvkI/W5FaaJyDkgGmcehsrAbo9a\nUVOAATilms8Dk0Tka+Brbw9MVY+IyC5XzZo/cAq3/eh63YzEmRtn7gDP9+luEemP8/+6FFANp9yF\np0au9T+69pMb530zBrCkYALHEOAQTsXPHDgfysmo6ici8jPQCZgvIn/HqYUzRVWf9WIfvVR1zaUF\nEbk+pUauejwNcIqw9QAG4pRt9tYM4G5gG/CVqqo4n9BexwmsxelPGA3cKSIRwFCgvqoeF5HJQN4U\nnivAIlW9NwPxmhBil49MoCgCHHRNntIbpzhaMiJyM7DLdclkNs5llO+AHiJyg6vN9SJyk5f73A6U\nF5GKruXewDLXNfgiqjofJ1mlNN/1KaBQKq/7Fc7sWffiJAgyGqerTPQLQCMRqQIUBs4AceJUCu2Q\nSiyrgKaXjklECohISmddJkRZUjCBYizwgIisx7nkciaFNncDm0RkHVADZ8rCLTjX0L8VkQ3AIpxL\nK+lS1fM4FSg/d1XgTALG4XzAfu16vRWkfE1+MjDuUkfzZa97HKec9U2qutq1LsNxuvoq/gv8U1XX\nA7/hnH18gnNJ6pLxwAIRWaKqR3BGRk137ecnnPfTGMCqpBpjjPFgZwrGGGPcLCkYY4xxs6RgjDHG\nzZKCMcYYN0sKxhhj3CwpGGOMcbOkYIwxxu3/AUzPMz2tLdk5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1184a5978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGX2wPHvIfQqAiIQMEhvIfSORARpAiooihRdZFWK\n4rqKuipr+cmuu6uCICooogg2pCiCqFQVEZDeS4AAUgKEGkg5vz/uME4gZQKZTGbmfJ4nD7n3vnPv\nuQPMmfu+955XVBVjjDEGII+/AzDGGJN7WFIwxhjjZknBGGOMmyUFY4wxbpYUjDHGuFlSMMYY42ZJ\nwRhjjJslBRNURCRGRM6JyGkR+UNEJotI0UvatBSRH0XklIjEi8gcEal9SZviIvKGiOx17Wuna7l0\nOscVERkuIhtE5IyIxIrI5yJSz5fna0x2s6RggtFtqloUiAIaAE9f3CAiLYDvgFlAeaAysBb4SURu\ndLXJD/wA1AE6AcWBFsBRoGk6x3wTeBQYDlwLVAdmAl2zGryI5M3qa4zJLmJPNJtgIiIxwCBV/d61\n/G+gjqp2dS0vBdar6iOXvO5b4Iiq9heRQcArQBVVPe3FMasBW4AWqroinTaLgI9VdaJreaArztau\nZQWGAo8BeYF5wBlVfcJjH7OAxar6PxEpD4wF2gKngddVdYwXb5ExGbIrBRO0RCQc6AzscC0XBloC\nn6fR/DOgg+v3W4B53iQEl/ZAbHoJIQt6As2A2sA04G4REQARKQl0BKaLSB5gDs4VTgXX8R8TkVuv\n8vjGWFIwQWmmiJwC9gGHgRdc66/F+Td/MI3XHAQujheUSqdNerLaPj2vquoxVT0HLAUUaOPa1gv4\nRVUPAE2AMqr6oqpeUNVdwHtAn2yIwYQ4SwomGPVU1WJAO6Amf37YHwdSgHJpvKYczpgBQFw6bdKT\n1fbp2XfxF3X6dacD97hW3QtMdf1+A1BeRE5c/AGeAcpmQwwmxFlSMEFLVRcDk4H/uJbPAL8AvdNo\nfhfO4DLA98CtIlLEy0P9AISLSOMM2pwBCnssX59WyJcsTwN6icgNON1KX7rW7wN2q+o1Hj/FVLWL\nl/Eaky5LCibYvQF0EJH6ruWRwADX7aPFRKSkiLyMc3fRP11tPsL54P1SRGqKSB4RKSUiz4jIZR+8\nqrodGA9ME5F2IpJfRAqKSB8RGelqtga4Q0QKi0hV4C+ZBa6qv+NcvUwE5qvqCdemFcApEXlKRAqJ\nSJiI1BWRJlfyBhnjyZKCCWqqegSYAjzvWl4G3ArcgTMOsAfnttXWrg93VPU8zmDzFmABcBLng7g0\n8Gs6hxoOvAWMA04AO4HbcQaEAV4HLgCHgA/5sysoM5+4YvnE45ySgW44t9zu5s/EUcLLfRqTLrsl\n1RhjjJtdKRhjjHGzpGCMMcbNkoIxxhg3SwrGGGPcAq7wVunSpTUiIsLfYRhjTEBZtWrVUVUtk1m7\ngEsKERERrFy50t9hGGNMQBGRPd60s+4jY4wxbpYUjDHGuFlSMMYY42ZJwRhjjJslBWOMMW4+Swoi\n8r6IHBaRDelsFxEZIyI7RGSdiDT0VSzGGGO848srhck4k56npzNQzfUzGHjbh7EYY4zxgs+Sgqou\nAY5l0KQHMEUdy4FrRCQ7Zq8yxpjgkpwIR3fkyKH8+fBaBTymHwRiXesum+tWRAbjXE1QqVKlHAnO\nmJxS/5/fEX8u0d9hmFyqjsTwr3zvUlriiT7/XzaPvtOnxwuIJ5pV9V3gXYDGjRvbBBAmqMSfSyRm\ndFd/h2Fyo5QUGN8czp2FrmPYXLu7zw/pz6SwH6josRzuWmeMMaFt3wooWwfyF4Hek6F4OShUMkcO\n7c9bUmcD/V13ITUH4lX1sq4jY4wJGedPwTdPwKQO8PNYZ13Z2jmWEMCHVwoiMg1oB5QWkVjgBSAf\ngKpOAOYCXYAdwFngfl/FYowxud6O72HOYxAfC80eghZD/RKGz5KCqt6TyXYFhvjq+MYYEzAWvwYL\nX4bS1eGB+VCpmd9CCYiBZmOMCUrJiRCWD6p1gKQEaPt3yFfQryFZUjDGmJx26g+Y+wQUvAZ6vAXl\no5yfXMBqHxljTE5Rhd8/hnFNYdt3UKqKsy4XsSsFY4zJCSf2wuzhsGshVGoJ3cdC6ar+juoylhSM\nMSYnJCbAH+ug63+h0QOQJ3d21FhSMMYYXzmyFTbMgOinoUx1GLER8hXyd1QZCtmkYPVmTG5RolA+\nf4dgsltyIvz0Biz+t/NUcqMBULx8rk8IEMJJwerNGGN84sDvMGsoHNoAde6Azv+GomX8HZXXQjYp\nGGNMtjt/Gqb0hLwFoc8nUDPwvnhaUjDGmKt1cB1cXw8KFIW7P4LrI6HQNf6O6orkzuFvY4wJBAkn\n4evH4Z02sP4LZ13ltgGbEMCuFIwx5sps+w6+HgEn90PzR6BmF39HlC0sKRhjTFZ9+xT8OgHK1IS/\nLICKTfwdUbaxpGCMMd64WI5CBMKbQMES0OZvkLeAf+PKZpYUjDEmMycPwjePO+MFzR+Ger38HZHP\n2ECzMcakRxVWfQjjmsHOH0HC/B2Rz9mVgjHGpOXYbpgzHHYvgRtaQ/cxTlXTIBdSScGztIWVFjDG\nZOjoNjiwBrq9AQ0H5NoCdtktpJKClbYwxmTo8GanTEXUvVD9Vnh0LRS+1t9R5aiQSgrGGJOmpAuw\n7HVY8hoUKQ21e0L+wiGXEMCSgjEm1O1fBbOGweGNULcXdP6XkxBClCUFY0zoOr4HJnWEItfBPdOh\nRmd/R+R3lhSMMaHn2G64tjKUvAF6ToDqHZ2H0Yw9p2CMCSEJ8TDnURjbCGJXOesie1tC8GBXCsaY\n0LB1nlPA7vQf0GIIXFfL3xHlSpYUjDHBTRW+egjWTYfrasPdH0N4I39HlWtZUjDGBDcRZ+yg3TPQ\negTkze/viHI1SwrGmOATvx+++Rs0+ytUiYboZ/wdUcCwgWZjTPBISYGVH8D45rBrEZw84O+IAo5d\nKRhjgkPcTufOopilTonr28Y4t52aLLGkYIwJDptnw8G1TjJo2N8ZSzBZ5tPuIxHpJCJbRWSHiIxM\nY3sJEZkjImtFZKOI3O/LeIwxQebQRqebCKDFUBiyAhoNsIRwFXyWFEQkDBgHdAZqA/eISO1Lmg0B\nNqlqfaAd8F8RsVsDjDEZSzoPC/8P3mkL8591bjsNywfFy/k7soDny+6jpsAOVd0FICLTgR7AJo82\nChQTEQGKAseAJB/GZIwJdPt+g9lD4cgWiLwbOo22K4Ns5MukUAHY57EcCzS7pM1bwGzgAFAMuFtV\nUy7dkYgMBgYDVKpUySfBGmMCwN7l8H4nKF4e7v3cqVlkspW/b0m9FVgDlAeigLdEpPiljVT1XVVt\nrKqNy5Qpk9MxGmP87cxR58/wptDxJXhkuSUEH/FlUtgPVPRYDnet83Q/MEMdO4DdQE0fxmSMCSTn\nTsDsYU4Bu1N/OFNithwGBS/77miyiS+Twm9ANRGp7Bo87oPTVeRpL9AeQETKAjWAXT6MyRgTKLZ8\nA+Oawe8fO3cUWSXTHOGzMQVVTRKRocB8IAx4X1U3ishDru0TgJeAySKyHhDgKVU96quYjDEBIDEB\nZj4MG2dA2bpwzzSo0NDfUYUMnz68pqpzgbmXrJvg8fsBwDoGjTF/ylsAUhIh+h/Q+jHnVlOTY/w9\n0GyMMRAfC5/2c2ZEE4G7PoKb/m4JwQ8sKRhj/CclBX6b6Iwd7PjeeUIZ7LkDP7LaR8YY/zi6w7mz\naO/PcGM03PYGlIzwd1Qhz5KCMcY/lv4XDm+EHuMh6l67OsglLCkYY3LOH+shLD+UqQG3vgK3vADF\nrvd3VMaDjSkYY3wv6Tz8+DK82w4WPO+sK3ytJYRcyK4UjDG+tW8FzBoKR7dC/Xvg1v/zd0QmA5YU\njDG+s3EmfD4QSoRD3y+h2i3+jshkwpKCMSb7nT8NBYpClZuhzePQegQUKObvqIwXvBpTEJH8IlLV\n18EYYwLcueMwcwhMvMUZRyhYHNo/bwkhgGSaFESkK7AeWOBajhKRr3wdmDEmwGya7TyEtnYa1Ojs\n72jMFfKm++hFnMlxFgKo6hq7ajDGuJ07DrOHw+bZcH096Ps5lKvv76jMFfImKSSq6glJ/WCJ+ige\nY0ygyVsQju1yuolaDrd6RQHOm6SwWUTuAvKISGVgOLDct2EZY3K1E3th8b+g878hfxEYvBjC7L6V\nYODNQPNQoBGQAswAzgOP+jIoY0wulZICv74L45rDhq/gwBpnvSWEoOHN3+StqvoU8NTFFSJyB06C\nMMaEiiPbnAJ2+5ZDlfZOAbtrKvk7KpPNvLlS+Eca657N7kCMMbmYKsweCke2QM+34b4vLSEEqXSv\nFETkVqATUEFE/uexqThOV5IxJtgdXAslKjp1inqMd543KFbW31EZH8roSuEwsAFIADZ6/HwH2E3I\nxgSzxAT4fhS8G+0MKAOUrmoJIQSke6Wgqr8Dv4vIVFVNyMGYjDH+tOcXp6sobgdE3QftRvo7IpOD\nvBloriAirwC1gYIXV6pqdZ9FZYzxjxXvwdy/wzUVod9XTu0iE1K8GWieDHwACE630WfApz6MyRiT\n05ITnT+r3AzNH4GHf7GEEKK8SQqFVXU+gKruVNV/YGMKxgSHs8fgq4fgi/ud5VJVoNP/ORVOTUjy\npvvovIjkAXaKyEPAfsBKHhoTyFRh0yyY+4RTu6j1CEhJhjxh/o7M+Jk3SWEEUASnvMUrQAngAV8G\nZYzxodOH4esRsOVrKBfljB1cX8/fUZlcItOkoKq/un49BfQDEJEKvgzKGONDyYnOFJkdXoTmQ6xE\nhUklwzEFEWkiIj1FpLRruY6ITAF+zeh1xphc5ngM/PCi021UogI8tg5aPWoJwVwm3aQgIq8CU4G+\nwDwRGYUzp8JawG5HNSYQpCTD8rdhfAunkN3R7c76fIX8G5fJtTL6mtADqK+q50TkWmAfUE9Vd+VM\naMaYq3J4i1PALnYFVO3gFLArEe7vqEwul1FSSFDVcwCqekxEtllCMCZAJCfCx3dA4jm44z2o1xtS\nT5RlTJoySgo3isjF8tgCVPZYRlXvyGznItIJeBMIAyaq6ug02rQD3gDyAUdV9SbvwzfGpHJoI5Sp\n6cx+duckKFUVipbxd1QmgGSUFO68ZPmtrOxYRMKAcUAHIBb4TURmq+omjzbXAOOBTqq6V0Suy8ox\njDEuiedg0avw81vQaTQ0Gww3tPB3VCYAZVQQ74er3HdTYMfFLicRmY4zTrHJo829wAxV3es65uGr\nPKYxoSdmGcweDsd2QsP+EHmXvyMyAcybMhdXqgLO4PRFsa51nqoDJUVkkYisEpH+ae1IRAaLyEoR\nWXnkyBEfhWtMAFr4KkzuCpoM/WdB97FQ6Bp/R2UCmL9vUs6LM/9ze6AQ8IuILFfVbZ6NVPVd4F2A\nxo0ba45HaUxuo+oMHIc3dh5Au/lZyF/E31GZIOB1UhCRAqp6Pgv73g9U9FgOd63zFAvEqeoZ4IyI\nLAHqA9swxlzuTBzMGwnXVoboZ6BaB+fHmGySafeRiDQVkfXAdtdyfREZ68W+fwOqiUhlEckP9AFm\nX9JmFtBaRPKKSGGgGbA5S2dgTChQhQ1fwrimsHGGc3eRMT7gzZXCGKAbMBNAVdeKSHRmL1LVJBEZ\nCszHuSX1fVXd6Kq0iqpOUNXNIjIPWIcz7/NEVd1whediTHA6eRC+eRy2zoXyDaDHbChbx99RmSDl\nTVLIo6p7JPWDL8ne7FxV5wJzL1k34ZLl14DXvNmfMSHpeAzsWgQdX4ZmD1u9IuNT3vzr2iciTQF1\nPXswDOvzN8a3ju2GXQuh8QPO8wYjNkLha/0dlQkB3iSFh3G6kCoBh4DvXeuMMdktJRl+nQA/vAR5\n80Ptnk4ysIRgcog3SSFJVfv4PBJjQt2hTTB7KOxfBdU7Qdf/WTIwOc6bpPCbiGwFPsV5+viUj2My\nJvSciYOJ7Z2S1ndOgrp3WgE74xfezLxWRURa4txS+k8RWQNMV9XpPo/OmGB3PAZKRkCRUtDzbYho\n4/xujJ94VeZCVX9W1eFAQ+AkzuQ7xpgrdeEszH8WxjSAnT866+r0tIRg/C7TKwURKYpTyK4PUAvn\ngbOWPo7LmOC1e6kz+c3x3dDofqjQyN8RGePmzZjCBmAO8G9VXerjeIwJbt+OhF/fhpKVYcDXULmN\nvyMyJhVvksKNqpri80iMCQUlwqHlMGj3DOQv7O9ojLlMuklBRP6rqn8DvhSRyyqTejPzmjEh78xR\n+PYpqNUN6twOLYf6OyJjMpTRlcKnrj+zNOOaMQangN36L+DbJ+H8KafEtTEBIKOZ11a4fq2lqqkS\ng6vQ3dXOzGZMcIrf7xSw2zYPKjSGHm/BdbX8HZUxXvHmltQH0lj3l+wOxJigsWMB7F4Ct74Kf/nO\nEoIJKBmNKdyNcxtqZRGZ4bGpGHDC14EZE1Didjo/1TtCg/5Q9RZnUNmYAJPRmMIKIA5nxrRxHutP\nAb/7MihjAkZyEiwfDwtfgSLXQZXVzgQ4lhBMgMpoTGE3sBunKqox5lJ/bHAK2B34HWp0ha7/tRnR\nTMDLqPtosareJCLHAc9bUgVQVbXyjSZ0Hd4M794EBa+BXh84t5taATsTBDLqPro45WbpnAjEmIBw\nJs6pT1SmJnR4CSLvtnpFJqike/eRx1PMFYEwVU0GWgB/BYrkQGzG5B4XzsC8Z+DNSGdAWQRaPGIJ\nwQQdb25JnYkzFWcV4AOgGvCJT6MyJjfZtQjGt4Dl41xXBmX8HZExPuNN7aMUVU0UkTuAsao6RkTs\n7iMT/FKSYc6j8PtHcG0VGDgXIlr5OypjfMqr6ThFpDfQD+jpWme3WJjglyfMKVfR6jFoN9KZFc2Y\nIOftE83ROKWzd4lIZWCab8Myxk9OH4Yv/uLcbgpOiYoO/7SEYEJGpklBVTcAw4GVIlIT2Keqr/g8\nMmNykiqs/RTGNYXNs+GPdc56u83UhBhvZl5rA3wE7Md5RuF6Eemnqj/5OjhjcsSJffD1CKdmUXhT\n5+qgTA1/R2WMX3gzpvA60EVVNwGISC2cJGG1gE1wWP427PkZOv8bmgxyxhKMCVHeJIX8FxMCgKpu\nFpH8PozJGN87uh0Sz0K5+hD9NDQbDCUj/B2VMX7nTVJYLSITgI9dy32xgngmUCUnwc9jYNFoKB/l\nlLYuUMz5McZ4lRQewhloftK1vBQY67OIjPGVg+ucAnYH10Kt26DLf/wdkTG5ToZJQUTqAVWAr1T1\n3zkTkjE+sHMhTO0Fha6Fu6ZA7R7+jsiYXCndW1JF5BmcEhd9gQUiktYMbMbkbhfOOH9WagEth8GQ\nXy0hGJOBjJ5T6AtEqmpvoAnwcFZ3LiKdRGSriOwQkZEZtGsiIkki0iurxzAmTedPw9wnnZpF509B\nvoJwyygobBXfjclIRt1H51X1DICqHhERb55+dhORMJwZ2zoAscBvIjLb804mj3b/Ar7LUuTGpGfH\nDzDnMYjfB00fxHm8xhjjjYySwo0eczMLUMVzrmZVvSOTfTcFdqjqLgARmQ70ADZd0m4Y8CXO1Ygx\nV+7CGZj7d1gzFUpVg/u/hRta+DsqYwJKRknhzkuW38rivisA+zyWY4Fmng1EpAJwO05tpXSTgogM\nBgYDVKpUKYthmJARVgCOboPWj8NNTzldRsaYLMlojuYfcuD4bwBPqWqKZFBjRlXfBd4FaNy4sabb\n0ISeU4fgx5egw4vOeMH98yDMmzutjTFp8eX/nv04s7ZdFO5a56kxMN2VEEoDXUQkSVVn+jAuEwxU\nYc0nMP8ZSDznPHdQ/VZLCMZcJV/+D/oNqOYqtb0f6APc69lAVStf/F1EJgNfW0IwmTq+B75+DHb+\n6Nxq2n0slK7m76iMCQpeJwURKaCq571tr6pJIjIUmA+EAe+r6kYReci1fUKWozUG4NsnYd8K54nk\nxn+BPFm6Mc4YkwFvSmc3BSYBJYBKIlIfGKSqwzJ7rarOBeZesi7NZKCqA70J2ISoI9uc+kTFyznV\nTEXgGrvpwJjs5s1XrDFANyAOQFXX4twtZIzvJSfCkv/AhFbw/QvOupI3WEIwxke86T7Ko6p7Lrk7\nKNlH8RjzpwNrnAJ2f6x3SlN0fNnfERkT9LxJCvtcXUjqevp4GLDNt2GZkLfuM/jqIShSGu76CGp3\n93dExoQEb5LCwzhdSJWAQ8D3XEEdJGO8kpwIYfkgojU0Ggjtn4NCJf0dlTEhI9OkoKqHcW4nNcZ3\nzp+C70c5TyT3nw3Fy0O3//k7KmNCjjd3H70HXPYUsaoO9klEJvRsX+AUsDu5H5o/7Fwt5LUZX43x\nB2+6j773+L0gTq2ifem0NcZ7547DtyNh3XQoXcOZGrNiU39HZUxI86b76FPPZRH5CFjms4hM6FCF\n3Uug7ZPQ9gnIW8DfERkT8q6kzEVloGx2B2JCxKk/YPl4uPl5p4DdsFWQv7C/ozLGuHgzpnCcP8cU\n8gDHgHRnUTMmTarw+8fw3bOQdB5q9YDwRpYQjMllMkwK4jyxVp8/q5umqKqVrjZZczwG5jwKuxbB\nDa3gtjFQuqq/ozLGpCHDpKCqKiJzVbVuTgVkgkxKCnxyN8Tvh67/g0b3WwE7Y3Ixb8YU1ohIA1X9\n3efRmOBxZJtTnyhfQegxHoqVhRLh/o7KGJOJdL+yicjFhNEA+E1EtorIahH5XURW50x4JuAkXYDF\n/3YK2P30prMuvJElBGMCREZXCiuAhoAVnTHe2b8aZg+DQxug7p3Q5C/+jsgYk0UZJQUBUNWdORSL\nCWTLJ8D8p6FoWegzDWp28XdExpgrkFFSKCMij6e3UVWtMI1xbjUVgQoNoUE/6PAiFLrG31EZY65Q\nRkkhDCiK64rBmFQSTjqT3uTJB13+7ZSnsBIVxgS8jJLCQVV9McciMYFj23z4egScOgjNH/nzasEY\nE/AyHVMwxu1MHMwbCes/gzK14K4pEN7Y31EZY7JRRkmhfY5FYQLDqYOw5Wu4aSS0+ZuVtzYmCKWb\nFFT1WE4GYnKpkwdg02xo/hBcXxdGbHQK2RljgtKVVEk1oUAVVn8I3z3nTHpTsytcU9ESgjFBzpKC\nudyxXTB7OMQshYg20H2MkxCMMUHPkoJJ7cJZmHiLc3Vw25vQcIDdWWRMCLGkYBzH9zgF7PIXhu5j\noVwUlKjg76iMMTnMahiHuqQLsGg0jG0Em2Y662p2tYRgTIiyK4VQFrsKZg+Fw5ugXm+IaOvviIwx\nfmZJIVQtGg2L/wVFr4d7PoUanfwdkTEmF7CkEKqKl3cGkTv8EwqW8Hc0xphcwqdjCiLSyTU5zw4R\nGZnG9r4isk5E1ovIzyJS35fxhLSEeOc205UfOMsN+8Ntb1hCMMak4rMrBREJA8YBHYBYnNnbZqvq\nJo9mu4GbVPW4iHQG3gWa+SqmkLX1W6eA3elDUNwGkI0x6fNl91FTYIeq7gIQkelAD8CdFFT1Z4/2\nywGbszE7nT4C856CDV/CdXWgzyfOvAfGGJMOXyaFCsA+j+VYMr4K+AvwbVobRGQwMBigUqVK2RVf\n8Nv7s1O3KPpZaPWYFbAzxmQqVww0i0g0TlJondZ2VX0Xp2uJxo0baw6GFnjiY+HA71DrNqjVHYav\ndh5KM8YYL/gyKewHPAvmhLvWpSIikcBEoLOqxvkwnuCWkgKrJ8N3z0NYPqhyM+QvYgnBGJMlvkwK\nvwHVRKQyTjLoA9zr2UBEKgEzgH6qus2HsQS3uJ3OnUV7lkHlm5yaRfmL+DsqY0wA8llSUNUkERkK\nzMeZ7/l9Vd0oIg+5tk8AngdKAePFKbqWpKo2lVdWxMfC260gLD90fwsa3GcF7IwxV8ynYwqqOheY\ne8m6CR6/DwIG+TKGoHUmDoqUghLh0PElqNkNipfzd1TGmABnBfECTdJ5+PEVeL0OHFzrrGv6oCUE\nY0y2yBV3Hxkv7VsBs4bC0a1Q/x4oYRPfGGOylyWFQKAK85+F5eOdJ5L7fgHVOvg7KmNMELKkEAhE\nAIUmg+CWF6BAMX9HZIwJUpYUcqtzJ+C7f0BUX7ihBdz6f3ZXkTHG52ygOTfa/DWMawZrPvlzMNkS\ngjEmB9iVQm5y+jDM/bszLWbZenDvdCjfwN9RBZ3ExERiY2NJSEjwdyjGZLuCBQsSHh5Ovnz5ruj1\nlhRyk1Ufwta5cPNz0OpRp1yFyXaxsbEUK1aMiIgIxK7ATBBRVeLi4oiNjaVy5cpXtA9LCv52Yh+c\n+gMqNoFWw6FOTyhdzd9RBbWEhARLCCYoiQilSpXiyJEjV7wPG1Pwl5QUWPEejG8Os4Y4y3kLWELI\nIZYQTLC62n/bdqXgD0e3w+xhsPcXuDHaKWCXx/KzMcb/7JMop+1f5RSwO7wZer4N/b6Ckjf4OyqT\nw0SE++67z72clJREmTJl6NatW5b2ExERwdGjR6+oTUREBPXq1SMqKop69eoxa9asNF+vqtx8882c\nPHnSvW7mzJmICFu2bHGvW7Ro0WXxDxw4kC+++AJwBvhHjhxJtWrVaNiwIS1atODbb9OcVytLXn31\nVapWrUqNGjWYP39+mm3Wrl1LixYtqFevHrfddpv7XKZOnUpUVJT7J0+ePKxZswaAW265hePHj191\nfIHGkkJOuXDG+bNcFLQYAkNWQNS9dqtpiCpSpAgbNmzg3LlzACxYsIAKFXJ+/uyFCxeyZs0avvji\nC4YPH55mm7lz51K/fn2KFy/uXjdt2jRat27NtGnTvD7Wc889x8GDB9mwYQOrV69m5syZnDp16qri\n37RpE9OnT2fjxo3MmzePRx55hOTk5MvaDRo0iNGjR7N+/Xpuv/12XnvtNQD69u3LmjVrWLNmDR99\n9BGVK1cmKioKgH79+jF+/Pirii8QWVLwtcQE+OFFGNvIqWyaJ8x5KrlYWX9HZvysS5cufPPNN4Dz\nIXvPPfe4tx07doyePXsSGRlJ8+bNWbduHQBxcXF07NiROnXqMGjQIFT/nIjw448/pmnTpkRFRfHX\nv/41zQ/H9Jw8eZKSJUumuW3q1Kn06NHDvXz69GmWLVvGpEmTmD59ulf7P3v2LO+99x5jx46lQIEC\nAJQtW5a0he7IAAAXtElEQVS77rrL6xjTMmvWLPr06UOBAgWoXLkyVatWZcWKFZe127ZtG23btgWg\nQ4cOfPnll5e1mTZtGn369HEvd+/ePUtJL1jYmIIv7f0VZg+Fo9ug/r02bpBLRYz8Jtv3GTO6a6Zt\n+vTpw4svvki3bt1Yt24dDzzwAEuXLgXghRdeoEGDBsycOZMff/yR/v37s2bNGv75z3/SunVrnn/+\neb755hsmTZoEwObNm/n000/56aefyJcvH4888ghTp06lf//+GcYQHR2NqrJr1y4+++yzNNv89NNP\nvPPOO+7lWbNm0alTJ6pXr06pUqVYtWoVjRo1yvA4O3bsoFKlSqmuNtIzYsQIFi5ceNn6Pn36MHLk\nyFTr9u/fT/Pmzd3L4eHh7N9/2QSP1KlTh1mzZtGzZ08+//xz9u3bd1mbTz/9NFUXWsmSJTl//jxx\ncXGUKlUq07iDhSUFX0i64JSoWPGuM9/BfV9C1Vv8HZVJhzcf4L4QGRlJTEwM06ZNo0uXLqm2LVu2\nzP1t9uabbyYuLo6TJ0+yZMkSZsyYAUDXrl3d3+5/+OEHVq1aRZMmTQA4d+4c1113XaYxLFy4kNKl\nS7Nz507at29Pu3btKFq0aKo2x44do1ixP+ttTZs2jUcffRRwPqinTZtGo0aN0r3rJat3w7z++utZ\nau+N999/n+HDh/PSSy/RvXt38ufPn2r7r7/+SuHChalbt26q9ddddx0HDhywpGCuUlg+5+qg6YPQ\n/nkrYGfS1b17d5544gkWLVpEXNyVT1GuqgwYMIBXX331il5fpUoVypYty6ZNm2jatGmqbXnz5iUl\nJYU8efJw7NgxfvzxR9avX4+IkJycjIjw2muvUapUqcsGZo8dO0bp0qWpWrUqe/fu5eTJk5leLWTl\nSqFChQqpvvXHxsamOTZTs2ZNvvvuO8DpSrrYbXfR9OnTU3XfXZSQkEChQoUyjDfoqGpA/TRq1Eiv\n1A1PfX3Fr83UmTjV2Y+qnoh1lpMSfXcsc1U2bdrk7xC0SJEiqqq6b98+ffPNN1VVdeHChdq1a1dV\nVR02bJi++OKL7vVRUVHu9S+99JKqqs6dO1cBPXLkiG7cuFGrVq2qhw4dUlXVuLg4jYmJUVXVG264\nQY8cOXJZDJ7rDx06pGXKlNE//vjjsnbNmjXT7du3q6rqO++8o4MHD061vW3btrp48WJNSEjQiIgI\n9/sbExOjlSpV0hMnTqiq6t///ncdOHCgnj9/XlVVDx8+rJ999lnW3zwPGzZs0MjISE1ISNBdu3Zp\n5cqVNSkp6bJ2F9+X5ORk7devn06aNMm9LTk5WcuXL687d+5M9ZqUlBQtX768JiYG3v/ltP6NAyvV\ni89Y6+TODptmOQXsVk+BPT8568LsIsxkLjw8PM27fkaNGsWqVauIjIxk5MiRfPjhh4Az1rBkyRLq\n1KnDjBkzqFSpEgC1a9fm5ZdfpmPHjkRGRtKhQwcOHjyY6fGjo6OJiooiOjqa0aNHU7bs5TdAdO3a\nlUWLFgFO19Htt9+eavudd97JtGnTKFCgAB9//DH3338/UVFR9OrVi4kTJ1KiRAkAXn75ZcqUKUPt\n2rWpW7cu3bp182qMISN16tThrrvuonbt2nTq1Ilx48YRFhYGOHccrVy50h139erVqVmzJuXLl+f+\n++9372PJkiVUrFiRG2+8MdW+V61aRfPmzcmbN7T+L4t63L0QCBo3bqwX/6KzKmLkN9nbf3zqD5j7\nBGyeA9dHQo9xUC4y+/ZvfGLz5s3UqlXL32EEjIMHD9K/f38WLFjg71By1KOPPkr37t1p3769v0PJ\nsrT+jYvIKlVtnNlrQysFZrfvR8G27+CWUdBimF0dmKBUrlw5HnzwQa/GA4JJ3bp1AzIhXC37FMuq\n43sAhZIRTjJo8zerV2SC3tU+TxCIHnzwQX+H4Bc2puCtlBT49R0Y38KZ8wCg2PWWEIwxQSWkrhSu\neDzhyDangN2+5c7zBl3/m72BGWNMLhFSSeGKbJsPn/aD/IXh9ncg8m6rV2SMCVqWFNKTnOg8hBbe\nBOr3gZv/AUUzf0LUGGMCmY0pXCrxHCx4ASZ1hOQkKHwtdB9jCcFkq4iICGJiYmjXrl2q9Y899hgV\nKlQgJSXFvW7y5MkMHTo0hyP806hRo6hQoUKqEtMnTpzwWzyzZ89m9OjRV/Tai+/5okWLGDhwYJpt\nVqxYQdu2balRowYNGjRg0KBBnD17Ntv/Hrp06eJ+H8eMGUOtWrXo27fvVZ1fdrArBU97fnbGDuJ2\nQIN+kHzebjM1OSYlJYWvvvqKihUrsnjxYqKjo/0dktuIESN44okn/B0G4JQG6d69u0/2fejQIXr3\n7s306dNp0aIFAF988cVVl/hOy9y5c92/jx8/nu+//57w8HCALJ1fUlJStj5gZ1cKAOdPwzd/gw86\nO91G/WdBj7cgfxF/R2ZywgddL//5acyVb/dCmTJlCAsL49prr3WvW7RoEXXq1OHhhx9Os2RzcnIy\nlStXRlU5ceIEYWFhLFmyBIC2bduyfft2VqxYQYsWLWjQoAEtW7Zk69at7u0XJ48BaN26NWvXrmXx\n4sXub/8NGjTI0off66+/zgMPPADA+vXrqVu3LmfPnmXUqFH069ePFi1aUK1aNd577z3AKbndvn17\nGjZsmGpSn5iYGGrVqsWDDz5InTp16Nixo3ueiTFjxlC7dm0iIyPdZa09v7HHxMRw8803ExkZSfv2\n7dm7dy/gTO4zfPhwWrZsyY033uie6Ofie54/f373k9aexo0bx4ABA9wJAaBXr16XPek9Z84cmjVr\nRoMGDbjllls4dOgQQJrv58GDB2nbti1RUVHUrVvXXQn34uRHDz30ELt27aJz5868/vrrqc7vyJEj\n3HnnnTRp0oQmTZrw009OxYSL73GrVq3o16+f139nXvGmFkZu+rma2kfpSjil+kak6rcjVc+fzv79\nm1zlsrow73e5/GfZm1e+/QoNGjRIp0yZovHx8Vq+fHm9cOGCqqp+8MEHOmTIEFVVvfXWW3XDhg06\nZ84cbdy4sb788svumkOqqvHx8e5aPQsWLNA77rhDVVUnT56sjz76qKqqbt26VS/+P+rWrZsuW7ZM\nVVVPnTqVZp2fF154QcuXL6/169fX+vXra7t27VTVqRnUpk0bnTFjhjZq1Mi9nxdeeEEjIyP17Nmz\neuTIEQ0PD9f9+/drYmKixsfHq6rqkSNHtEqVKpqSkqK7d+/WsLAw/f3331VVtXfv3vrRRx+pqmq5\ncuU0ISFBVVWPHz9+2fvRrVs3nTx5sqqqTpo0SXv06KGqqgMGDNBevXppcnKybty4UatUqeLV38Ht\nt9+uM2fOTHOb53GPHTumKSkpqqr63nvv6eOPP57u+/mf//xHX375ZVVVTUpK0pMnT6pq6tpTnr97\nHueee+7RpUuXqqrqnj17tGbNmu73uGHDhnr27Nk0Y72a2keh2zdy9hj89Aa0ewYKFIWHf3HuMDKh\n5/5M5lO42u1euHDhAnPnzuV///sfxYoVo1mzZsyfP/+y6S3btGnDkiVL2L17N08//TTvvfceN910\nk7tkdnx8PAMGDGD79u2ICImJiQD07t2bl156iddee43333/f3Z/eqlUrHn/8cfr27csdd9zh7r64\nVFrdR3ny5GHy5MlERkby17/+lVatWrm39ejRg0KFClGoUCGio6NZsWIFXbt25ZlnnmHJkiXkyZOH\n/fv3u79he8541qhRI2JiYgCnvHjfvn3p2bMnPXv2vCyuX375xV1KvF+/fjz55JPubT179iRPnjzU\nrl3bfZzsEhsby913383Bgwe5cOEClStXBtJ+P5s0acIDDzxAYmIiPXv2dJ+nN77//ns2bdrkXj55\n8iSnT58GnC4mX1Rw9Wn3kYh0EpGtIrJDREamsV1EZIxr+zoRaejLeABQhY1fwbim8Ms42Puzs94S\ngvGj+fPnc+LECerVq0dERATLli1Lswupbdu2LF26lBUrVrgHKhctWkSbNm0AZ8rL6OhoNmzYwJw5\nc0hISACgcOHCdOjQgVmzZvHZZ5/Rt29fAEaOHMnEiRM5d+4crVq1YsuWLTz77LPuLpDMbN++naJF\ni3LgwIFU6y+dQ0FEmDp1KkeOHGHVqlWsWbOGsmXLuuO7OBsbOF08SUlJAHzzzTcMGTKE1atX06RJ\nE/d6b3juU72s8VanTh1WrVqVabthw4YxdOhQ1q9fzzvvvOM+j7Tez7Zt27JkyRIqVKjAwIEDmTJl\nitfnkJKSwvLly91Thu7fv98930WRIr7p3vZZUhCRMGAc0BmoDdwjIrUvadYZqOb6GQy87at4ADh5\nED69Dz4fCMUrwOBFUOVmnx7SGG9MmzaNiRMnEhMTQ0xMDLt372bBggWcPXs2VbumTZvy888/kydP\nHgoWLEhUVBTvvPOOe6rJ+Ph493wCkydPTvXaQYMGMXz4cJo0aeKenGfnzp3Uq1ePp556iiZNmrBl\nyxZeeeUV94dQRuLj4xk+fDhLliwhLi7O3W8PzuxsCQkJxMXFsWjRIpo0aUJ8fDzXXXcd+fLlY+HC\nhezZsyfD/aekpLBv3z6io6P517/+RXx8vPtb8kUtW7Z0Twk6depUd3K8UkOHDuXDDz/k119/da+b\nMWPGZVcanu/zxQq2kPb7uWfPHsqWLcuDDz7IoEGDWL16tdfxdOzYkbFjx7qXM/s7yQ6+vFJoCuxQ\n1V2qegGYDvS4pE0PYIqry2s5cI2IlPNZRJ8PhB3fQ4cXYdAPcH09nx3KGG+dPXuWefPm0bXrn4PU\nRYoUoXXr1syZMydV2wIFClCxYkX3FJRt2rTh1KlT1Kvn/Ft+8sknefrpp2nQoMFl36obNWpE8eLF\nU5WNfuONN6hbty6RkZHky5ePzp07pxnj66+/nuqW1JiYGEaMGMGQIUOoXr06kyZNYuTIkRw+fBhw\nun2io6Np3rw5zz33HOXLl6dv376sXLmSevXqMWXKFGrWrJnh+5KcnMx9991HvXr1aNCgAcOHD+ea\na65J1Wbs2LF88MEHREZG8tFHH/Hmm29muM/MlC1blunTp/PEE09Qo0YNatWqxfz581PNPAfOQG/v\n3r1p1KgRpUuXdq9P6/1ctGgR9evXp0GDBnz66afuWeu8MWbMGFauXElkZCS1a9dmwoQJV3V+3vBZ\n6WwR6QV0UtVBruV+QDNVHerR5mtgtKoucy3/ADylqisv2ddgnCsJKlWq1Cizbxjp+mM95C0Epate\n2etNUAjV0tkHDhygXbt2bNmyhTw+nC981KhRFC1aNNfcwhqKrqZ0dkDckqqq76pqY1VtXKZMmSvf\n0fX1LCGYkDRlyhSaNWvGK6+84tOEYAKfL+8+2g9U9FgOd63LahtjzFXq378//fv3z5FjjRo1KkeO\nY3zDl18ZfgOqiUhlEckP9AFmX9JmNtDfdRdScyBeVTOfQ9CYq+SrblNj/O1q/2377EpBVZNEZCgw\nHwgD3lfVjSLykGv7BGAu0AXYAZwF7k9vf8Zkl4IFCxIXF0epUqUuu3XSmECmqsTFxVGwYMEr3kdI\nzdFsDEBiYiKxsbHue8uNCSYFCxYkPDycfPnypVpvczQbk458+fK5n0A1xqRmtyEYY4xxs6RgjDHG\nzZKCMcYYt4AbaBaRI8AVPtJMaeBoNoYTCOycQ4Odc2i4mnO+QVUzffo34JLC1RCRld6MvgcTO+fQ\nYOccGnLinK37yBhjjJslBWOMMW6hlhTe9XcAfmDnHBrsnEODz885pMYUjDHGZCzUrhSMMcZkwJKC\nMcYYt6BMCiLSSUS2isgOERmZxnYRkTGu7etEpKE/4sxOXpxzX9e5rheRn0Wkvj/izE6ZnbNHuyYi\nkuSaDTCgeXPOItJORNaIyEYRWZzTMWY3L/5tlxCROSKy1nXOAV1tWUTeF5HDIrIhne2+/fxS1aD6\nwSnTvRO4EcgPrAVqX9KmC/AtIEBz4Fd/x50D59wSKOn6vXMonLNHux9xyrT38nfcOfD3fA2wCajk\nWr7O33HnwDk/A/zL9XsZ4BiQ39+xX8U5twUaAhvS2e7Tz69gvFJoCuxQ1V2qegGYDvS4pE0PYIo6\nlgPXiEi5nA40G2V6zqr6s6oedy0ux5nlLpB58/cMMAz4Ejick8H5iDfnfC8wQ1X3AqhqoJ+3N+es\nQDFxJscoipMUknI2zOyjqktwziE9Pv38CsakUAHY57Ec61qX1TaBJKvn8xecbxqBLNNzFpEKwO3A\n2zkYly958/dcHSgpIotEZJWI5MwcnL7jzTm/BdQCDgDrgUdVNSVnwvMLn35+2XwKIUZEonGSQmt/\nx5ID3gCeUtWUEJphLS/QCGgPFAJ+EZHlqrrNv2H51K3AGuBmoAqwQESWqupJ/4YVmIIxKewHKnos\nh7vWZbVNIPHqfEQkEpgIdFbVuByKzVe8OefGwHRXQigNdBGRJFWdmTMhZjtvzjkWiFPVM8AZEVkC\n1AcCNSl4c873A6PV6XDfISK7gZrAipwJMcf59PMrGLuPfgOqiUhlEckP9AFmX9JmNtDfNYrfHIhX\n1YM5HWg2yvScRaQSMAPoFyTfGjM9Z1WtrKoRqhoBfAE8EsAJAbz7tz0LaC0ieUWkMNAM2JzDcWYn\nb855L86VESJSFqgB7MrRKHOWTz+/gu5KQVWTRGQoMB/nzoX3VXWjiDzk2j4B506ULsAO4CzON42A\n5eU5Pw+UAsa7vjknaQBXmPTynIOKN+esqptFZB6wDkgBJqpqmrc2BgIv/55fAiaLyHqcO3KeUtWA\nLaktItOAdkBpEYkFXgDyQc58flmZC2OMMW7B2H1kjDHmCllSMMYY42ZJwRhjjJslBWOMMW6WFIwx\nxrhZUjC5jogku6p8XvyJyKBtRHrVJLN4zEWuSpxrReQnEalxBft46GJZCREZKCLlPbZNFJHa2Rzn\nbyIS5cVrHnM9s2BMpiwpmNzonKpGefzE5NBx+6pqfeBD4LWsvtj1nMAU1+JAoLzHtkGquilbovwz\nzvF4F+djgCUF4xVLCiYguK4IlorIatdPyzTa1BGRFa6ri3UiUs21/j6P9e+ISFgmh1sCVHW9tr2I\n/C7OPBTvi0gB1/rRIrLJdZz/uNaNEpEnxJm3oTEw1XXMQq5v+I1dVxPuD3LXFcVbVxjnL3gUQhOR\nt0VkpThzCvzTtW44TnJaKCILXes6isgvrvfxcxEpmslxTAixpGByo0IeXUdfudYdBjqoakPgbmBM\nGq97CHhTVaNwPpRjRaSWq30r1/pkoG8mx78NWC8iBYHJwN2qWg+nAsDDIlIKp/pqHVWNBF72fLGq\nfgGsxPlGH6Wq5zw2f+l67UV349RnupI4OwGeZTuedT2lHgncJCKRqjoGp3potKpGi0hp4B/ALa73\nciXweCbHMSEk6MpcmKBwzvXB6Ckf8JarDz0Zp0T0pX4BnhWRcJw5BbaLSHucqqG/ucp7FCL9uRWm\nisg5IAZnHoYawG6PWlEfAkNwSjUnAJNE5Gvga29PTFWPiMguV82a7TiF235y7TcrcebHmTvA8326\nS0QG4/y/LgfUxil34am5a/1PruPkx3nfjAEsKZjAMQI4hFPxMw/Oh3IqqvqJiPwKdAXmishfcWrh\nfKiqT3txjL6quvLigohcm1YjVz2epjhF2HoBQ3HKNntrOnAXsAX4SlVVnE9or+MEVuGMJ4wF7hCR\nysATQBNVPS4ik4GCabxWgAWqek8W4jUhxLqPTKAoARx0TZ7SD6c4WioiciOwy9VlMgunG+UHoJeI\nXOdqc62I3ODlMbcCESJS1bXcD1js6oMvoapzcZJVWvNdnwKKpbPfr3Bmz7oHJ0GQ1ThdZaKfA5qL\nSE2gOHAGiBenUmjndGJZDrS6eE4iUkRE0rrqMiHKkoIJFOOBASKyFqfL5Uwabe4CNojIGqAuzpSF\nm3D60L8TkXXAApyulUypagJOBcrPXRU4U4AJOB+wX7v2t4y0++QnAxMuDjRfst/jOOWsb1DVFa51\nWY7TNVbxX+DvqroW+B3n6uMTnC6pi94F5onIQlU9gnNn1DTXcX7BeT+NAaxKqjHGGA92pWCMMcbN\nkoIxxhg3SwrGGGPcLCkYY4xxs6RgjDHGzZKCMcYYN0sKxhhj3P4fpBfE+fcVBaEAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1184a57b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(y_test, y_pred_proba, classifier_label):\n",
    "    fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba) \n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    \n",
    "    figure, ax = plt.subplots(1,1)\n",
    "    \n",
    "    # if multiple_plots:\n",
    "    \n",
    "    ax.plot(fpr, tpr, lw=1, label = classifier_label + ' (AUC = %.2f)' %(roc_auc))\n",
    "    \n",
    "    label_kwargs = {}\n",
    "    label_kwargs['bbox'] = dict(\n",
    "                boxstyle='round,pad=0.3', fc='darkorange', alpha=0.5,\n",
    "            )\n",
    "    \n",
    "#     for thres in [0.3, 0.5, 0.6]:\n",
    "#         idx = np.where(np.round(thresholds, 2) == thres)[0][0]\n",
    "#         threshold = np.round(thresholds[idx], 2)\n",
    "#         ax.annotate('Threshold = ' + str(thres), xy=(fpr[idx], tpr[idx]),\n",
    "#                 xytext=(fpr[idx]-0.12, tpr[idx]+0.06),\n",
    "#                 arrowprops=dict(facecolor='gray', shrink=5, headwidth=2), horizontalalignment='left',\n",
    "#                     fontsize=8\n",
    "#                    )\n",
    "               \n",
    "    ax.plot([0, 1], [0, 1], linestyle='--', label='\"Always-Expansion\" Classifier')\n",
    "    \n",
    "    ax.set_xlim([-0.05, 1.05])\n",
    "    ax.set_ylim([-0.05, 1.05])\n",
    "    \n",
    "    ax.set_xlabel('False Positive Rate')\n",
    "    ax.set_ylabel('True Positive Rate')\n",
    "    \n",
    "    ax.set_title('ROC Curve')\n",
    "    plt.legend(frameon=True, loc='lower right')\n",
    "    \n",
    "\n",
    "recession_probs_A = all_predictions_topics[0]['p1'].as_data_frame()\n",
    "recession_probs_B = all_predictions_topics_CFNAI[0]['p1'].as_data_frame()\n",
    " \n",
    "plot_roc_curve(test['D_NBER'], recession_probs_A, classifier_label='Model A')\n",
    "plot_roc_curve(test['D_NBER'], recession_probs_B, classifier_label='Model B')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4FOX2wPHvSQi9ShUChF5CQuhVBCnSFKQoXgQs2BHF\n61UsF0LxJ5YrChYU9SKKYKOIcGkK0juho7QAoRMg1EBIzu+PXdYAKRvIZpPN+TxPHjMz786cWeKe\nfd+ZOa+oKsYYYwyAn7cDMMYYk3lYUjDGGONiScEYY4yLJQVjjDEulhSMMca4WFIwxhjjYknBGGOM\niyUF41NEJFJELorIORE5IiITRCT/dW2aisjvInJWRGJEZKaI1LyuTUER+UBE9jv3tdu5XCyZ44qI\nDBSRLSJyXkSiRORHEQnx5Pkak94sKRhfdI+q5gfCgDrAq1c3iEgTYB4wAygNVAA2AstEpKKzTU7g\nNyAYaA8UBJoAJ4CGyRzzQ+B5YCBwG1AVmA50SmvwIpIjra8xJr2IPdFsfImIRAL9VXWBc/kdIFhV\nOzmXlwCbVfWZ6173P+C4qvYVkf7Am0AlVT3nxjGrADuAJqq6Opk2i4BvVfUL5/LDzjibO5cVGAC8\nAOQA5gDnVfWlRPuYAfyhqu+LSGlgLNACOAeMVtUxbrxFxqTIegrGZ4lIINAB2OVczgs0BX5MovkP\nQFvn722AOe4kBKfWQFRyCSENugKNgJrAZOABEREAESkCtAOmiIgfMBNHD6eM8/gviMjdt3h8Yywp\nGJ80XUTOAgeAY8BQ5/rbcPzNH07iNYeBq9cLiibTJjlpbZ+ct1T1pKpeBJYACtzh3NYDWKGqh4AG\nQHFVHa6ql1V1DzAe6JUOMZhszpKC8UVdVbUA0BKozt8f9qeABOD2JF5zO45rBgDRybRJTlrbJ+fA\n1V/UMa47BXjQueofwCTn7+WB0iJy+uoP8BpQMh1iMNmcJQXjs1T1D2AC8J5z+TywAuiZRPP7cVxc\nBlgA3C0i+dw81G9AoIjUT6HNeSBvouVSSYV83fJkoIeIlMcxrPSzc/0BYK+qFk70U0BVO7oZrzHJ\nsqRgfN0HQFsRqe1cHgz0c94+WkBEiojISBx3Fw1ztvkGxwfvzyJSXUT8RKSoiLwmIjd88KrqTuAT\nYLKItBSRnCKSW0R6ichgZ7MIoJuI5BWRysBjqQWuqhtw9F6+AOaq6mnnptXAWRF5RUTyiIi/iNQS\nkQY38wYZk5glBePTVPU4MBEY4lxeCtwNdMNxHWAfjttWmzs/3FHVSzguNu8A5gNncHwQFwNWJXOo\ngcBHwMfAaWA3cB+OC8IAo4HLwFHga/4eCkrNd85Yvkt0TvFAZxy33O7l78RRyM19GpMsuyXVGGOM\ni/UUjDHGuFhSMMYY42JJwRhjjIslBWOMMS5ZrvBWsWLFNCgoyNthGGNMlrJu3boTqlo8tXZZLikE\nBQWxdu1ab4dhjDFZiojsc6edDR8ZY4xxsaRgjDHGxZKCMcYYF0sKxhhjXCwpGGOMcfFYUhCRr0Tk\nmIhsSWa7iMgYEdklIptEpK6nYjHGGOMeT/YUJuCY9Dw5HYAqzp8ngE89GIsxxhg3eCwpqOpi4GQK\nTboAE9VhJVBYRNJj9ipjjPEpcQlxRMZEZsixvPnwWhkSTT8IRDnX3TDXrYg8gaM3Qbly5TIkOGMy\nzKjyEHs69XYmW9qeM4ChxYoS7e/HzKjD5B3q2b+VLPFEs6p+DnwOUL9+fZsAwviW2NMQHpPmlwUN\nnkXkqE4eCMhkFgmawOAZ9xFzKYY3Gr9B3vJtPH5MbyaFg0DZRMuBznXGGJOtRRyLoGqRquQNyMt7\nd75HibwlKJQrYybW8+Ytqb8AfZ13ITUGYlT1hqEjY4zJLs7HnefNlW/S5399+Hrr1wBUKVIlwxIC\neLCnICKTgZZAMRGJAoYCAQCqOg6YDXQEdgEXgEc8FYsxxmR2yw4uY9iKYRw5f4TeNXrTL7ifV+Lw\nWFJQ1QdT2a7As546vjHGZBWfbfyMjyI+okKhCkzsMJGwEmFeiyVLXGg2xhhfFJcQR4BfAHcE3sGl\n+Es8WftJcvnn8mpMlhSMMSaDHb9wnP9b9X8UzFWQYU2HUbNoTWoWrentsACrfWSMMRlGVZm2cxpd\nZnRhcdRiyhUoh2MkPfOwnoIxxmSAQ+cOEb48nBWHV1C3RF2GNR1GUKEgb4d1A0sKxhiTAWLjY9lx\ncgdvNHqDntV64ieZc6DGkoIxxnjIntN7mBM5h2fCnqFioYrM6zGP3DlyezusFGXfpGD1Zkxmkbuw\n201rD5tHzMU4AArlCfBUROYWxSXE8d8t/2XcxnHkDchL9yrdKZmvZKZPCJCdk8JN1psxxptiLsZZ\nvaNMbmv0VoYsG8Jfp/6ifVB7BjccTNE8Rb0dltuyb1Iwxph0diHuAk/Me4Jc/rn4sNWH3FXuLm+H\nlGaWFIwx5hbtOLmDakWqkTcgL6NbjqZ60eoUzFnQ22HdlMx5+dsYY7KAc5fPMXLlSHrO7MnsvbMB\naHh7wyybEMB6CsYYc1MWRy1mxMoRHD1/lIdqPESrsq28HVK6sKRgjDFpNGr1KCZtn0SlQpX4puM3\n1C5e29shpRtLCsYY44ar5ShEhNBioRSoXYDHQx4np39OL0eWviwpGGNMKo5dOMaIlSNoVKoRD9V8\niI4VO3o7JI+xC83GGJMMVeXnv36m6/SurDi0ItOWpkhP1lMwxpgkHDh7gGHLh7HqyCrql6zPsKbD\nKFewnLfD8rjslRQSl7ZIQ2mBrCBx+QPju6y0RcbZG7PX8XRykyF0r9I9W/QSILslBR8ubWHlD4y5\ndbtO7WJr9Fa6VO5Ci8AWzOk+h0K5Cnk7rAyVvZKCMcYkIS4+ji+2fMHnmz7ntly30S6oHXly5Ml2\nCQEsKRhjsrktJ7YwZPkQdp7aSYcKHRjccDB5cuTxdlheY0nBGJNtHTx3kD6z+3BbntsYe9dYWpZt\n6e2QvM6SgjEm2zlw9gBlC5SlTP4yjGw+khaBLSiQs4C3w8oUssfldGOMAc5ePsuwFcO4Z9o9bD6+\nGYBOFTtZQkjEegrGmGzhjwN/MHzlcE5cPEHfmn2pXKSyt0PKlCwpGGN8mqry+tLXmblnJpULV+aD\nlh8QUjzE22FlWpYUjDE+TUQoU6AMz4Q9Q/9a/QnwtwcAU2JJwRjjc46cP8KbK9/kHzX+QZPSTXg2\n7Flvh5Rl2IVmY4zPSNAEfvzrR+6bcR8rD6/k6IWj3g4py7GeQiaWlnpGVhPHZHf7z+wnfEU4a46s\noVGpRgxtOpSyBcp6O6wsx5JCJmb1jIxx3/x989kevZ3wJuF0q9INEfF2SFmSR4ePRKS9iPwpIrtE\nZHAS2wuJyEwR2SgiW0XkEU/GY4zxLX+d+ouVh1cC0De4LzO6zqB71e6WEG6Bx5KCiPgDHwMdgJrA\ngyJS87pmzwLbVLU20BL4j4j41tx2xph0dzn+Mh9HfMwDMx/g3TXvoqoE+AVQIm8Jb4eW5Xly+Kgh\nsEtV9wCIyBSgC7AtURsFCogjrecHTgJXPBiTMSaL23h8I0OXDWV3zG46V+zMKw1esZ5BOvJkUigD\nHEi0HAU0uq7NR8AvwCGgAPCAqiZcvyMReQJ4AqBcOd+f+cgYk7QNxzbQ73/9KJG3BB+3/pgWgS28\nHZLP8fYtqXcDEUBpIAz4SEQKXt9IVT9X1fqqWr948eIZHaMxxstOxp4EoHbx2vyz/j+Z3mW6JQQP\n8WRSOAgkvh8s0LkusUeAqeqwC9gLVPdgTMaYLOTM5TOELw/nnmn3cPzCcfzEj37B/cifM7+3Q/NZ\nnkwKa4AqIlLBefG4F46hosT2A60BRKQkUA3Y48GYjDFZxO/7f6fr9K5M2zWN7lW7WyXTDOKxawqq\nekVEBgBzAX/gK1XdKiJPObePA0YAE0RkMyDAK6p6wlMxGWMyv0vxl3hj6RvMiZxD1SJVGXvXWIKL\nBXs7rGzDow+vqepsYPZ168Yl+v0Q0M6TMRhjspacfjm5knCFAWEDeDTkUQL87Gn9jOTtC83GGMOR\n80d4cdGLHDh7ABHh/Zbv82TtJy0heIGVuchkEtc7snpGxtclaAI//vkj7697H0XpVLETZQuUtecO\nvMiSQiZj9Y5MdhEZE8nQ5UNZf2w9TW5vwpAmQwgsEOjtsLI9SwrGGK8Yv3k8O0/vZESzEXSp1MV6\nB5mEJQVjTIb58+SfBPgFULFwRf5V/1+8UPcFiue1B1IzE7vQbIzxuMvxlxm7YSy9fu3F++veB6Bw\n7sKWEDIh6ykYYzwq4lgEQ5cPZU/MHu6tdC//qv8vb4dkUmBJwRjjMfMi5/HSHy9RKl8pPm3zKc3L\nNPd2SCYVlhSMMenuQtwF8gbkpWnppvQP6c9jIY+RLyCft8MybnDrmoKI5BSRyp4OxhiTtcVciuHf\ny/5N79m9uRx/mfw58zOw7kBLCFlIqklBRDoBm4H5zuUwEZnm6cCMMVnLgn0L6DqjKzN3z6Rl2Zbe\nDsfcJHeGj4bjmBxnIYCqRlivwRhzVcylGIatGMb8ffOpflt1Pmn9CTWK1vB2WOYmuZMU4lT19HUP\nlqiH4sk2EpezSMxKW5isJpd/Lvad2cfzdZ+nX3A/q1eUxbmTFLaLyP2An4hUAAYCKz0blu+zchYm\nKzt07hDjNo5jcMPB5A3Iy/edvyeHn9234gvcudA8AKgHJABTgUvA854MyhiTOSVoAt9t/46uM7oy\nJ3IO26K3AVhC8CHu/EveraqvAK9cXSEi3XAkCGNMNrEnZg/hy8PZcGwDzUo3Y0iTIZTOX9rbYZl0\n5k5P4Y0k1r2e3oEYYzIvVWXosqHsPr2bkc1G8mmbTy0h+KhkewoicjfQHigjIu8n2lQQx1CSMcbH\nbY/eTun8pSmUqxAjmo0gf878FMtTzNthGQ9KqadwDNgCxAJbE/3MAzp4PjRjjLdcir/EB+s+4MFZ\nD/Lpxk8BCCoUZAkhG0i2p6CqG4ANIjJJVWMzMCZjjBetP7qeocuHEnkmkq6Vu/J07ae9HZLJQO5c\naC4jIm8CNYHcV1eqalWPRWWM8YrJOybz1qq3KJ2/NJ+1/YympZt6OySTwdxJChOAkcB7OIaNHsEe\nXjPGp8QlxBHgF0DT0k15qOZDDAgbQN6AvN4Oy3iBO3cf5VXVuQCqultV38CuKRjjE2IuxfD60td5\n+Y+XAShfsDwvN3jZEkI25k5SuCQifsBuEXlKRO4BCng4LmOMB6kq8yLnce/0e5m9ZzYVC1ckPiHe\n22GZTMCd4aNBQD4c5S3eBAoBj3oyKF+VuN6R1Tgy3nLi4glGrhzJb/t/o2bRmnze9nOq3VbN22GZ\nTCLVpKCqq5y/ngX6AIhIGU8G5aus3pHJDK4kXCHiWAQv1nuRPjX7WIkKc40Uh49EpIGIdBWRYs7l\nYBGZCKxK6XXGmMwl6mwUY9aPQVUpla8Uc7rP4ZFaj1hCMDdINimIyFvAJKA3MEdEwnHMqbARsNtR\njckC4hPi+Xbbt3T7pRvf7fiOvWf2ApA7R+5UXmmyq5S+JnQBaqvqRRG5DTgAhKjqnowJzRhzK3af\n3s3Q5UPZeHwjzcs0Z2iToZTKV8rbYZlMLqWkEKuqFwFU9aSI/GUJwZisIS4hjifnP8ml+Eu8dcdb\ndKrQiesmyjImSSklhYoicrU8tgAVEi2jqt1S27mItAc+BPyBL1R1VBJtWgIfAAHACVW90/3wjTGJ\n/XXqLyoVqkSAXwDvtHiH8gXLUzRPUW+HZbKQlJJC9+uWP0rLjkXEH/gYaAtEAWtE5BdV3ZaoTWHg\nE6C9qu4XkRJpOYYxxiH2SiyfbPyEiVsn8nKDl/lHjX9Qt2Rdb4dlsqCUCuL9dov7bgjsujrkJCJT\ncFyn2JaozT+Aqaq633nMY7d4TGOynTVH1jBsxTD2ndlH9yrd6Vyps7dDMlmYJ+9HK4Pj4vRVUUCj\n69pUBQJEZBGOp6Q/VNWJ1+9IRJ4AngAoV66cR4I1Jiv6JOITPt34KYH5AxnfbjyNb2/s7ZBMFuft\nm5Rz4Jj/uTWQB1ghIitV9a/EjVT1c+BzgPr161sxPpPtqSoiQkixEPrU7GMF7Ey6cTspiEguVb2U\nhn0fBMomWg50rkssCohW1fPAeRFZDNQG/sIYc4NTsad4e83blC1QlmfDnuWOwDu4I/AOb4dlfEiq\nBfFEpKGIbAZ2Opdri8hYN/a9BqgiIhVEJCfQC/jlujYzgOYikkNE8uIYXtqepjPI5GoPm0fQ4FkE\nDZ5l9Y7MTVNV5uydQ9cZXZm7dy4Bfva3ZDzDnZ7CGKAzMB1AVTeKSKvUXqSqV0RkADAXxy2pX6nq\nVhF5yrl9nKpuF5E5wCYc8z5/oapbbvJcMiWrd2Ru1bELxxixcgSLDiwiuGgw49uNp2oRKypgPMOd\npOCnqvuue/DFrRq7qjobmH3dunHXLb8LvOvO/ozJjqLORrHq8Cpeqv8SvWv0tnpFxqPc+es6ICIN\nAXU+e/AcNuZvjEcdOHuAFYdWcH+1+6lbsi7zus+jcO7C3g7LZAPuJIWncQwhlQOOAguc64wx6Sw+\nIZ5J2ycxdsNYAvwDaFe+HYVzF7aEYDKMO0nhiqr28ngkxmRzO0/tZOjyoWw+sZk7A+/kjcZvWDIw\nGc6dpLBGRP4Evsfx9PFZD8dkTLZzKvYUvWf3Jrd/bt5p8Q7tg9pbATvjFe7MvFZJRJriuKV0mIhE\nAFNUdYrHozPGx0WdjSKwQCBFchdhZLORNCjVgCK5i3g7LJONpfqcAoCqLlfVgUBd4AyOyXeMMTfp\n4pWLvLfmPTpN68TyQ8sBaBfUzhKC8bpUewoikh9HIbteQA0cD5w19XBcxvisNUfWMHT5UA6cPUDP\nqj0JKRbi7ZCMcXHnmsIWYCbwjqou8XA8xvi0t1e/zbfbv6VsgbJ8dfdXNCjVwNshGXMNd5JCRVVN\n8HgkPqD2sHnEXIy7Zp2VtjCJlcpXioeDH+aZsGfIkyOPt8Mx5gbJJgUR+Y+q/hP4WURuqEzqzsxr\n2Y2VtDDXOxl7klGrR9G6XGvuDrqbfsH9vB2SMSlKqafwvfO/aZpxzRjjKGA3e+9sRq0exbm4c4QW\nC/V2SMa4JaWZ11Y7f62hqtckBmehu1udmc0Yn3Tk/BFGrhzJH1F/EFoslGFNh1G5SGVvh2WMW9y5\nJfXRJNY9lt6BGOMrlh5cyuojq3m5wctM7DDREoLJUlK6pvAAjttQK4jI1ESbCgCnPR2YMVnJ/jP7\niTwTSYvAFnSr0o3mZZpTKl8pb4dlTJqldE1hNRCNY8a0jxOtPwts8GRQxmQVVxKu8O22b/ko4iOK\n5i5Kk25NCPALsIRgsqyUrinsBfbiqIpqjLnOnyf/ZOjyoWyN3kqrsq14o/EbNiOayfJSGj76Q1Xv\nFJFTQOJbUgVQVb3N49EZk0ntOrWLXr/2omCugrx757vcXf5uK2BnfEJKw0dXp9wslhGBGJMVnIo9\nRZHcRahUuBIv1n+RzhU7W70i41OSvfso0VPMZQF/VY0HmgBPAvkyIDZjMo0LcRd4Z807tP+5PfvP\n7EdE6FOzjyUE43PcuSV1Oo6pOCsB/wWqAN95NCpjMpGVh1fS7ZdufLPtG+6pdA+35baRU+O73Kl9\nlKCqcSLSDRirqmNExO4+Mj4vPiGe4SuHM3XnVMoXLM9/7/4v9UvV93ZYxniUW9NxikhPoA/Q1bnO\nbrEwPs/fz58ETeDRWo/ydO2nyZ0jt7dDMsbj3EkKjwLP4CidvUdEKgCTPRuWMd5x4uIJ3lnzDo/V\neoxqt1VjeNPhdleRyVZSvaagqluAgcBaEakOHFDVNz0emTEZSFWZuXsmXWd0ZcG+Bew4uQPAEoLJ\ndtyZee0O4BvgII5nFEqJSB9VXebp4IzJCIfPHWb4yuEsPbiU2sVrM7zpcCoWrujtsIzxCneGj0YD\nHVV1G4CI1MCRJOyKm/EJ32z/hnVH1zG44WB6VeuFv5+/t0MyxmvcSQo5ryYEAFXdLiI5PRiTMR63\nN2YvsVdiqVG0Bs+GPcs/qv+DwAKB3g7LGK9zJymsF5FxwLfO5d5YQTyTRV1JuMKErRP4NOJTahat\nyTcdvyFfQD7yBdjzmMaAe0nhKRwXml92Li8BxnosImM8ZMfJHQxZNoTtJ7fTplwbXmv0mrdDMibT\nSTEpiEgIUAmYpqrvZExIxqS/FYdW8MyCZyiUqxDvt3yftuXbejskYzKllKqkvoZjhrX1QAMRGa6q\nX2VYZMakgwtxF8gbkJe6JevSL7gfj9R6hEK5Cnk7LGMyrZSeU+gNhKpqT6AB8HRady4i7UXkTxHZ\nJSKDU2jXQESuiEiPtB7DmKRciLvAW6veotsv3Tgfd55c/rl4od4LlhCMSUVKw0eXVPU8gKoeFxF3\niue5iIg/jhnb2gJRwBoR+SXxnUyJ2r0NzEtT5B5Se9g8Yi7G3dRrC+Wx6h+ZwfKDyxm2YhiHzx+m\nV/VeCPYAmjHuSikpVEw0N7MAlRLP1ayq3VLZd0Ngl6ruARCRKUAXYNt17Z4DfsbRG/G6mItxRI7q\n5O0wzE24EHeB/1v1f8zYPYOggkFMaD+BuiXrejssY7KUlJJC9+uWP0rjvssABxItRwGNEjcQkTLA\nfTgm9Ek2KYjIE8ATAOXKlUtjGCa7yOmfk71n9tI/pD9P1X6KXP65vB2SMVlOSnM0/5YBx/8AeEVV\nE1KqMaOqnwOfA9SvX1+TbWiynRMXTzB2w1gG1R1E4dyF+br91+Twc+dOa2NMUjz5f89BHLO2XRXo\nXJdYfWCKMyEUAzqKyBVVne7BuIwPUFVm7J7Bu2veJfZKLK3LtaZFYAtLCMbcIk/+H7QGqOIstX0Q\n6AX8I3EDVa1w9XcRmQD8agnBpObguYMMXzGc5YeWU7dEXcKbhlOhUIXUX2iMSZXbSUFEcqnqJXfb\nq+oVERkAzAX8ga9UdauIPOXcPi7N0RoDvLXqLSKORfB6o9e5v9r9+KXtxjhjTArcKZ3dEPgSKASU\nE5HaQH9VfS6116rqbGD2deuSTAaq+rA7AZvsaU/MHvIH5KdE3hK82uhVBKF0/tLeDssYn+POV6wx\nQGcgGkBVN+K4W8gYj4tLiGP8pvH0+KUHo9eNBqBM/jKWEIzxEHeGj/xUdd91dwfFeygeY1y2RW9j\n6PKh7Di5g7bl2/LP+v/0dkjG+Dx3ksIB5xCSOp8+fg74y7Nhmezu1z2/8sbSNyiSuwijW46mTfk2\n3g7JmGzBnaTwNI4hpHLAUWABN1EHKTNLXNrCSlV4V1xCHAF+ATQo2YAeVXvwXJ3nrF6RMRko1aSg\nqsdw3E7qs6y0hfedjzvP6HWjiYyJZHy78ZTMV5I3Gr/h7bCMyXbcuftoPHDDU8Sq+oRHIjLZzpKo\nJQxfOZyj54/Su0ZvriRcIcDfemzGeIM7w0cLEv2eG0etogPJtDXGbTGXYnh79dvM3DOTioUqMrHD\nRMJKhHk7LGOyNXeGj75PvCwi3wBLPRaRyTZUlVVHVvFk6JM8EfoEOf1zejskY7K9mylzUQEomd6B\nmOzh+IXjfLP9GwbWGUjh3IX59b5fyZMjj7fDMsY4uXNN4RR/X1PwA04Cyc6iZkxSVJXpu6bz7tp3\nuRx/mbbl2hJSPMQSgjGZTIpJQRxPrNXm7+qmCapqpatNmkSdjWLYimGsPLySeiXrEd4knKBCQd4O\nyxiThBSTgqqqiMxW1VoZFZDxLQmawIDfBnDkwhH+3fjf9KjawwrYGZOJuXNNIUJE6qjqBo9HY3zG\nnpg9lMlfhlz+uRjRbATF8xanVL5S3g7LGJOKZL+yicjVhFEHWCMif4rIehHZICLrMyY8k9XExccx\nbuM4evzSg6+2fAVASPEQSwjGZBEp9RRWA3WBezMoFpPFbT2xlSHLh/DXqb/oENSBB6o94O2QjDFp\nlFJSEABV3Z1BsWQoq3eUviZtn8Q7a96hWO5ijGk1hlblrLq6MVlRSkmhuIi8mNxGVX3fA/FkGKt3\nlD5UFREhuGgw91W+jxfrv0jBnAW9HZYx5iallBT8gfw4ewzGJHbu8jlGrxtNDr8cvNroVcJKhFmJ\nCmN8QEpJ4bCqDs+wSEyWsThqMcNXDOf4xeP0qdHH1VswxmR9qV5TMOaqU7GneHvN28zaM4vKhSvz\nfsv3CS0e6u2wjDHpKKWk0DrDojBZwrELx/h9/+88XftpHg953MpbG+ODkk0KqnoyIwMxmdPR80dZ\nsH8BvWv0ptpt1ZjXfR6Fcxf2dljGGA+5mSqpJhtQVX7e+TP/WfsfriRc4a6yd3F7/tstIRjj4ywp\nmBscOHOA8BXhrD6ymoalGhLeJJzb89/u7bCMMRnAkoK5xsUrF+k9uzdxCXEMbTKU7lW6251FxmQj\nlhQMAAfPHaR0vtLkyZGH8Kbh1Cxa0+oVGZMNWQ3jbC4uPo5PIz6l87TOzNs3D4C7yt1lCcGYbMp6\nCtnY5uObGbJ8CLtO76JjhY40LNXQ2yEZY7zMkkI29WnEp4zbNI5ieYrx0V0fcWfZO70dkjEmE7Ck\nkE2VzFeS7lW6M6jeIArkLODtcIwxmYRHk4KItAc+xFFc7wtVHXXd9t7AKzhKapwFnlbVjZ6MKbs6\ne/ks/1n7H4KLBdOzak+6VelGtyrdvB2WV8TFxREVFUVsbKy3QzEm3eXOnZvAwEACAm6u4oDHkoKI\n+AMfA22BKByzt/2iqtsSNdsL3Kmqp0SkA/A50MhTMWVXiw4sYsSKEZyIPUHJfCW9HY7XRUVFUaBA\nAYKCgux2W+NTVJXo6GiioqKoUKHCTe3Dkz2FhsAuVd0DICJTgC6AKymo6vJE7VcCgR6MJ9uJvhjN\n26vf5n+R/6NKkSqMuWsMwcWCvR2W18XGxlpCMD5JRChatCjHjx+/6X14MimUAQ4kWo4i5V7AY8D/\nktogIk8ATwCUK1cuveLzeeuPrWf+/vk8G/Ysj9V6zArYJWIJwfiqW/3bzhQXmkWkFY6k0Dyp7ar6\nOY6hJerXr68ZGFqWc+T8Ebae2Err8q1pU64Ns+6bRen8pb0dljEmi/Dkw2sHgbKJlgOd664hIqHA\nF0AXVY2rrLu+AAAdzElEQVT2YDw+LUET+OHPH+g6oyvhK8K5EHcBEbGEkEmJCA899JBr+cqVKxQv\nXpzOnTunaT9BQUGcOHHipttEREQgIsyZMyfZ16sqd911F2fOnHGtmz59OiLCjh07XOsWLVp0Q/wP\nP/wwP/30E+C4wD948GCqVKlC3bp1adKkCf/7X5KDA2ny1ltvUblyZapVq8bcuXOTbBMREUHjxo0J\nCwujfv36rF69GoBJkyYRFhbm+vHz8yMiIgKANm3acOrUqVuOL6vxZFJYA1QRkQoikhPoBfySuIGI\nlAOmAn1U9S8PxuLT9p3Zx2NzH2PEyhHUKlaL7zp9R96AvN4Oy6QgX758bNmyhYsXLwIwf/58ypQp\nk+FxTJ48mebNmzN58uRk28yePZvatWtTsGDBNL3uev/+9785fPgwW7ZsYf369UyfPp2zZ8/eUvzb\ntm1jypQpbN26lTlz5vDMM88QHx9/Q7uXX36ZoUOHEhERwfDhw3n55ZcB6N27NxEREURERPDNN99Q\noUIFwsIc08r26dOHTz755Jbiy4o8lhRU9QowAJgLbAd+UNWtIvKUiDzlbDYEKAp8IiIRIrLWU/H4\nqiPnj9Djlx78efJPhjcdzvi24ylboGzqLzRe17FjR2bNmgU4PmQffPBB17aTJ0/StWtXQkNDady4\nMZs2bQIgOjqadu3aERwcTP/+/VH9ezT122+/pWHDhoSFhfHkk08m+eGYmKry448/MmHCBObPn5/s\nLbqTJk2iS5curuVz586xdOlSvvzyS6ZMmeLWuV64cIHx48czduxYcuXKBUDJkiW5//773Xp9cmbM\nmEGvXr3IlSsXFSpUoHLlyq5eQGIi4urpxMTEULr0jT3oyZMn06tXL9fyvffem6ak5ys8ek1BVWcD\ns69bNy7R7/2B/p6MwVedij1FkdxFKJWvFC/Wf5HW5VpTIm8Jb4eVJQUNnpXu+4wc1SnVNr169WL4\n8OF07tyZTZs28eijj7JkyRIAhg4dSp06dZg+fTq///47ffv2JSIigmHDhtG8eXOGDBnCrFmz+PLL\nLwHYvn0733//PcuWLSMgIIBnnnmGSZMm0bdv32SPv3z5cipUqEClSpVo2bIls2bNonv37je0W7Zs\nGZ999plrecaMGbRv356qVatStGhR1q1bR7169VI81127dlGuXLlrehvJGTRoEAsXLkzy/Ro8ePA1\n6w4ePEjjxo1dy4GBgRw8eMMoNR988AF33303L730EgkJCSxfvvyGNt9//z0zZsxwLRcpUoRLly4R\nHR1N0aJFU43bV2SKC83GfZfjL/P5ps/5euvXTOwwkRpFa/Bg9QdTf6FJljsf4J4QGhpKZGQkkydP\npmPHjtdsW7p0KT///DMAd911F9HR0Zw5c4bFixczdepUADp16kSRIkUA+O2331i3bh0NGjQA4OLF\ni5QokfKXhMTfjHv16sXEiROTTAonT56kQIEC17zu+eefd71u8uTJ1KtXL9m7XtJ6N8zo0aPT1N4d\nn376KaNHj6Z79+788MMPPPbYYyxYsMC1fdWqVeTNm5datWpd87oSJUpw6NAhSwomc4o4FsHQ5UPZ\nE7OHeyvdy+35bOKbrO7ee+/lpZdeYtGiRURH3/x9FqpKv379eOutt9xqHx8fz88//8yMGTN48803\nXQ89nT179poEAJAjRw4SEhLw8/Pj5MmT/P7772zevBkRIT4+HhHh3XffpWjRojdcmD158iTFihWj\ncuXK7N+/nzNnzqTaW0hLT6FMmTIcOPD3ne9RUVFJXpv5+uuv+fDDDwHo2bMn/ftfO0AxZcqUa4bv\nroqNjSVPnjwpxutzVDVL/dSrV09v2tCCrl/Lv/Lrze8ngyUkJOjbq9/WkAkh2ubHNrr4wGJvh5Sl\nbdu2zdshaL58+VRV9cCBA/rhhx+qqurChQu1U6dOqqr63HPP6fDhw13rw8LCXOtHjBihqqqzZ89W\nQI8fP65bt27VypUr69GjR1VVNTo6WiMjI1VVtXz58nr8+PFrjj937lxt167dNev69u2rX3/99Q2x\nNmrUSHfu3Kmqqp999pk+8cQT12xv0aKF/vHHHxobG6tBQUGu9zcyMlLLlSunp0+fVlXVf/3rX/rw\nww/rpUuXVFX12LFj+sMPP6TtjbvOli1bNDQ0VGNjY3XPnj1aoUIFvXLlyg3tqlevrgsXLlRV1QUL\nFmjdunVd2+Lj47V06dK6e/fua16TkJCgpUuX1ri4uFuK0RuS+hsH1qobn7E2n0IWICKoKg9Ue4Dp\nXaZzR+Ad3g7JpJPAwEAGDhx4w/rw8HDWrVtHaGgogwcP5uuvvwYc1xoWL15McHAwU6dOdT3MWbNm\nTUaOHEm7du0IDQ2lbdu2HD58ONnjTp48mfvuu++add27d0/ywmqnTp1YtGhRqq/LlSsX3377LY88\n8ghhYWH06NGDL774gkKFCgEwcuRIihcvTs2aNalVqxadO3d26xpDSoKDg7n//vupWbMm7du35+OP\nP8bf3x+A/v37s3at496V8ePH889//pPatWvz2muv8fnnn7v2sXjxYsqWLUvFihWv2fe6deto3Lgx\nOXJkswEVdzJHZvrJLj2FmEsxOmTZEF13ZJ2qOr61mPSRGXoKWcmhQ4e0TZs23g4jww0cOFAXLFjg\n7TBuivUUfMxv+3+j6/SuzNg1g+0ntwNWlsF4z+23387jjz9+zcNr2UGtWrVo3bq1t8PIcNmsX5S5\nnbh4grdWvcW8ffOoVqQaY1uPJbioFbAz3nerzxNkRY8//ri3Q/AKSwqZyM9//czCAwsZWGcgD9d6\nmAA/K2BnjMlYlhS87PC5wxy7eIzaxWvzSK1HaBfUjgqFbq4OujHG3Cq7puAlCZrA5B2T6TqjK0OW\nDSFBE8jpn9MSgjHGq6yn4AV7Y/YSvjyc9cfW0+T2JgxtOhQ/sfxsjPE++yTKYFtObKHHLz3YdXoX\nI5uN5LO2n1Emf8ZXxzTelRlKZwcFBRESEkJYWBghISHX1P1JTH2gdPbGjRtp0qQJISEh3HPPPa5z\nsdLZN7KkkEEuxF0AoMZtNegb3JcZXWfQpXIXu9U0m8ospbMXLlxIREQEP/30U5IP0YFvlM7u378/\no0aNYvPmzdx33328++67gJXOToolBQ+7FH+JMevHcM+0ezgVewp/P3+er/s8xfIU83Zoxsu8XTo7\nsTNnzriK613PF0pn//XXX7Ro0QKAtm3buooNJmalsx3smoIHRRyLYMjyIeyN2cu9le616waZVXgh\nD+wzJtUm3i6dDdCqVStUlT179vDDDz8k2cYXSmcHBwczY8YMunbtyo8//nhNEb2rrHS2gyUFD4iL\nj+O9te8xecdkSuUrxbg242hWppm3wzLJceMD3BO8XTobHMNHxYoVY/fu3bRu3ZqWLVuSP3/+a9r4\nQunsr776ioEDBzJixAjuvfdecubMec12K539N0sKHpDDLwd7Y/bSq3ovnq/7PPkC8nk7JJNJeat0\n9vUqVapEyZIl2bZtGw0bNrxmmy+Uzq5evTrz5s0DHENJV4ftrrLS2Ym4UyApM/1k1oJ4p2NP67Dl\nw/TwucOqqhoXn/XK7WYXmaEgnrdLZ1+//ujRo1q8eHE9cuTIDe18oXT21fclPj5e+/Tpo19++aVr\nm5XOtoJ46W7+vvl0md6FqTunsvaoo1RvDj/rhJnUeat09lWtWrUiLCyMVq1aMWrUKEqWLHlDG18o\nnT158mSqVq1K9erVKV26NI888ohrH1Y6+zruZI7M9JOZegrHzh/TF35/QWtNqKU9f+mp26O33/I+\njedlhp5CVmKls7OeW+kpZLMUmL4+WP8Bi6MW80LdF+gX3M96B8YnJS6dfavf7LMSK51t3HLw3EFU\nlcACgbxQ9wX6h/S3ekXG51np7Owj2yWFoMGOuw4K5UlbWeqrBew+XP8h9UvW55M2n1A8b3GKU9wT\nYRpjjFdkr6QQHkPkTbxsT8wewpeHs+HYBpqVacYbjd9I78iMMSZTyF5J4SYsjlrMoIWDyBOQh/9r\n/n90rtjZ6hUZY3yWJYVkxCXEEeAXQO3itbmn0j0MqDPA6hUZY3yePadwndgrsYxeN5q+s/tyJeEK\nhXIVIrxpuCUEk66CgoKIjIykZcuW16x/4YUXKFOmDAkJCa51EyZMYMCAARkc4d/Cw8MpU6bMNSWm\nT58+7bV4fvnlF0aNGnVTr736ni9atIiHH344yTarV6+mRYsWVKtWjTp16tC/f38uXLiQ7v8OHTt2\ndL2PY8aMoUaNGvTu3fuWzi89WE8hkXVH1xG+PJzIM5F0q9KNy/GX7TZTk2ESEhKYNm0aZcuW5Y8/\n/qBVq1beDsll0KBBvPTSS94OA3CUBrn33ns9su+jR4/Ss2dPpkyZQpMmTQD46aefbrnEd1Jmz57t\n+v2TTz5hwYIFBAYGAqTp/K5cuZKuD9hZTwHHXAcjV47k4TkPE5cQx/h24xnWdBh5A/J6OzSTAR6Z\n88gNPxO2TLjp7e4oXrw4/v7+3Hbbba51ixYtIjg4mKeffjrJks3x8fFUqFABVeX06dP4+/uzePFi\nAFq0aMHOnTtZvXo1TZo0oU6dOjRt2pQ///zTtf3q5DEAzZs3Z+PGjfzxxx+ub/916tRJ04ff6NGj\nefTRRwHYvHkztWrV4sKFC4SHh9OnTx+aNGlClSpVGD9+POAoud26dWvq1q17zaQ+kZGR1KhRg8cf\nf5zg4GDatWvnmmdizJgx1KxZk9DQUFdZ68Tf2CMjI7nrrrsIDQ2ldevW7N+/H3BM7jNw4ECaNm1K\nxYoVXRP9XH3Pc+bM6XrSOrGPP/6Yfv36uRICQI8ePW540nvmzJk0atSIOnXq0KZNG44ePQqQ5Pt5\n+PBhWrRoQVhYGLVq1XJVwr06+dFTTz3Fnj176NChA6NHj77m/I4fP0737t1p0KABDRo0YNmyZQCu\n97hZs2b06dPH7X8zd1hScFp2cBkP1XiIqfdOpfHtjVN/gTG3YM2aNZQtW9ZV8RT+nlPhvvvuY9as\nWcTFxV3zGn9/f6pVq8a2bdtYunQpdevWZcmSJVy6dIkDBw5QpUoVqlevzpIlS9iwYQPDhw/ntdde\nA+Cxxx5jwoQJgKMgXGxsLLVr1+a9997j448/JiIigiVLliRb/G306NGuD7urPZjnn3+eXbt2MW3a\nNB555BE+++wz8uZ1fJHatGkTv//+OytWrGD48OEcOnSI3LlzM23aNNavX8/ChQv55z//6ZoPYufO\nnTz77LNs3bqVwoULuyrEjho1ig0bNrBp0ybGjRt3Q1zPPfcc/fr1Y9OmTfTu3fuakiGHDx9m6dKl\n/Prrr65Celff86ZNm/Lhhx/esL8tW7akWgYcHEl15cqVbNiwgV69evHOO+8AJPl+fvfdd9x9991E\nRESwceNG1yQ+V40bN47SpUuzcOFCBg0adM22559/nkGDBrFmzRp+/vln+vfv79q2bds2FixYkO5z\nPmTbsZHTsaf5autXPBv2LHkD8jK1y1Ty5Mhm1RANAP9t/1+PbnfH5cuXmT17Nu+//z4FChSgUaNG\nzJ0794bpLe+44w4WL17M3r17efXVVxk/fjx33nmnq2R2TEwM/fr1Y+fOnYiIK7H07NmTESNG8O67\n7/LVV1+5xtObNWvGiy++SO/evenWrZtr+OJ6SQ0f+fn5MWHCBEJDQ3nyySdp1uzv8vBdunQhT548\n5MmTh1atWrF69Wo6derEa6+9xuLFi/Hz8+PgwYOub9iJZzyrV68ekZGRgKO8eO/evenatStdu3a9\nIa4VK1a4EmufPn14+eWXXdu6du2Kn58fNWvWdB0nvURFRfHAAw9w+PBhLl++TIUKjgdYk3o/GzRo\nwKOPPkpcXBxdu3a9ISmkZMGCBWzbts21fObMGc6dOwc4hpg8UcHVoz0FEWkvIn+KyC4RGZzEdhGR\nMc7tm0SkrifjAUetp7mRc+kyowvfbP2GdUfXAVhCMF41d+5cTp8+TUhICEFBQSxdujTJb4AtWrRg\nyZIlrF692nWhctGiRdxxxx2AY8rLVq1asWXLFmbOnElsbCwAefPmpW3btsyYMYMffviB3r17AzB4\n8GC++OILLl68SLNmzdixYwevv/66q1eQmp07d5I/f34OHTp0zfrrb9sWESZNmsTx48dZt24dERER\nlCxZ0hXf1dnYwNEjunLlCgCzZs3i2WefZf369TRo0MC13h2J93m1R5Ka4OBg1q1bl2q75557jgED\nBrB582Y+++wz13kk9X62aNGCxYsXU6ZMGR5++GEmTpzo9jkkJCSwcuVK15ShBw8edM13kS+fZ0ry\neywpiIg/8DHQAagJPCgiNa9r1gGo4vx5AvjUU/EAHLtwjBcWvsBLf7xEqXylmNJ5Ck1LN/XkIY1x\ny+TJk/niiy+IjIwkMjKSvXv3Mn/+fC5cuHBNu4YNG7J8+XL8/PzInTs3YWFhfPbZZ66pJmNiYlzz\nCVwdLrqqf//+DBw4kAYNGrgm59m9ezchISG88sorNGjQgB07dvDmm2+6PoRSEhMTw8CBA1m8eDHR\n0dGucXtwzM4WGxtLdHQ0ixYtokGDBsTExFCiRAkCAgJYuHAh+/btS3H/CQkJHDhwgFatWvH2228T\nExPj+pZ8VdOmTV1Tgk6aNMmVHG/WgAED+Prrr1m1apVr3dSpU2/oaSR+n69WsIWk3899+/ZRsmRJ\nHn/8cfr378/69evdjqddu3aMHTvWtZzav0l68GRPoSGwS1X3qOplYArQ5bo2XYCJziJ+K4HCInK7\npwJ66Y+XWHZoGS/We5FJHSdR7bZqnjqUMW67cOECc+bMoVOnTq51+fLlo3nz5sycOfOatrly5aJs\n2bKuKSjvuOMOzp49S0hICAAvv/wyr776KnXq1LnhW3W9evUoWLDgNWWjP/jgA2rVqkVoaCgBAQF0\n6NAhyRgTX1MICwsjMjKSQYMG8eyzz1K1alW+/PJLBg8ezLFjxwDHsE+rVq1o3Lgx//73vyldujS9\ne/dm7dq1hISEMHHiRKpXr57i+xIfH89DDz1ESEgIderUYeDAgRQuXPiaNmPHjuW///0voaGhfPPN\nN0leJ0iLkiVLMmXKFF566SWqVatGjRo1mDt37jUzz4HjQm/Pnj2pV68exYr9fbt6Uu/nokWLqF27\nNnXq1OH77793zVrnjjFjxrB27VpCQ0OpWbNmktdV0pu4261K845FegDtVbW/c7kP0EhVByRq8ysw\nSlWXOpd/A15R1bXX7esJHD0JypUrVy+1bxjJ+fPkn+Tyz0VQoaCber3xDdu3b6dGjRreDiPDHTp0\niJYtW7Jjxw78/Dz3fTA8PJz8+fNnmltYs6Ok/sZFZJ2q1k/ttVni7iNV/VxV66tq/eLFb74AXbXb\nqllCMNnSxIkTadSoEW+++aZHE4LJ+jx599FBoGyi5UDnurS2Mcbcor59+9K3b98MOVZ4eHiGHMd4\nhie/MqwBqohIBRHJCfQCfrmuzS9AX+ddSI2BGFVNfQ5BY26Rp4ZNjfG2W/3b9lhPQVWviMgAYC7g\nD3ylqltF5Cnn9nHAbKAjsAu4ALj3OKgxtyB37txER0dTtGhRq3hrfIqqEh0dTe7cuW96Hx670Owp\n9evX16uTcRtzM+Li4oiKinLdW26ML8mdOzeBgYEEBFw7kZi7F5qz7RPNJvsKCAhwPYFqjLmW3YZg\njDHGxZKCMcYYF0sKxhhjXLLchWYROQ7c3CPNUAw4kY7hZAV2ztmDnXP2cCvnXF5VU336N8slhVsh\nImvdufruS+ycswc75+whI87Zho+MMca4WFIwxhjjkt2SwufeDsAL7JyzBzvn7MHj55ytrikYY4xJ\nWXbrKRhjjEmBJQVjjDEuPpkURKS9iPwpIrtEZHAS20VExji3bxKRut6IMz25cc69nee6WUSWi0ht\nb8SZnlI750TtGojIFedsgFmaO+csIi1FJEJEtorIHxkdY3pz42+7kIjMFJGNznPO0tWWReQrETkm\nIluS2e7Zzy9V9akfHGW6dwMVgZzARqDmdW06Av8DBGgMrPJ23Blwzk2BIs7fO2SHc07U7nccZdp7\neDvuDPh3LgxsA8o5l0t4O+4MOOfXgLedvxcHTgI5vR37LZxzC6AusCWZ7R79/PLFnkJDYJeq7lHV\ny8AUoMt1bboAE9VhJVBYRG7P6EDTUarnrKrLVfWUc3EljlnusjJ3/p0BngN+Bo5lZHAe4s45/wOY\nqqr7AVQ1q5+3O+esQAFxTI6RH0dSuJKxYaYfVV2M4xyS49HPL19MCmWAA4mWo5zr0tomK0nr+TyG\n45tGVpbqOYtIGeA+4NMMjMuT3Pl3rgoUEZFFIrJORDJmDk7PceecPwJqAIeAzcDzqpqQMeF5hUc/\nv2w+hWxGRFrhSArNvR1LBvgAeEVVE7LRDGs5gHpAayAPsEJEVqrqX94Ny6PuBiKAu4BKwHwRWaKq\nZ7wbVtbki0nhIFA20XKgc11a22Qlbp2PiIQCXwAdVDU6g2LzFHfOuT4wxZkQigEdReSKqk7PmBDT\nnTvnHAVEq+p54LyILAZqA1k1Kbhzzo8Ao9Qx4L5LRPYC1YHVGRNihvPo55cvDh+tAaqISAURyQn0\nAn65rs0vQF/nVfzGQIyqHs7oQNNRqucsIuWAqUAfH/nWmOo5q2oFVQ1S1SDgJ+CZLJwQwL2/7RlA\ncxHJISJ5gUbA9gyOMz25c877cfSMEJGSQDVgT4ZGmbE8+vnlcz0FVb0iIgOAuTjuXPhKVbeKyFPO\n7eNw3InSEdgFXMDxTSPLcvOchwBFgU+c35yvaBauMOnmOfsUd85ZVbeLyBxgE5AAfKGqSd7amBW4\n+e88ApggIptx3JHziqpm2ZLaIjIZaAkUE5EoYCgQABnz+WVlLowxxrj44vCRMcaYm2RJwRhjjIsl\nBWOMMS6WFIwxxrhYUjDGGONiScFkOiIS76zyefUnKIW2QclVk0zjMRc5K3FuFJFlIlLtJvbx1NWy\nEiLysIiUTrTtCxGpmc5xrhGRMDde84LzmQVjUmVJwWRGF1U1LNFPZAYdt7eq1ga+Bt5N64udzwlM\ndC4+DJROtK2/qm5Llyj/jvMT3IvzBcCSgnGLJQWTJTh7BEtEZL3zp2kSbYJFZLWzd7FJRKo41z+U\naP1nIuKfyuEWA5Wdr20tIhvEMQ/FVyKSy7l+lIhscx7nPee6cBF5SRzzNtQHJjmPmcf5Db++szfh\n+iB39ig+usk4V5CoEJqIfCoia8Uxp8Aw57qBOJLTQhFZ6FzXTkRWON/HH0UkfyrHMdmIJQWTGeVJ\nNHQ0zbnuGNBWVesCDwBjknjdU8CHqhqG40M5SkRqONs3c66PB3qncvx7gM0ikhuYADygqiE4KgA8\nLSJFcVRfDVbVUGBk4her6k/AWhzf6MNU9WKizT87X3vVAzjqM91MnO2BxGU7Xnc+pR4K3Ckioao6\nBkf10Faq2kpEigFvAG2c7+Va4MVUjmOyEZ8rc2F8wkXnB2NiAcBHzjH0eBwloq+3AnhdRAJxzCmw\nU0Ra46gausZZ3iMPyc+tMElELgKROOZhqAbsTVQr6mvgWRylmmOBL0XkV+BXd09MVY+LyB5nzZqd\nOAq3LXPuNy1x5sQxd0Di9+l+EXkCx//XtwM1cZS7SKyxc/0y53Fy4njfjAEsKZisYxBwFEfFTz8c\nH8rXUNXvRGQV0AmYLSJP4qiF87WqvurGMXqr6tqrCyJyW1KNnPV4GuIowtYDGICjbLO7pgD3AzuA\naaqq4viEdjtOYB2O6wljgW4iUgF4CWigqqdEZAKQO4nXCjBfVR9MQ7wmG7HhI5NVFAIOOydP6YOj\nONo1RKQisMc5ZDIDxzDKb0APESnhbHObiJR385h/AkEiUtm53Af4wzkGX0hVZ+NIVknNd30WKJDM\nfqfhmD3rQRwJgrTG6SwT/W+gsYhUBwoC54EYcVQK7ZBMLCuBZlfPSUTyiUhSvS6TTVlSMFnFJ0A/\nEdmIY8jlfBJt7ge2iEgEUAvHlIXbcIyhzxORTcB8HEMrqVLVWBwVKH90VuBMAMbh+ID91bm/pSQ9\nJj8BGHf1QvN1+z2Fo5x1eVVd7VyX5jid1yr+A/xLVTcCG3D0Pr7DMSR11efAHBFZqKrHcdwZNdl5\nnBU43k9jAKuSaowxJhHrKRhjjHGxpGCMMcbFkoIxxhgXSwrGGGNcLCkYY4xxsaRgjDHGxZKCMcYY\nl/8HjWyJUrMAoR8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1070e5dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "recession_probs_A = all_predictions_topics[0]['p1'].as_data_frame()\n",
    "recession_probs_B = all_predictions_topics_CFNAI[0]['p1'].as_data_frame()\n",
    "\n",
    "predictions = {'Model A': recession_probs_A,\n",
    "              'Model B': recession_probs_B}\n",
    "\n",
    "# plt.figure(figsize=(15,8))\n",
    "for model, preds in predictions.items():\n",
    "    fpr, tpr, thresholds = roc_curve(test['D_NBER'], preds) \n",
    "    roc_auc = auc(fpr, tpr)\n",
    "        \n",
    "    plt.plot(fpr, tpr, lw=1, label = model + ' (AUC = %.2f)' %(roc_auc))\n",
    "    \n",
    "plt.plot([0, 1], [0, 1], linestyle='--', label='\"Always-Expansion\" Classifier')\n",
    "    \n",
    "plt.xlim([-0.05, 1.05])\n",
    "plt.ylim([-0.05, 1.05])\n",
    "    \n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "    \n",
    "plt.title('ROC Curve')\n",
    "plt.legend(frameon=True, loc='lower right')\n",
    "plt.savefig('roc_curve.png', format='png', dpi=600, transparent=False,\n",
    "           bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_predictions_topics[0].as_data_frame().to_csv('discr_nowcast_preds.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}