model2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5a72ff89-8ff8-4abb-a5c8-0604c1801550",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn import preprocessing\n",
    "import warnings\n",
    "from itertools import tee\n",
    "\n",
    "# Suppress scientific notation\n",
    "pd.options.display.float_format = '{:20,.2f}'.format\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65cc423b-d6c4-4442-9715-b66b5d73fbd4",
   "metadata": {},
   "source": [
    "## 1. Data Pre-Processing\n",
    "1. Join session level data with track context/features\n",
    "2. Identify all unique transitions from song a to song b --> store in dictionary for future sampling during training\n",
    "3. Convert rewards to binary 0 = skipped, 1 = not skipped\n",
    "4. Encode categorical track and session features\n",
    "5. Create Features dataframe which containes cleaned/OHE vector of track features\n",
    "6. Normalize the session level response to song features on a 0-1 scale\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "09115307-f7cd-4459-85ea-e5f35c40c3ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sessions in DF: 178342\n",
      "Number of tracks in DF: 319008\n"
     ]
    }
   ],
   "source": [
    "# Load in Data\n",
    "df_sessions_sampled = pd.read_csv('data/training_set/log_sampled.csv')\n",
    "df_tracks_sampled = pd.read_csv('data/track_features/tf_sampled.csv')\n",
    "print(\"Number of sessions in DF: \" + str(len(df_sessions_sampled['session_id'].unique())))\n",
    "print(\"Number of tracks in DF: \" + str(len(df_tracks_sampled['track_id'].unique())))\n",
    "# Sanity check num tracks in tracks df equals num tracks in sessions df\n",
    "assert len(df_sessions_sampled['track_id_clean'].unique()) == len(df_tracks_sampled['track_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea0b02b0-ba7b-48d8-a49c-92c666e67bc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Join dataframes together on 'track_id'\n",
    "spotify = df_sessions_sampled.merge(df_tracks_sampled, left_on=\"track_id_clean\", right_on=\"track_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b45988e3-9e6e-4569-bd18-329995be9f1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# response = -1, track was skipped \n",
    "# response = 1, track was not skipped\n",
    "spotify['response'] = np.where(\n",
    "    spotify['not_skipped']==True,\n",
    "    1, -1)\n",
    "spotify = spotify.drop(columns=['skip_1', 'skip_2', 'skip_3', 'not_skipped'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cf7f642b-7948-4c78-8970-b6c7ef145786",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1    1985901\n",
       " 1    1004708\n",
       "Name: response, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotify['response'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93f869bc-b999-4741-937b-57d1f552c607",
   "metadata": {},
   "source": [
    "## Encode relevant user context data - behavior\n",
    "To booleans:\n",
    "- shuffle (0/1)\n",
    "- premium (0/1)\n",
    "\n",
    "To numeric:\n",
    "- date --> year, month\n",
    "\n",
    "To OHE vectors:\n",
    "- context (catalog, charts, editorial playlist, personalized playlist, radio, user collection)\n",
    "- track end reason (appload, backbtn, clickrow, endplay, fwdbtn, logout, remote, trackdone)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bf1027be-28e1-43b9-af15-efab05829a64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert binary columns to booleans\n",
    "data_bool_cols = spotify.columns[spotify.dtypes=='bool']\n",
    "for col in data_bool_cols:\n",
    "    spotify[col]=spotify[col].astype(int)\n",
    "spotify['mode'] = np.where(spotify['mode']=='major', 1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "40fb94c3-d0b5-4f5e-a576-429ac1b5bdea",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "spotify['date'] = pd.to_datetime(spotify.date, format='%Y-%m-%d')\n",
    "spotify['year']=spotify['date'].dt.year\n",
    "spotify['month']=spotify['date'].dt.month\n",
    "# spotify = spotify.drop(columns='date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "13229984-bef0-42ec-8796-96573a183e73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>session_position</th>\n",
       "      <th>session_length</th>\n",
       "      <th>track_id_clean</th>\n",
       "      <th>context_switch</th>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <th>...</th>\n",
       "      <th>appload_end</th>\n",
       "      <th>backbtn_end</th>\n",
       "      <th>clickrow_end</th>\n",
       "      <th>endplay_end</th>\n",
       "      <th>fwdbtn_end</th>\n",
       "      <th>logout_end</th>\n",
       "      <th>popup_end</th>\n",
       "      <th>remote_end</th>\n",
       "      <th>trackdone_end</th>\n",
       "      <th>uriopen_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_00079a23-1600-486a-91bd-5208be0c745a</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_012b0fb4-0cc3-429f-9a78-cc6e622153fb</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_013cc010-c476-4ad2-8972-73449e0b2ef4</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_01a5f0dc-9938-48c9-92f1-c7e51f34d290</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id  session_position  session_length  \\\n",
       "0  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 1              20   \n",
       "1  0_00079a23-1600-486a-91bd-5208be0c745a                 7              12   \n",
       "2  0_012b0fb4-0cc3-429f-9a78-cc6e622153fb                 6              20   \n",
       "3  0_013cc010-c476-4ad2-8972-73449e0b2ef4                 9              13   \n",
       "4  0_01a5f0dc-9938-48c9-92f1-c7e51f34d290                 7              12   \n",
       "\n",
       "                           track_id_clean  context_switch  \\\n",
       "0  t_0479f24c-27d2-46d6-a00c-7ec928f2b539               0   \n",
       "1  t_0479f24c-27d2-46d6-a00c-7ec928f2b539               0   \n",
       "2  t_0479f24c-27d2-46d6-a00c-7ec928f2b539               0   \n",
       "3  t_0479f24c-27d2-46d6-a00c-7ec928f2b539               0   \n",
       "4  t_0479f24c-27d2-46d6-a00c-7ec928f2b539               0   \n",
       "\n",
       "   no_pause_before_play  short_pause_before_play  long_pause_before_play  \\\n",
       "0                     0                        0                       0   \n",
       "1                     0                        1                       1   \n",
       "2                     0                        1                       1   \n",
       "3                     1                        0                       0   \n",
       "4                     0                        1                       1   \n",
       "\n",
       "   hist_user_behavior_n_seekfwd  hist_user_behavior_n_seekback  ...  \\\n",
       "0                             0                              0  ...   \n",
       "1                             0                              0  ...   \n",
       "2                             0                              0  ...   \n",
       "3                             0                              0  ...   \n",
       "4                             0                              0  ...   \n",
       "\n",
       "           appload_end          backbtn_end         clickrow_end  \\\n",
       "0                 0.00                 0.00                 0.00   \n",
       "1                 0.00                 0.00                 0.00   \n",
       "2                 0.00                 0.00                 0.00   \n",
       "3                 0.00                 0.00                 0.00   \n",
       "4                 0.00                 0.00                 0.00   \n",
       "\n",
       "           endplay_end           fwdbtn_end           logout_end  \\\n",
       "0                 0.00                 0.00                 0.00   \n",
       "1                 1.00                 0.00                 0.00   \n",
       "2                 0.00                 0.00                 0.00   \n",
       "3                 0.00                 0.00                 0.00   \n",
       "4                 0.00                 1.00                 0.00   \n",
       "\n",
       "             popup_end           remote_end        trackdone_end  \\\n",
       "0                 0.00                 0.00                 1.00   \n",
       "1                 0.00                 0.00                 0.00   \n",
       "2                 0.00                 0.00                 1.00   \n",
       "3                 0.00                 0.00                 1.00   \n",
       "4                 0.00                 0.00                 0.00   \n",
       "\n",
       "           uriopen_end  \n",
       "0                 0.00  \n",
       "1                 0.00  \n",
       "2                 0.00  \n",
       "3                 0.00  \n",
       "4                 0.00  \n",
       "\n",
       "[5 rows x 74 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# OHE Categorical Columns\n",
    "spotify['hist_user_behavior_reason_end'] = spotify['hist_user_behavior_reason_end']+'_end'\n",
    "spotify['hist_user_behavior_reason_start'] = spotify['hist_user_behavior_reason_start']+'_start'\n",
    "\n",
    "categorical_columns =['context_type', 'hist_user_behavior_reason_start', 'hist_user_behavior_reason_end']\n",
    "enc = OneHotEncoder()\n",
    "array_hot_encoded = enc.fit_transform(spotify[categorical_columns]).toarray()\n",
    "feature_labels = enc.categories_\n",
    "feature_labels = np.concatenate(feature_labels, axis=0)\n",
    "data_hot_encoded = pd.DataFrame(array_hot_encoded, columns=feature_labels, index=spotify.index) # Convert OHE array to df\n",
    "spotify = spotify.drop(columns=categorical_columns)\n",
    "spotify_enc = pd.concat([spotify, data_hot_encoded], axis=1)\n",
    "spotify_enc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2a354536-6ff0-468a-94af-956dc17957ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spotify_enc.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "644a2e12-26dc-47cb-8830-a4175b042386",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['session_id', 'track_id_clean', 'track_id'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_bool_objects = spotify_enc.columns[spotify_enc.dtypes=='object']\n",
    "data_bool_objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "25e7ebfa-c907-4054-94eb-6101a3910a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "ori_columns = ['session_position', 'session_length', 'context_switch', 'no_pause_before_play', 'short_pause_before_play',\n",
    "       'long_pause_before_play', 'hist_user_behavior_n_seekfwd',\n",
    "       'hist_user_behavior_n_seekback', 'hist_user_behavior_is_shuffle',\n",
    "       'hour_of_day', 'premium','duration',\n",
    "       'release_year', 'us_popularity_estimate', 'acousticness',\n",
    "       'beat_strength', 'bounciness', 'danceability', 'dyn_range_mean',\n",
    "       'energy', 'flatness', 'instrumentalness', 'key', 'liveness', 'loudness',\n",
    "       'mechanism', 'mode', 'organism', 'speechiness', 'tempo',\n",
    "       'time_signature', 'valence', 'acoustic_vector_0', 'acoustic_vector_1',\n",
    "       'acoustic_vector_2', 'acoustic_vector_3', 'acoustic_vector_4',\n",
    "       'acoustic_vector_5', 'acoustic_vector_6', 'acoustic_vector_7',\n",
    "       'response', 'year', 'month', 'catalog', 'charts', 'editorial_playlist',\n",
    "       'personalized_playlist', 'radio', 'user_collection', 'appload_start',\n",
    "       'backbtn_start', 'clickrow_start', 'endplay_start', 'fwdbtn_start',\n",
    "       'playbtn_start', 'popup_start', 'remote_start', 'trackdone_start',\n",
    "       'trackerror_start', 'uriopen_start', 'appload_end', 'backbtn_end',\n",
    "       'clickrow_end', 'endplay_end', 'fwdbtn_end', 'logout_end', 'popup_end',\n",
    "       'remote_end', 'trackdone_end', 'uriopen_end']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "94d3e9d5-a241-4630-8e6c-b9ea240de5a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ori_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a2960c1e-0d4a-47e5-9f94-4eabdcc3ca79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize all features to be between 0 and 1\n",
    "for i in ori_columns:\n",
    "    # print(i)\n",
    "    #spotify_enc[i] /= np.max(np.abs(spotify_enc[i]),axis=0) # Between -1 and 1\n",
    "    spotify_enc[i] = (spotify_enc[i]-np.min(spotify_enc[i]))/(np.max(spotify_enc[i])-np.min(spotify_enc[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4eea2d5f-605c-4f4a-ae30-4f15b249812c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_position</th>\n",
       "      <th>session_length</th>\n",
       "      <th>context_switch</th>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <th>hist_user_behavior_is_shuffle</th>\n",
       "      <th>hour_of_day</th>\n",
       "      <th>...</th>\n",
       "      <th>appload_end</th>\n",
       "      <th>backbtn_end</th>\n",
       "      <th>clickrow_end</th>\n",
       "      <th>endplay_end</th>\n",
       "      <th>fwdbtn_end</th>\n",
       "      <th>logout_end</th>\n",
       "      <th>popup_end</th>\n",
       "      <th>remote_end</th>\n",
       "      <th>trackdone_end</th>\n",
       "      <th>uriopen_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>...</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "      <td>2,990,609.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.61</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.29</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.26</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.21</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.48</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.42</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.65</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.68</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          session_position       session_length       context_switch  \\\n",
       "count         2,990,609.00         2,990,609.00         2,990,609.00   \n",
       "mean                  0.44                 0.76                 0.04   \n",
       "std                   0.29                 0.34                 0.20   \n",
       "min                   0.00                 0.00                 0.00   \n",
       "25%                   0.21                 0.50                 0.00   \n",
       "50%                   0.42                 1.00                 0.00   \n",
       "75%                   0.68                 1.00                 0.00   \n",
       "max                   1.00                 1.00                 1.00   \n",
       "\n",
       "       no_pause_before_play  short_pause_before_play  long_pause_before_play  \\\n",
       "count          2,990,609.00             2,990,609.00            2,990,609.00   \n",
       "mean                   0.77                     0.15                    0.17   \n",
       "std                    0.42                     0.35                    0.38   \n",
       "min                    0.00                     0.00                    0.00   \n",
       "25%                    1.00                     0.00                    0.00   \n",
       "50%                    1.00                     0.00                    0.00   \n",
       "75%                    1.00                     0.00                    0.00   \n",
       "max                    1.00                     1.00                    1.00   \n",
       "\n",
       "       hist_user_behavior_n_seekfwd  hist_user_behavior_n_seekback  \\\n",
       "count                  2,990,609.00                   2,990,609.00   \n",
       "mean                           0.00                           0.00   \n",
       "std                            0.00                           0.00   \n",
       "min                            0.00                           0.00   \n",
       "25%                            0.00                           0.00   \n",
       "50%                            0.00                           0.00   \n",
       "75%                            0.00                           0.00   \n",
       "max                            1.00                           1.00   \n",
       "\n",
       "       hist_user_behavior_is_shuffle          hour_of_day  ...  \\\n",
       "count                   2,990,609.00         2,990,609.00  ...   \n",
       "mean                            0.30                 0.61  ...   \n",
       "std                             0.46                 0.26  ...   \n",
       "min                             0.00                 0.00  ...   \n",
       "25%                             0.00                 0.48  ...   \n",
       "50%                             0.00                 0.65  ...   \n",
       "75%                             1.00                 0.83  ...   \n",
       "max                             1.00                 1.00  ...   \n",
       "\n",
       "               appload_end          backbtn_end         clickrow_end  \\\n",
       "count         2,990,609.00         2,990,609.00         2,990,609.00   \n",
       "mean                  0.00                 0.08                 0.00   \n",
       "std                   0.00                 0.27                 0.00   \n",
       "min                   0.00                 0.00                 0.00   \n",
       "25%                   0.00                 0.00                 0.00   \n",
       "50%                   0.00                 0.00                 0.00   \n",
       "75%                   0.00                 0.00                 0.00   \n",
       "max                   1.00                 1.00                 1.00   \n",
       "\n",
       "               endplay_end           fwdbtn_end           logout_end  \\\n",
       "count         2,990,609.00         2,990,609.00         2,990,609.00   \n",
       "mean                  0.09                 0.48                 0.00   \n",
       "std                   0.28                 0.50                 0.06   \n",
       "min                   0.00                 0.00                 0.00   \n",
       "25%                   0.00                 0.00                 0.00   \n",
       "50%                   0.00                 0.00                 0.00   \n",
       "75%                   0.00                 1.00                 0.00   \n",
       "max                   1.00                 1.00                 1.00   \n",
       "\n",
       "                 popup_end           remote_end        trackdone_end  \\\n",
       "count         2,990,609.00         2,990,609.00         2,990,609.00   \n",
       "mean                  0.00                 0.00                 0.35   \n",
       "std                   0.00                 0.04                 0.48   \n",
       "min                   0.00                 0.00                 0.00   \n",
       "25%                   0.00                 0.00                 0.00   \n",
       "50%                   0.00                 0.00                 0.00   \n",
       "75%                   0.00                 0.00                 1.00   \n",
       "max                   1.00                 1.00                 1.00   \n",
       "\n",
       "               uriopen_end  \n",
       "count         2,990,609.00  \n",
       "mean                  0.00  \n",
       "std                   0.00  \n",
       "min                   0.00  \n",
       "25%                   0.00  \n",
       "50%                   0.00  \n",
       "75%                   0.00  \n",
       "max                   1.00  \n",
       "\n",
       "[8 rows x 70 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotify_enc.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "40d2071c-b836-42d8-920f-f1f5bdebabc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178342"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(spotify_enc['session_id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e00cf7d2-4d22-42ef-aa25-4d957649bf62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "89672"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(spotify_enc['session_id'].value_counts()==20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a4f0cc95-ddfa-4937-b733-c8375a0d5626",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88670"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(spotify_enc['session_id'].value_counts()<20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "17d076c8-a821-44ef-8ea8-95dfcd41e4cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select sessions with 20 songs/tracks\n",
    "session_id_20 = np.array([i[0] for i in dict(spotify_enc['session_id'].value_counts()==20).items() if i[1]==True])\n",
    "spotify_enc_20 = spotify_enc[spotify_enc['session_id'].isin(session_id_20)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b82b1fc-19ef-4b51-add9-237193af4217",
   "metadata": {},
   "source": [
    "## 2. Feature Selection: TBD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c690259-d57e-46d6-83f1-5d8348e3e63a",
   "metadata": {},
   "source": [
    "### Feature selection is implemented in file: feature_selection.ipynb. \n",
    "### The file can be found in github: https://github.com/eliserust/Spotify_RL.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f7287b49-a7b4-437b-8ca9-316e04be9d1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on the result on feature selection, 12 features should be removed. \n",
    "removed_features = ['acousticness', 'beat_strength', 'bounciness', 'dyn_range_mean', 'mechanism', 'mode', 'organism',\n",
    "'acoustic_vector_0', 'acoustic_vector_1', 'acoustic_vector_3', 'acoustic_vector_5', 'acoustic_vector_6']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "aba7789d-5618-4277-82db-ed7200b88519",
   "metadata": {},
   "outputs": [],
   "source": [
    "track_columns = ['duration','release_year', 'us_popularity_estimate', 'danceability', \n",
    "       'energy', 'flatness', 'instrumentalness', 'key', 'liveness', 'loudness',\n",
    "      'speechiness', 'tempo','time_signature', 'valence', \n",
    "       'acoustic_vector_2',  'acoustic_vector_4', 'acoustic_vector_7','response']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "16258e99-3ae1-4c56-8bfa-cf7f95f3e46f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(track_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "429277da-0679-4a8d-9ad1-bb8c966048dd",
   "metadata": {},
   "source": [
    "## 3. Model Building"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25d94b67-5cda-483f-9f38-ca75d41d2ba2",
   "metadata": {},
   "source": [
    "#### User context: last 15 songs within the session (both session-related features and song-related features)\n",
    "#### Song inventory: the other 5 songs within the session\n",
    "#### Song context: all features of a song within song inventory\n",
    "#### Used model: RNN (Recurrent Neural Network)\n",
    "#### Input: user context + song-related features of a chosen song\n",
    "#### Output: 0 (skip) / 1(not skip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5ee45fa3-22d1-4045-b38c-47128f191179",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_regret(selected_rated, inventory_ratings):\n",
    "    return max(inventory_ratings) - selected_rated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "267ccc50-b30c-44c0-8901-d16d7801ceb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of input features:  927\n"
     ]
    }
   ],
   "source": [
    "x_dimension = 70 - 12\n",
    "num_context = 15\n",
    "input_size = (x_dimension*num_context)+(x_dimension-1)\n",
    "output_size = 1\n",
    "\n",
    "print('number of input features: ', input_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "0979926e-587e-4495-b181-701159a8c98c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import SimpleRNN\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, SimpleRNN\n",
    "\n",
    "from keras.layers import Dense\n",
    "model = Sequential() \n",
    "model.add(Embedding(10000, 32))\n",
    "model.add(SimpleRNN(32, dropout=0.5, return_sequences=True))\n",
    "model.add(SimpleRNN(32, return_sequences=True))\n",
    "model.add(SimpleRNN(32, return_sequences=True))\n",
    "model.add(SimpleRNN(32)) \n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) \n",
    "# history = model.fit(x_train,one_hot_train_labels, epochs=20, batch_size=128, validation_data = (x_test, one_hot_test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "57d2ac1b-8228-4a9a-b272-307dd97b0aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eps_greedy(model, session_context, track_inventory, eps):\n",
    "    selected_track = None\n",
    "    selected_x = None\n",
    "    max_action_val = -100\n",
    "    ids = list(range(5))\n",
    "    \n",
    "    # IF A RANDOM NUMBER BETWEEN 0 AND 1 IS LESS THAN OUR EPS HYPERPARAMETER WE CHOOSE A RANDOM AD\n",
    "    if np.random.uniform() < eps:\n",
    "        selected_track = np.random.choice(ids)\n",
    "        selected_x = np.concatenate((session_context,track_inventory[selected_track]), axis = None).reshape(input_size,)\n",
    "\n",
    "    # OTHERWISE WE GET THE AD(S) WITH THE HIGHEST PREDICTED CTR(S)\n",
    "    else:\n",
    "        context = np.array([np.concatenate((session_context, i)) for i in track_invetory])\n",
    "        predictions = model.predict(context, verbose = 0) # set verbose = 0, so keras prediction would not print sth. like: 1/1 [==============================] - 0s 29ms/step\n",
    "        selected_track = np.argmax(predictions)\n",
    "        selected_x = context[selected_track]\n",
    "        \n",
    "    return selected_track, selected_x.reshape(input_size,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5553d551-c32b-453e-874f-786437211eba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e =  0.05 \n",
      "\n",
      "Updating the model after 128 impressions\n",
      "4/4 [==============================] - 5s 590ms/step - loss: 0.6816 - accuracy: 0.5469\n",
      "Updating the model after 256 impressions\n",
      "4/4 [==============================] - 2s 619ms/step - loss: 0.6950 - accuracy: 0.5625\n",
      "Updating the model after 384 impressions\n",
      "4/4 [==============================] - 2s 630ms/step - loss: 0.6546 - accuracy: 0.6094\n",
      "Updating the model after 512 impressions\n",
      "4/4 [==============================] - 2s 563ms/step - loss: 0.6222 - accuracy: 0.6719\n",
      "Updating the model after 640 impressions\n",
      "4/4 [==============================] - 2s 578ms/step - loss: 0.5940 - accuracy: 0.6953\n",
      "Updating the model after 768 impressions\n",
      "4/4 [==============================] - 2s 613ms/step - loss: 0.5044 - accuracy: 0.7891\n",
      "Updating the model after 896 impressions\n",
      "4/4 [==============================] - 2s 589ms/step - loss: 0.4716 - accuracy: 0.7734\n",
      "Updating the model after 1024 impressions\n",
      "4/4 [==============================] - 4s 1s/step - loss: 0.3948 - accuracy: 0.8594\n",
      "Updating the model after 1152 impressions\n",
      "4/4 [==============================] - 2s 602ms/step - loss: 0.3289 - accuracy: 0.9141\n",
      "Updating the model after 1280 impressions\n",
      "4/4 [==============================] - 2s 610ms/step - loss: 0.2541 - accuracy: 0.9297\n",
      "Updating the model after 1408 impressions\n",
      "4/4 [==============================] - 2s 589ms/step - loss: 0.1771 - accuracy: 0.9609\n",
      "Updating the model after 1536 impressions\n",
      "4/4 [==============================] - 2s 570ms/step - loss: 0.1870 - accuracy: 0.9453\n",
      "Updating the model after 1664 impressions\n",
      "4/4 [==============================] - 2s 587ms/step - loss: 0.1397 - accuracy: 0.9688\n",
      "Updating the model after 1792 impressions\n",
      "4/4 [==============================] - 2s 581ms/step - loss: 0.1347 - accuracy: 0.9688\n",
      "Updating the model after 1920 impressions\n",
      "4/4 [==============================] - 2s 598ms/step - loss: 0.1253 - accuracy: 0.9609\n",
      "Updating the model after 2048 impressions\n",
      "4/4 [==============================] - 3s 602ms/step - loss: 0.1487 - accuracy: 0.9531\n",
      "Updating the model after 2176 impressions\n",
      "4/4 [==============================] - 3s 650ms/step - loss: 0.1384 - accuracy: 0.9453\n",
      "Updating the model after 2304 impressions\n",
      "4/4 [==============================] - 2s 597ms/step - loss: 0.1285 - accuracy: 0.9609\n",
      "Updating the model after 2432 impressions\n",
      "4/4 [==============================] - 2s 580ms/step - loss: 0.1431 - accuracy: 0.9453\n",
      "Updating the model after 2560 impressions\n",
      "4/4 [==============================] - 2s 583ms/step - loss: 0.1338 - accuracy: 0.9609\n",
      "Updating the model after 2688 impressions\n",
      "4/4 [==============================] - 2s 628ms/step - loss: 0.1057 - accuracy: 0.9688\n",
      "Updating the model after 2816 impressions\n",
      "4/4 [==============================] - 2s 630ms/step - loss: 0.1269 - accuracy: 0.9688\n",
      "Updating the model after 2944 impressions\n",
      "4/4 [==============================] - 2s 574ms/step - loss: 0.1447 - accuracy: 0.9531\n",
      "Updating the model after 3072 impressions\n",
      "4/4 [==============================] - 2s 606ms/step - loss: 0.1446 - accuracy: 0.9609\n",
      "Updating the model after 3200 impressions\n",
      "4/4 [==============================] - 3s 650ms/step - loss: 0.1173 - accuracy: 0.9609\n",
      "Updating the model after 3328 impressions\n",
      "4/4 [==============================] - 2s 599ms/step - loss: 0.1095 - accuracy: 0.9688\n",
      "Updating the model after 3456 impressions\n",
      "4/4 [==============================] - 3s 611ms/step - loss: 0.1287 - accuracy: 0.9609\n",
      "Updating the model after 3584 impressions\n",
      "4/4 [==============================] - 2s 575ms/step - loss: 0.0914 - accuracy: 0.9766\n",
      "Updating the model after 3712 impressions\n",
      "4/4 [==============================] - 2s 599ms/step - loss: 0.1036 - accuracy: 0.9766\n",
      "Updating the model after 3840 impressions\n",
      "4/4 [==============================] - 2s 605ms/step - loss: 0.0873 - accuracy: 0.9766\n",
      "Updating the model after 3968 impressions\n",
      "4/4 [==============================] - 2s 585ms/step - loss: 0.1044 - accuracy: 0.9766\n",
      "Updating the model after 4096 impressions\n",
      "4/4 [==============================] - 3s 696ms/step - loss: 0.0850 - accuracy: 0.9766\n",
      "Updating the model after 4224 impressions\n",
      "4/4 [==============================] - 3s 632ms/step - loss: 0.0830 - accuracy: 0.9766\n",
      "Updating the model after 4352 impressions\n",
      "4/4 [==============================] - 2s 593ms/step - loss: 0.0928 - accuracy: 0.9688\n",
      "Updating the model after 4480 impressions\n",
      "4/4 [==============================] - 2s 632ms/step - loss: 0.0840 - accuracy: 0.9766\n",
      "Updating the model after 4608 impressions\n",
      "4/4 [==============================] - 2s 582ms/step - loss: 0.0939 - accuracy: 0.9766\n",
      "Updating the model after 4736 impressions\n",
      "4/4 [==============================] - 3s 629ms/step - loss: 0.0975 - accuracy: 0.9688\n",
      "Updating the model after 4864 impressions\n",
      "4/4 [==============================] - 2s 585ms/step - loss: 0.0926 - accuracy: 0.9766\n",
      "Updating the model after 4992 impressions\n",
      "4/4 [==============================] - 2s 601ms/step - loss: 0.0706 - accuracy: 0.9766\n",
      "Updating the model after 5120 impressions\n",
      "4/4 [==============================] - 3s 648ms/step - loss: 0.0756 - accuracy: 0.9844\n",
      "Updating the model after 5248 impressions\n",
      "4/4 [==============================] - 2s 625ms/step - loss: 0.0880 - accuracy: 0.9688\n",
      "Updating the model after 5376 impressions\n",
      "4/4 [==============================] - 2s 587ms/step - loss: 0.0870 - accuracy: 0.9766\n",
      "Updating the model after 5504 impressions\n",
      "4/4 [==============================] - 2s 623ms/step - loss: 0.0657 - accuracy: 0.9844\n",
      "Updating the model after 5632 impressions\n",
      "4/4 [==============================] - 3s 723ms/step - loss: 0.0669 - accuracy: 0.9766\n",
      "Updating the model after 5760 impressions\n",
      "4/4 [==============================] - 2s 597ms/step - loss: 0.0761 - accuracy: 0.9766\n",
      "Updating the model after 5888 impressions\n",
      "4/4 [==============================] - 2s 615ms/step - loss: 0.0679 - accuracy: 0.9844\n",
      "Updating the model after 6016 impressions\n",
      "4/4 [==============================] - 3s 677ms/step - loss: 0.0731 - accuracy: 0.9688\n",
      "Updating the model after 6144 impressions\n",
      "4/4 [==============================] - 2s 625ms/step - loss: 0.0789 - accuracy: 0.9688\n",
      "Updating the model after 6272 impressions\n",
      "4/4 [==============================] - 3s 733ms/step - loss: 0.0687 - accuracy: 0.9844\n",
      "Updating the model after 6400 impressions\n",
      "4/4 [==============================] - 2s 613ms/step - loss: 0.0723 - accuracy: 0.9688\n",
      "Updating the model after 6528 impressions\n",
      "4/4 [==============================] - 3s 651ms/step - loss: 0.0832 - accuracy: 0.9688\n",
      "Updating the model after 6656 impressions\n",
      "4/4 [==============================] - 3s 806ms/step - loss: 0.0632 - accuracy: 0.9844\n",
      "Updating the model after 6784 impressions\n",
      "4/4 [==============================] - 3s 638ms/step - loss: 0.0481 - accuracy: 0.9922\n",
      "Updating the model after 6912 impressions\n",
      "4/4 [==============================] - 2s 594ms/step - loss: 0.0777 - accuracy: 0.9844\n",
      "Updating the model after 7040 impressions\n",
      "4/4 [==============================] - 3s 736ms/step - loss: 0.0744 - accuracy: 0.9766\n",
      "Updating the model after 7168 impressions\n",
      "4/4 [==============================] - 3s 673ms/step - loss: 0.0424 - accuracy: 0.9844\n",
      "Updating the model after 7296 impressions\n",
      "4/4 [==============================] - 3s 704ms/step - loss: 0.0744 - accuracy: 0.9688\n",
      "Updating the model after 7424 impressions\n",
      "4/4 [==============================] - 2s 601ms/step - loss: 0.0585 - accuracy: 0.9922\n",
      "Updating the model after 7552 impressions\n",
      "4/4 [==============================] - 2s 570ms/step - loss: 0.0575 - accuracy: 0.9844\n",
      "Updating the model after 7680 impressions\n",
      "4/4 [==============================] - 2s 579ms/step - loss: 0.0634 - accuracy: 0.9844\n",
      "Updating the model after 7808 impressions\n",
      "4/4 [==============================] - 2s 577ms/step - loss: 0.0630 - accuracy: 0.9688\n",
      "Updating the model after 7936 impressions\n",
      "4/4 [==============================] - 3s 624ms/step - loss: 0.0509 - accuracy: 0.9922\n",
      "Updating the model after 8064 impressions\n",
      "4/4 [==============================] - 2s 590ms/step - loss: 0.0474 - accuracy: 0.9844\n",
      "Updating the model after 8192 impressions\n",
      "4/4 [==============================] - 2s 580ms/step - loss: 0.0563 - accuracy: 0.9922\n",
      "Updating the model after 8320 impressions\n",
      "4/4 [==============================] - 2s 624ms/step - loss: 0.0467 - accuracy: 0.9844\n",
      "Updating the model after 8448 impressions\n",
      "4/4 [==============================] - 3s 673ms/step - loss: 0.0685 - accuracy: 0.9688\n",
      "Updating the model after 8576 impressions\n",
      "4/4 [==============================] - 2s 579ms/step - loss: 0.0446 - accuracy: 0.9844\n",
      "Updating the model after 8704 impressions\n",
      "4/4 [==============================] - 2s 592ms/step - loss: 0.0519 - accuracy: 0.9844\n",
      "Updating the model after 8832 impressions\n",
      "4/4 [==============================] - 3s 632ms/step - loss: 0.0401 - accuracy: 0.9922\n",
      "Updating the model after 8960 impressions\n",
      "4/4 [==============================] - 2s 611ms/step - loss: 0.0522 - accuracy: 0.9922\n",
      "Updating the model after 9088 impressions\n",
      "4/4 [==============================] - 3s 691ms/step - loss: 0.0693 - accuracy: 0.9844\n",
      "Updating the model after 9216 impressions\n",
      "4/4 [==============================] - 2s 600ms/step - loss: 0.0566 - accuracy: 0.9844\n",
      "Updating the model after 9344 impressions\n",
      "4/4 [==============================] - 2s 610ms/step - loss: 0.0381 - accuracy: 0.9844\n",
      "Updating the model after 9472 impressions\n",
      "4/4 [==============================] - 3s 703ms/step - loss: 0.0539 - accuracy: 0.9766\n",
      "Updating the model after 9600 impressions\n",
      "4/4 [==============================] - 3s 672ms/step - loss: 0.0571 - accuracy: 0.9844\n",
      "Updating the model after 9728 impressions\n",
      "4/4 [==============================] - 3s 664ms/step - loss: 0.0644 - accuracy: 0.9844\n",
      "Updating the model after 9856 impressions\n",
      "4/4 [==============================] - 2s 605ms/step - loss: 0.0608 - accuracy: 0.9844\n",
      "Updating the model after 9984 impressions\n",
      "4/4 [==============================] - 3s 762ms/step - loss: 0.0612 - accuracy: 0.9844\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "impressions = 10000\n",
    "batch_size = 128\n",
    "eps = [0.05] \n",
    "regret_vec = {}\n",
    "train_loss = []\n",
    "for e in eps:\n",
    "    print('e = ',e, '\\n')\n",
    "    movie_model = model\n",
    "    X = []\n",
    "    y = []\n",
    "    total_regret = 0\n",
    "    regret_vec[e] = []\n",
    "    \n",
    "    for i in range(impressions):\n",
    "        # sample a session\n",
    "        session_id_sample = np.array(spotify_enc_20.sample(n = 1))[0][0]\n",
    "        \n",
    "        # sample last 15 songs within the session as the session context (including session-related and track-related features)\n",
    "        session_context = np.array(spotify_enc_20[spotify_enc_20.session_id == session_id_sample][:15][ori_columns].drop(removed_features, axis = 1)).flatten() # 1050 features\n",
    "        \n",
    "        # remaining 5 songs within the session as the track inventories (after remove reponse column)\n",
    "        track_inventory_all = spotify_enc_20[spotify_enc_20.session_id == session_id_sample][15:][ori_columns].drop(removed_features, axis = 1)\n",
    "        track_invetory = np.array(track_inventory_all.loc[:,track_inventory_all.columns!='response'])\n",
    "        \n",
    "        # true response for these five songs\n",
    "        track_response = np.array(track_inventory_all['response'])\n",
    "        \n",
    "        # use eps-greedy strategy with RNN model to choose a track\n",
    "        selected_track_id, selected_x = eps_greedy(model, session_context, track_invetory, e)        \n",
    "        \n",
    "        # get select track response and calculate regret\n",
    "        selected_response = track_response[selected_track_id]\n",
    "        regret = calculate_regret(selected_response, track_response)\n",
    "        \n",
    "        # SAVE SELECTED CONTEXT, REWARD, REGRET AND TOTAL REGRET\n",
    "        if math.isnan(selected_response):\n",
    "            continue\n",
    "        else:\n",
    "            X.append(selected_x)\n",
    "            y.append(selected_response)\n",
    "\n",
    "        total_regret += regret\n",
    "        regret_vec[e].append(total_regret)\n",
    "        \n",
    "        idx = batch_size\n",
    "\n",
    "        # UPDATE THE MODEL AT THE END OF THE BATCH\n",
    "        if (i + 1)%batch_size == 0:\n",
    "            print('Updating the model after {} impressions'.format(i+1))\n",
    "            train_x = np.array(X[idx-batch_size:idx])\n",
    "            train_y = np.array(y[idx-batch_size:idx])\n",
    "\n",
    "            # print(train_x.shape)\n",
    "            # print(train_y.shape)        \n",
    "            idx+=batch_size\n",
    "            model.fit(train_x, train_y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a4323331-1c6a-4c39-bf56-1b10f576b320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAGDCAYAAAALTociAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABJq0lEQVR4nO3dd3yV5f3/8dcnISFA2BvClA0qasC9F1ortdZdV62otbW7jvqtq7b212qrdWsVR1Gpuxa34kBlqshQ2RAIe4QQsj+/P+6beAhJCCYnd3LO+/l4nAfnvu773Odzzn1C3rnOdV+3uTsiIiIiIvLtpERdgIiIiIhIU6ZALSIiIiJSBwrUIiIiIiJ1oEAtIiIiIlIHCtQiIiIiInWgQC0iIiIiUgcK1CJSb8ysr5m5mTWLupZ4M7PTzGyFmeWb2X5R1xMFM1tqZsdFXccOZnaUmeU04POdZ2ZvNNTzNWZm9qqZXRh1HSJRUaAWaQBh8Ngehq/VZjbezDJj1o8Pg+jomLYBZuYxy5PNrNDMesW0HWdmS2vx/L3D595xczPbFrN8eDWPu8jMPvzWL3zX/dX4PjSk8D0YUIdd/A34qbtnuvun9VVXMjCzG82spNJncnPUddWkqj8W3f3f7n5CnJ7vOjNbEr43OWb2TMy6yWb24zru/0Yze7LulQbc/SR3f6y+9ifS1ChQizSc77p7JjAS2A+4ttL6jcAfd7OPbcD/7ekTu/vyMPhlhjUA7BvT9sGe7rMOdvc+1FkD9ZD3AeY2wPPUGzNLjbqGGM/EfibdvV2UxTSm9ybs6T0fOC78WckG3t6Dxyf8N0QijY0CtUgDc/fVwOsEgTLWY8A+ZnZkDQ+/Czinjj2rOzGztmb2uJmtM7NlZna9maWY2VDgfuDg2B5EM/uOmX1qZnnhkIcbv83zVvU+mNlBZvaRmW02s8/N7KiYdf3M7H0z22pmb5nZPTt62GJ6Dy8xs+XAO2H7j8xsvpltMrPXzaxP2P5+uNvPw9d2VhXvS0r4Xiwzs7Xhe9TWzJqbWT6QGj5+UTXv6xAze9PMNprZV2Z2Zsy68WZ2f7h+q5m9F1Obmdnfw+fcYmazzWxEde+jmf3OzHLNbJWZ/Ti25z18nvvMbJKZbQOONrMeZvZceLyXmNlVlV7zNWa2yMw2mNlEM+sQs/788P3YYGa/j2nvZmYFZtYxpu2A8DnSqqu9mtdziJmtt/CbGDPbN/w8DAmXl5rZtWY2Lzyuj5pZRjX7GmpBb+5mM5trZqdWOgaV35uaPts7PjObw8/MwVbpG5yw9unhcZtuZofErJtsZreY2ZTwmL9hZp2qeRtGAa+7+yIIflbc/cFwP7cChwN3h3XcHba7mV1pZguABWHbneHryDOzmRZ+E2VmY4DrgLPCfXwetrc1s3+Fn6eVZvZHC//QMLNUM7s9PDZLzOynFtNjb5V6za36n709+nyLNBnurptuusX5Biwl6G0CyAK+AO6MWT+eoHf6KuDDsG1A8CNasc1k4MfAHcCTYdtxwNKYbe4F7q1FPQ4MCO8/DrwEtAb6Al8Dl4TrLtpRT8xjjwL2JviDfB9gDfC9cF3fcN/N9vR9AHoCG4CTw30fHy53Dtd/TDDMIh04DMiLeR92PO/jQCugBfA9YCEwFGgGXA98VNV7UE2tPwof3x/IBJ4HnqjN48MaVgAXh8+9P7AeGB5zvLcCRwDNgTtjjvuJwEygHWBh/d2reZ4xwGpgONASeKLSsR0PbAEODd/TluG+/xC+j/2BxcCJ4fa/AD4Jj01z4AHgqXDdMCA/puY7gNKY4zkJuCKmtr8D/6ym7ht3HLtq1t9K8EdRC2A2wdCa2M/QHKAX0AGYAvwx5rOZE95PC4/fdeFrPSZ8zwdX895ksIefbWJ+PsJaNhH0LDcDzgmXO8b8/C4CBoWvazJwWzWv/4cE31j9lqB3OrXS+snAj6v4mX4zrKNFzH46hvX8muCzklHdMQBeDI95K6ALMA24LFx3OTAv/Gy0B96KfT9ia6KGnz324POtm25N6RZ5Abrplgy3MATkh7/QneDr23Yx68cTBOrmwHLgJKoP1J3DIDCcSoF6D+rxcP+pQBEwLGbdZcDk8H5FYKhhX/8A/h7e78vuA3WV7wNwNTGBNWx7HbgQ6E0Q3lrGrHuSXQN1/5j1rxL+YRAupwAFQJ/Y96CG1/U28JOY5cFASUyAqClQnwV8UKntAeCGmOP9dMy6TKCMICQeQ/BHzUFAym7e+0eAP8csD2DXQP14zPoDgeWV9nEt8Gh4fz5wbMy67jteM0EIj625FVDMN4H6LGBKeD+VILyNrqbuG8PHbo65vRuzPo0gdH0BvAZYpc/Q5THLJwOLwvtH8U2gPjysISVm26eAG6t6b77NZ5udA/X5wLRKj/8YuCjm5/f6mHU/AV6r4bnPIwit2wj+sLwmZt1kqg7Ux+zm9WwiGOq14xg8GbOuK8H/BS1i2s7ZcVwI/sC5LGbdcVQfqKv92WMPPt+66daUbhryIdJwvufurQl+6Q8Bdvm6192LgFvCm1W1E3dfB9wN3FwPNXUi6L1bFtO2jKC3uEpmdqCZvRt+nb+FoOequq+uq1Ld+9AHOCP8en6zBUNMDiMIdT2Aje5eELOfFVXsO7atD3BnzL42Eryn1b62Snqw6/vSjCB47E4f4MBKr+U8oFtVtbp7flhfD3d/h+D43gOsMbMHzayNVTqxNKbGFVXts5q2PkCPSnVdF/Oa+gAvxKybTxD0u1Z+LnffEfR2eAkYZmb9Cb5d2OLu02p4jya6e7uY29Ex+y4hCLwjgNvd3Wt4TcvC2irrAaxw9/JK28Ye/53erzp+tit/Xqp6vtUx9wsI/pCqkgcnPB5H0JN7OXCzmZ24mxoqv55fh8MutoTHsy3Vv54+BH/I5MYc/wcIeqqhdp+12H1V+bNX3ed7N69LpNFToBZpYO7+HkFY+Fs1mzxK8IvvtBp281fgaOCAOpaznqAHsk9MW29gZXi/cpABmAC8DPRy97YE46yrDP81qeJ9WEHQQx0bslq5+21ALtDBzFrG7KIXu4qtdwVBj1rs/lq4+0e1LHEVu74vpQTDAHZnBfBepefOdPcrqqrfgplOOoTPibvf5e4HEHwLMQj4rVd9YmkuwVfwu+wzRuX3ZEmlulq7+8kx60+qtD7D3VeGzxVbc0uC4QSENRcCEwn+cDifYPjJt2JmPYEbCH4Wbjez5pU2iX2dvQnft0pWAb3MLKXStitjlit/vmv6bFf1s1D5+fpUaqv8fHvM3Uvc/T8EQ192jDWurpaK9nC89NXAmUB7D0763EL1r2cFQQ91p5hj38bdh4fra/NZi91XtT97VX2+a9iXSJOgQC0SjX8Ax5vZyMor3L2U4OvYq6t7sLtvBm4HfleXIty9jCAE3WpmrcMTh35FMJwCgvCYZWbpMQ9rTdBbXGjBNH/n1qGEf/DN+/Ak8F0zOzE8ASrDgnmFs9x9GTADuNHM0s3sYOC7u9n3/cC1ZjYcKk64OiNm/RqCMcTVeQr4pQUnQ2YCfyKYmaK0Fq/rFWCQBSfxpYW3URac6LnDyWZ2WPje3gJMdfcV4XYHWnAy3zagkKCXuCoTgYstOPmuJcGwjJpMA/LM7GozaxG+zyPMbFS4/n6Cz0IfADPrbGZjw3XPAqfE1Hwzu/4OeZxgGMSpfPMZ2iNmZgR/aP0LuIQgyN1SabMrzSzLghMmrwOeYVdTCd6/34Xv/1EEn5mna3j6mj7b64Byqv/MTCI45ueaWTMLTnQdRvBZ2CMWnOz4nfBnMsXMTiIIn1PDTXb32d3xWkrDupuZ2R+A2J7gNUDfHX9wuHsu8AbBHzBtwufdy745SXoi8HMz62lm7ajh/ydq+Nnbw8+3SJOhQC0SgXDYxuNUPwXeUwRBoiZ3UukXkQUzR9y/h+X8jOAX22LgQ4JeukfCde8QTA232szWh20/Ifj6eStBgJu4h89XIfZ9cPcVwFiCgLSOoJfrt3zz/9R5wMEEwwz+SBCiimrY9wvAX4CnzSyP4ES2k2I2uRF4LPxa+swqdvEIQS/r+8ASgl/8P6vl69oKnACcTdBzuTqsJbandQJBL+xGgm8azgvb2wAPEYx3XRa+3iq/zXD3VwlmfnmX4CSwj8NVVb4v4R9Q3yWYWWUJwTcUDxN8IwLBZ+pl4I3w+H5CMO4ad58LXBnWnRvWt9NFVNx9CkHonOXuS6uqIcaOGSZib10ITsztSvCZcIITOy+2nedKn0AQ/haHt12mm3T3YoJgf1L4Ou8FLnD3L2uoqdrPdjjc6FZgSviZOajS820ATiE4+W8DwR+7p7j7evZcHsHPwXKC8eX/j+CEzx0zitwJ/MCCGTTuqmYfrxOMZf6a4HNUyM7DNP4T/rvBzGaF9y8gGAI2j+D4Pksw5AqCz+QbBD3lnxL8AVFKFWF4Nz97tf58izQltuvQNBGRxs+CC1186e43RF3LnjKz8QQnz11fz/sdShBemteyJ73emdk7wAR3fzhO+19KcPLbW/HYv9RO2Gt+v7tXHuYikpTUQy0iTUL4VfFe4VfRYwh6s1+MuKzIWXAJ9HQza0/QK/jfCMP0KIIpAqsagiFNWDhE6ORwOMuOMe4vRF2XSGOhQC0iTUU3gqm58gmGOVzhuuQ3BNMcriOY47gMuKLmzePDzB4jmObtF+GQF0ksBtxEMFTjU4IZYHY3Zl8kaWjIh4iIiIhIHaiHWkRERESkDhSoRURERETqoFm8dmxmGQTTTTUPn+dZd7/BzG4ELiUY8wdwnbtPCh9zLcG8o2XAVe7+ek3P0alTJ+/bt298XoCIiIiISGjmzJnr3b1zVeviFqgJ5kE9xt3zwwncPzSzV8N1f3f3neadNLNhBHO2Die4xOlbZjYonDe1Sn379mXGjBlxKl9EREREJGBmy6pbF7chHx7IDxfTwltNZ0COBZ529yJ3X0JwkYLR8apPRERERKQ+xHUMdXhZ28+AtcCb7r7jsqk/NbPZZvZIOHcqQE92vopTTthWeZ/jzGyGmc1Yt25d5dUiIiIiIg0qroHa3cvcfSSQBYw2sxHAfcBeBJe+zQVuDze3qnZRxT4fdPdsd8/u3LnKYSwiIiIiIg0mnmOoK7j7ZjObDIyJHTttZg8Br4SLOUCvmIdlAav29LlKSkrIycmhsLCwDhU3fhkZGWRlZZGWlhZ1KSIiIiJJLZ6zfHQGSsIw3QI4DviLmXV399xws9OAOeH9l4EJZnYHwUmJA4Fpe/q8OTk5tG7dmr59+2JWVad30+fubNiwgZycHPr16xd1OSIiIiJJLZ491N2Bx8wslWBoyUR3f8XMnjCzkQTDOZYSXDYXd59rZhOBeUApcGVNM3xUp7CwMKHDNICZ0bFjRzSGXERERCR6cQvU7j4b2K+K9vNreMytwK11fe5EDtM7JMNrFBEREWkKdKXEOHnttdcYPHgwAwYM4Lbbbttlvbtz1VVXMWDAAPbZZx9mzZpVsa5v377svffejBw5kuzs7IYsW0RERET2UIOclJhsysrKuPLKK3nzzTfJyspi1KhRnHrqqQwbNqxim1dffZUFCxawYMECpk6dyhVXXMHUqVMr1r/77rt06tQpivJFREREZA+ohzoOpk2bxoABA+jfvz/p6emcffbZvPTSSztt89JLL3HBBRdgZhx00EFs3ryZ3NzcavYoIiIiIo1VQvdQ3/TfucxblVev+xzWow03fHd4jdusXLmSXr2+mQEwKytrp97n6rZZuXIl3bt3x8w44YQTMDMuu+wyxo0bV6+vQURERETqT0IH6qi473qF9conEda0zZQpU+jRowdr167l+OOPZ8iQIRxxxBHxKVZERESkkVmwZis5m7dXuW6/Xu1o1zK9gSuqWUIH6t31JMdLVlYWK1Z8cxX1nJwcevToUettdvzbpUsXTjvtNKZNm6ZALSIiIkmhrNwZe88UCoqrnj154mUHM7pfhwauqmYJHaijMmrUKBYsWMCSJUvo2bMnTz/9NBMmTNhpm1NPPZW7776bs88+m6lTp9K2bVu6d+/Otm3bKC8vp3Xr1mzbto033niDP/zhDxG9EhEREZH4W7h2Ky98uhJ3KCwpp6C4jMuP3IsTh3fdZdsBXTIjqLBmCtRx0KxZM+6++25OPPFEysrK+NGPfsTw4cO5//77Abj88ss5+eSTmTRpEgMGDKBly5Y8+uijAKxZs4bTTjsNgNLSUs4991zGjBkT2WsRERERiYeHP1jMFyu3APDSZ6sASEsNhr9mNm/GSSO6sW+vdlGVt0esqrG8TUV2drbPmDFjp7b58+czdOjQiCpqWMn0WkVERCRxPPT+Ym6dNB+Avh1b4sCYEd249qTGm2vMbKa7V3mBEPVQi4iIiEjcxHbePjN9BXe+vYDcLYUAvPKzwxjRs21UpdUbBWoRERERqXcFxaVc/dwX/PfzVbusOyu7F2eN7pUQYRoUqEVERESkHhWWlPH5is388F9TKSkLeqd/cdzAivWH7NWp0c3SUVcJGajdfZd5nxNNUx77LiIiIonpq9VbuXLCLBauzQdg755tufe8/enVoWXElcVXwgXqjIwMNmzYQMeOHRM2VLs7GzZsICMjI+pSRERERACYtyqPk+/6AIDWGc14+IJsRvZuR/NmqRFXFn8JF6izsrLIyclh3bp1UZcSVxkZGWRlZUVdhoiIiCS5t+at4a35a3h6enDBuksO68dvTxxMRlriB+kdEi5Qp6Wl0a9fv6jLEBEREUl4yzcU8OPHgymMu7RuznkH9uHnMeOlk0XCBWoRERERiZ+i0jIufGQaa/OKWLx+GwC3njaC8w7sE3Fl0VGgFhEREZFdlJSVs2V7CXe/s5AJU5ez49S0otJyADplpvPdfXvQu0MLzh3dO8JKo6dALSIiIiI7WZ9fxEl3fsC6rUUVbZcd2b/ifou0VMYd0Z+W6YqSoEAtIiIikrTcnfe+XkdBcdlO7T/59ywAstq34LIj+nNQ/44M7No6ihKbBAVqERERkSSzcvN2Xv0il+lLN/L63DVVbnPkoM7ce97+tGquuLg7eodEREREksjCtVs57o73d2qbcOmBdGzVvGI5NQX6d8okJSUxr+lR3xSoRURERBLUR4vWM37KUmKvr/zmvKBH+jt7d+e20/cmvVlKUlx8JZ4UqEVEREQS1H9m5DD563Xs1Tmzom1o9zYcMbAT1548NMLKEosCtYiIiEgC2V5cRrk7HyxYxwufrmT/3u14/ieHRl1WQlOgFhEREUkQT36yjOtfnLNT2y+OGxRRNclDgVpERESkiSsqLWPq4o0VYfq6k4cAMLBra44Y1DnK0pKCArWIiIhIE3fTf+cxYepyAA4f2IlxR+wVcUXJRYFaREREpIn5eNEG3l+wrmJ5R5h+8cpDGdQ1s7qHSZwoUIuIiIg0EVMWrueZ6St4+fNVAKSnpgCQlmrceOpwRvZqF2F1yUuBWkRERKSJGP/RUt77eh39OrXi8iP7c9ao3lGXJChQi4iIiESuvNxrXF9a7pz70CfMztnCgf078MQlBzZQZVIbCtQiIiIiEXrgvUX8+dUva7Vtx1bpXH6kTjhsbBSoRURERCKwaVsxX6zcUhGmf7mb+aKbp6Vw3oG9aZ2R1hDlyR5QoBYRERGJo+3FZbwxbzUlZTsP67j+xS8oLCkH4KpjBvDz4wZGUZ7Ug7gFajPLAN4HmofP86y732BmHYBngL7AUuBMd98UPuZa4BKgDLjK3V+PV30iIiIi8Tbpi1xuf+MrFq3bVuX6vXu21ewcCSCePdRFwDHunm9macCHZvYq8H3gbXe/zcyuAa4BrjazYcDZwHCgB/CWmQ1y97I41igiIiJSb96ct4a35q0BoKSsnOc/XQlA+5ZpvHTlYZjtvH2Pdi1ITbHKu5EmJm6B2t0dyA8X08KbA2OBo8L2x4DJwNVh+9PuXgQsMbOFwGjg43jVKCIiIlJfJn2Ry0/+PQuAbm0ycJye7Vpw+5n7cmC/DljlNC0JI65jqM0sFZgJDADucfepZtbV3XMB3D3XzLqEm/cEPol5eE7YVnmf44BxAL17a+5FERERaTi3v/EVr8zOrXLdkvXBsI67z92PU/bp0ZBlScTiGqjD4Rojzawd8IKZjahh86r+bNtlUkZ3fxB4ECA7O7vmSRtFRERE6sHUxRu46NHpbC8JRqKeuu+ugXnvnm05ee/ujBnRraHLk4g1yCwf7r7ZzCYDY4A1ZtY97J3uDqwNN8sBesU8LAtY1RD1iYiIiFTnwwXr+eG/pgJw+v5ZjDuiP4O7tY64KmlM4jnLR2egJAzTLYDjgL8ALwMXAreF/74UPuRlYIKZ3UFwUuJAYFq86hMRERGpSmFJGe99vY7ScJq7KycE46JvPW0E5x3YJ8rSpJGKZw91d+CxcBx1CjDR3V8xs4+BiWZ2CbAcOAPA3eea2URgHlAKXKkZPkRERKQhzVy2kWuf/4Kv1+Tv1H74wE4K01ItCybjaJqys7N9xowZUZchIiIiTdj24jLuemcB24vLGP/RUgAymzfjuSsOqZjmrl+nVqSlpkRXpETOzGa6e3ZV63SlRBEREUlqFz46jWlLNgLQtkUavzhuIOcf1IdmCtBSSwrUIiIikpRih3eYwfybx5CRlhp1WdIEKVCLiIhIwiosKaO0vOrhrXe9vZCv1+QzZng3rj5piMK0fGsK1CIiIpIQysqdJevzKSsPlqcsXM/Nr8yr8TGdMptz//kHNEB1ksgUqEVERKTJcnemLtlIQXEpj05ZygcL1u+yzS+PG0TL9Kp7nw/q3zHeJUoSUKAWERGRJmXh2nw+XhQE55nLNvHiZztfB+7e8/avuN+jXQtG9mrXkOVJElKgFhERkSblllfm8d7X63Zq+9eF2XTKbE7P9i3olNk8osokWSlQi4iISJPxv9m5vPf1Oo4d0oW//GAfAFqkpdKquSKNREefPhEREWkyHnh/EQAn791dPdHSaChQi4iISKNWFk57t72kjNk5WzjjgCxOPyAr4qpEvqFALSIiIo3Cuq1FbC8u26ntT5Pm89rc1Tu1DeiS2ZBlieyWArWIiIhEJnfLdpas38aXuVtrnDP6V8cPAqBleio/PKhPQ5UnUisK1CIiIhJX6/OLmPzVOtx3vWLhb5+dvdPyr44fRM92LXZqO7B/B7Lat4xrjSJ1oUAtIiIicfXzpz9lysIN1a4/flhXLjmsH+1apjGkW5sGrEykfihQi4iISL2btXwTz0xbgeNMWbiBtFTjnV8ftct2KSlGj7YZmFnDFylSTxSoRUREpF7lFZbw/Xs/AqB72wx6tmvB/50ylF4dNGxDEpMCtYiIiFRpwZqt/HTCpxSVlu1+4xhLNxQAcFZ2r4qLr4gkMgVqERER2Ym7k7ulkOP//j4Ahw/sRIdW6bV+/L692tG2RRq//87QeJUo0qgoUIuIiEiFTduKufG/c3nps1UAHDOkC49cNCriqkQaNwVqERERqXD8399jfX4xAH87Y19O2ad7xBWJNH4K1CIiIgLAP99ewPr8Ynq2a8HT4w7SSYQitaRALSIikoQmfZHL1MXfzA29taiU52etBOCJS0YrTIvsAQVqERGRJLGtqJTfv/AF+UWlvDV/LQDtWqYB4A6dMtO597wD6N85M8oyRZocBWoREZEkcdurX/LiZ6tIb5bCiJ5t+OVxgzh2aNeoyxJp8hSoRUREmrhZyzcx7vGZFO9mvui8wlIAZt9wAhlpqQ1RmkhSUKAWERFpgtbkFbKpIJiN4/W5q1mfX8T5B/UhNaXmS3ifNKKbwrRIPVOgFhERaULcnTfmreGyJ2bu1N46oxk3jx2OWc2BWkTqnwK1iIhIEzJr+eaKMH3+QX04ZK+OAPTu2FJhWiQiCtQiIiJNyCNTlgAw/uJRHDmos0K0SCOgQC0iItIEuDt/mjSf/83OBeCg/h0VpkUaCQVqERGRRuzzFZv54//mkbulkJxN2wH4948P1ImFIo2IArWIiEgj88bc1fzhpbmUubNuaxEAhw7oyNDubbh57HC6t20RcYUiEkuBWkREpJHILyrl9Tmr+fV/Pgfg7FG9MINh3dtw/sF9oy1ORKqlQC0iItIIzFuVx8l3fVCxfOfZIxk7smeEFYlIbSlQi4iIRKCkrJxX56ymsKSMlZu2c+fbCwA4oE97fnPCYA4Op8MTkcYvboHazHoBjwPdgHLgQXe/08xuBC4F1oWbXufuk8LHXAtcApQBV7n76/GqT0REJArrthbxzPTlvDV/LZ+t2LzTuj9+bwRnZvcivVlKNMWJyLcSzx7qUuDX7j7LzFoDM83szXDd3939b7Ebm9kw4GxgONADeMvMBrl7WRxrFBERaTA5mwo44e/vU1Ac/GprmZ7Kyz89lIy0VFqkpdIxs3nEFYrItxG3QO3uuUBueH+rmc0HahoMNhZ42t2LgCVmthAYDXwcrxpFREQa0u9fmENBcRlHDe7MoxeNAtBc0iIJoEHGUJtZX2A/YCpwKPBTM7sAmEHQi72JIGx/EvOwHKoI4GY2DhgH0Lt37/gWLiIiUgcFxaX88OGprM8vxgyWbSigVXoq4y8eHXVpIlKP4j5Iy8wygeeAX7h7HnAfsBcwkqAH+/Ydm1bxcN+lwf1Bd8929+zOnTvHp2gREZE6uvOtBQz7w+vMWr6Z0rJy9uvVjtP268kjYc+0iCSOuPZQm1kaQZj+t7s/D+Dua2LWPwS8Ei7mAL1iHp4FrIpnfSIiIvUtd8t2Xv5sFX9/62sAfnPCIH58eH9d2VAkgcVzlg8D/gXMd/c7Ytq7h+OrAU4D5oT3XwYmmNkdBCclDgSmxas+ERGR+rZxWzEH//mdiuUJlx7IIXt1irAiEWkI8eyhPhQ4H/jCzD4L264DzjGzkQTDOZYClwG4+1wzmwjMI5gh5ErN8CEiIk3JOQ8GpwKdum8PLjuyP8N7tI24IhFpCPGc5eNDqh4XPamGx9wK3BqvmkREROJl2YZtfLVmK63SU7nz7JGavUMkiehKiSIiInWwpaCE61+aw1vzglOE7jlvf4VpkSSjQC0iIlKD8nLnsidnsmJjQZXrv1y9FYDWGc04dq8uHDpAY6ZFko0CtYiISBXKy53/fZHLVU9/ijukN0vh6MG7Ttfap2NLOrRK56ZTR+iS4SJJSoFaRESE4CIsyzZ80wv9m/98ztxVeQBcdkR/Lj9yL9q3So+qPBFpxBSoRUQk6c1dtYWLHp3Ouq1FO7W3bt6Mu8/bnyMH6UJiIlI9BWoREUlqHy1cz7kPTwWgd4eWXHfykHCNMbpfBzqoV1pEdkOBWkREklJ5ufPgB4u57dUvAfjtiYO55LB+uqKhiOwxBWoREUk6pWXlXDx+Oh8sWA/AzWOHc8HBfaMtSkSaLAVqERFJOldOmMUHC9ZjBlOvPZYubTKiLklEmjAFahERSWgFxaWcft/HrNtahBm4w/r84OTDGb8/jo6ZzSOuUESaOgVqERFJGKVl5eRs2r5T2zMzVjA/N49eHVpw2IBgto7UFLjksP4K0yJSLxSoRUSkSXN3Pl2xme3FZfxp0vyKuaMre/0XR9AyXb/2RKT+6X8WERFpUqYv3ciSddsqlqcu2chzs3Iqllump3LraSN2ekyfjq0UpkUkbvS/i4iINEpL12/jxc9W4v5NW7k7/3xnYZXb3//DA+jQKp1BXTNp11JzR4tIw1GgFhGRyC1Ys5WHP1hCWUx6fnZmTrXb3zJ2OMcM7VqxnNm8GW1bpMW1RhGR6ihQi4hIg3v/63Xc/96iit7njxdvAKBH2wzMrOL+8cO6ctPYEdXtRkSkUVCgFhGRuFu3tYgLH5nG1qISDGP5xgIARvftUPHvAX3bc/WYITXtRkSkUVKgFhGRuNhcUExRaTkAVzw5k3m5eQzt3oYh3VpzQJ/2HDagE6cfkBVxlSIidadALSIidbZw7Va2bC+tWP50+Sb++L/5O22TkZbC/352GCkp1tDliYjElQK1iIh8a58u38SUhev52xtfV7n+NycMokOr4OIphw7oqDAtIglJgVpERPaIu/PszBxytxRyx5vfBOnrvzOUgV1bVyx3ykxneI+2UZQoItKgFKhFRKRG46csYVHMhVSWbtjGBwvWVyz/3ynD+M7e3enWNiOK8kREIqdALSIi1Zq+dCM3/nceAB1aBRdLcXe6t81g/MWj6dupJc2bpUZZoohI5BSoRURkF1MWrueP/5vP/Nw8AF79+eEM7d4m4qpERBonBWoRkSRXUlZOQXEZAFsLS/jBfR+zOq8QgBOHd2X/3u0VpkVEaqBALSKShNydpRsKKCgu5bR7P6I4nC96hxE923Dp4f0ZO7JnRBWKiDQdCtQiIklk4dp8lqzfxvSlG3nw/cUV7cO6t6m4yEqr9FR+cEAWzVJToipTRKRJUaAWEUkS83PzOOnOD3Zq+/P396ZL6+YcOqATGWk6uVBE5NtQoBYRSQJr8gorwvQFB/fhzOxetGuZRlb7lhFXJiLS9ClQi4gkuOLSck78x/sAXH7kXlxz0pCIKxIRSSwaICciksBKy8o59e4P2VxQwqCumQrTIiJxoB5qEZEEc+PLc3lldi5msHFbMWXlTvuWafzn8kOiLk1EJCEpUIuINFHbi8sq5ove4a63F/DCpysBOGd0bwAy0lL4+bEDadsircFrFBFJBgrUIiJNxFert7Jua1HF8k+fmsXmgpIqt33n10fSv3NmQ5UmIpLUFKhFRBqp8nJn0pxcthWVsmV7CX+a9OUu2/Tv1Iqrjh24U9t+vdvRp2OrhipTRCTpxS1Qm1kv4HGgG1AOPOjud5pZB+AZoC+wFDjT3TeFj7kWuAQoA65y99fjVZ+ISGNWWlbOlRNm8frcNTu1//bEwYzu1wEAA4b3aEuLdM0fLSISpXj2UJcCv3b3WWbWGphpZm8CFwFvu/ttZnYNcA1wtZkNA84GhgM9gLfMbJC7l8WxRhGRRmP8lCV8sTIPgM9zNrNwbT4Ar/zsMDq0Sqd5sxQ6ZjaPskQREalC3AK1u+cCueH9rWY2H+gJjAWOCjd7DJgMXB22P+3uRcASM1sIjAY+jleNIiJRmbcqjz9Nmk9peTkA5Q7TlmwEoGe7FkBwOfCnLj2Iti11MqGISGO220BtZk+4+/m7a9vNPvoC+wFTga5h2Mbdc82sS7hZT+CTmIflhG2V9zUOGAfQu3fv2pYgItKonPnAx+QXlXJAn/akphgAh+zVkavHDGHfXu2iLU5ERPZIbXqoh8cumFkqcEBtn8DMMoHngF+4e56ZVbtpFW2+S4P7g8CDANnZ2busFxFpzNyd616YQ35RKfv2asdzV2huaBGRpq7aKyWa2bVmthXYx8zyzGxruLwWeKk2OzezNIIw/W93fz5sXmNm3cP13cP9QdAj3Svm4VnAqj16NSIijdxfXvuKp6YtB+CRC7MjrkZEROpDtYHa3f/s7q2Bv7p7G3dvHd46uvu1u9uxBV3R/wLmu/sdMateBi4M71/IN+H8ZeBsM2tuZv2AgcC0b/GaREQapclfreX+9xYB8NavjtQJhiIiCaI2Qz5+b2Y/BPq5+y3hdHjd3X13YfdQ4HzgCzP7LGy7DrgNmGhmlwDLgTMA3H2umU0E5hHMEHKlZvgQkUTx5CfLuP7FOQD8bsxgBnTRRVdERBKFudc8DNnM7iOYR/oYdx9qZu2BN9x9VEMUWJPs7GyfMWNG1GWIiNSovNzpf90kAB770WiOGNiJGs4nERGRRsjMZrp7lWP1atNDfaC7729mnwK4+yYzS6/XCkVEEtjVz80G4CdH7cWRgzpHXI2IiNS32gTqknBmDwcws84EPdYiIlKDZRu28bOnPmV2zhYALjykb7QFiYhIXNQmUN8FvAB0MbNbgR8A18e1KhGRJqq4tJxnZ+bwp0nzyS8qBWB03w7cNHY4XdtkRFydiIjEQ42B2sxSgCXA74BjCeaK/p67z2+A2kREGrXSsnIWrM2nPDwXxR1Ov+8jikqDL/EuOawf3dtmcMlh/TRmWkQkgdUYqN293Mxud/eDgS8bqCYRkSbhJ/+exRvz1uzSPrR7G2747jAO6t8xgqpERKSh1WbIxxtmdjrwvO9uShARkSTw9vw1fLRoQ0WYfuD8by4em5ZqHLJXJzLSUqMqT0REGlhtAvWvgFZAqZkVEgz7cHdvE9fKREQaoYLiUi59fAblYffCxMsOZnS/DtEWJSIikdptoA6vligiIsD83DzKHf74vRGctl9PWjWvTb+EiIgkst3+JjCz/ato3gIsc/fS+i9JRKTxWrxuGwBDu7dWmBYREaB2Qz7uBfYHvgiX9wY+Bzqa2eXu/ka8ihMRiUJJWXnFzB2x3OG3zwYXaRneo21DlyUiIo1UbQL1UuASd58LYGbDgN8CtwDPAwrUIpIw3py3hsue+GaMdFUO6NNeJx2KiEiF2gTqITvCNIC7zzOz/dx9seZVFZFEUV7uzFy+iVe/yAXgtycOrnK79NQUzhzVqyFLExGRRq42gforM7sPeDpcPgv42syaAyVxq0xEJE4Wr8tn+tKNO7VN/modr85ZDUDvDi258ugBUZQmIiJNUG0C9UXAT4BfEEyZ9yHwG4IwfXS8ChMRiYdXZq/ipxM+rXb9+ItHMaSbZgUVEZHaq820edvN7F7gFXf/qtLq/PiUJSJSv1ZsLODOtxfw7MwcAM4e1Yurjh240zatM5rROiMtivJERKQJq820eacCfwXSgX5mNhK42d1PjXNtIiL15sH3F/PszBx6tmvB7Wfuq8uCi4hIvanNkI8bgNHAZAB3/8zM+saxJhGRepdXGJzyMeWaYyKuREREEk1tAnWpu2/RjB4i0hRtLSzh0NveIa+wlNF9dYlwERGpf7UJ1HPM7Fwg1cwGAlcBH8W3LBGRb2/L9hIWrNkKwD3vLiSvsJT9e7fj6pOGRFyZiIgkotoE6p8BvweKgKeA1wgu6iIi0uiUlJVz4t/fZ3Ve4U7tEy87mGapKRFVJSIiiaw2s3wUEATq3wOY2RDgbuDS+JYmIrJn3J0rnpzF6rxCurZpzt/O2BeAPh1aKUyLiEjcVBuozWwf4G9AD+AFghB9L3AgcHuDVCciUoPHPlrK1+HQDoCvVm9lxrJNALz5qyNpoynwRESkAdTUQ/0QcB/wMTAGmAVMAM5z98IaHiciElertxRy3Qtf8M6XawHolJkOgDtktW/BxMsOVpgWEZEGU1Ogbu7u48P7X5nZb4Br3L0s/mWJiOyqvNy57MmZvDlvDQB9OrbknnP3Z0TPthFXJiIiyaymQJ1hZvsRXG4cgqsi7mPh/HnuPivexYmIQNAjPfaeD1mTVwRAerMUzj+oD78/eSgpKZrSU0REolVToM4F7ohZXh2z7ICujiAicbUmr5B1W4uYtXwTa/KK+N7IHvRo14IfH96fDq3Soy5PREQEqCFQu/vRDVmIiCS2vMISPlm0Aa/l9iVl5fx0wqc7tV178lC6tsmo/+JERETqoDbzUIuI1NnVz87m1Tmr9/hx39+vJ2NGdKNjZnOFaRERaZQUqEWkXpSVO/dNXsjmgpIq1+8I0/+76rBa77N5sxT26pxJeOqGiIhIo6RALSJ7zN257dUvWb6xoKLt6zVbWbRuGwCt0lN3eUzL9FSuHjOE4T00I4eIiCSWmi7ssn9ND9QsHyLJ64lPlvHA+4sBGNQ1E4DUFGO/3u14+IJsOmY2j7I8ERGRBlVTD3VNV0PULB8iSaSkrJyy8uB0wpv+O5enpq0A4IPfHU2vDi2jLE1ERCRymuVDRGr02pxcLn9y1y+kHr14lMK0iIgItRxDbWYjgGFAxSn27v54vIoSkeiszSvky9VbAVi+sYDrX5wDwGVH9KdtyzQM4+S9u9GnY6soyxQREWk0dhuozewG4CiCQD0JOAn4EFCgFklA37/vI3I2bd+p7ZbvjeD8g/pEVJGIiEjjVpse6h8A+wKfuvvFZtYVeHh3DzKzR4BTgLXuPiJsuxG4FFgXbnadu08K110LXAKUAVe5++t7+FpEpJaWbyjg2Vk5uO98mZVtRWXkbNpOVvsW3Hn2SAAym6cxuFvrCKoUERFpGmoTqLe7e7mZlZpZG2At0L8WjxsP3M2uPdl/d/e/xTaY2TDgbGA40AN4y8wGuXtZLZ5HRGppzsotPDplKc/NygEgpdL0zuUOzVKMv52xLwf06RBBhSIiIk1PbQL1DDNrBzwEzATygWm7e5C7v29mfWtZx1jgaXcvApaY2UJgNPBxLR8vIrXwzPQVvPBpDlntW3DU4M788Xt7R12SiIhIk7fbQO3uPwnv3m9mrwFt3H12HZ7zp2Z2ATAD+LW7bwJ6Ap/EbJMTtu3CzMYB4wB69+5dhzJEEk95uXP+I1NZtqGgyvUb8osZ2KU1r//yiAauTEREJHHV5qTEt939WAB3X1q5bQ/dB9xCMI/1LQRzXf8IqOq6wl5FG+7+IPAgQHZ2dpXbiCSjwpIyFq7NZ8rCDezfux19O1U9C8cxQ7o0cGUiIiKJraYrJWYALYFOZtaeb0JvG4JxznvM3dfE7P8h4JVwMQfoFbNpFrDq2zyHSLIpLStn9sotXPzodLZsLwHgyqMHcOzQrhFXJiIikhxq6qG+DPgFQXiOvapDHnDPt3kyM+vu7rnh4mnAnPD+y8AEM7sjfL6B1GKctojAX1//quIy4P07t+Lnxw7kiEGdI65KREQkedR0pcQ7gTvN7Gfu/s893bGZPUUwf3UnM8sBbgCOMrORBMM5lhKEdtx9rplNBOYBpcCVmuFDpGYFxaVMmLq8IkxP+PGB7Ne7PS3SUyOuTEREJLlY5Xlod9nALB24HNhxFtNk4AF3L4lvabuXnZ3tM2bMiLoMkQbx1eqtPPHJUsrDH9l35q9ldV4hAFcdM4BfnTA4wupEREQSm5nNdPfsqtbVZtq8e4G08F+A8wlOLvxx/ZQnIjVZsbGAm1+Zx5vzglMQOmU2D9c4w3u0YeJlB9OqeW1+lEVERCQeajopsZm7lwKj3H3fmFXvmNnn8S9NRAAefH8xb85bQ79OrThpRDd+N2ZI1CWJiIhIjJq6taYB+wNlZraXuy8CMLP+BJcHF5E4KSkr5z8zcvjj/+ZRUBz8uL3z6yMxq2qGSREREYlSTYF6x2/u3wDvmtnicLkvcHE8ixJJNtuKSlmyflvF8kWPTmN9fjEAlx7ej0MHdFKYFhERaaRqCtSdzexX4f0HgFRgG5AB7Ae8G+faRJLGGfd/zLzcvJ3aendoyf/7wT4c1L9jRFWJiIhIbdQUqFOBTHa+imFm+G/ruFUkkiC2FJTw0ucrKS2reSadDduKmJebR0ZaCv88Z38AUlPgwH4ddbKhiIhIE1DTb+tcd7+5wSoRSSArNhZw2r1TKoZt1MZjF4/mQPVGi4iINDm1GUMtIrXw3tfreGrqcgBem7sagH2y2vLEjw7c7WPTmhkt09UbLSIi0hTV9Bv82AarQqSJm52zmQsfmQbA4K6tGdy1Nfv3acefv79PxJWJiIhIvNV06fGNDVmISFP11rw1/Pjx4Iqdt31/b84e3TviikRERKQh6TtmkT20Pr+IzQUlAORu2V4Rpm/53giFaRERkSSkQC1SS+7O5K/WcfH46bus++2Jgzn/oD4RVCUiIiJRU6AWqYU1eYXc8cbXPDNjBQBnj+rFIQM6AdA6oxlHDeocZXkiIiISIQVqkd0oLCnj+DveI6+wFIDHfjSaIwbqyoUiIiISUKAWqcHMZRv5+dOfkVdYyqCumTx7xSG0yUiLuiwRERFpRBSoRarxxCfL+L8X5wAwslc7/v3jA3XlQhEREdmF0oFINf74yjwA/u+UYVxyWL+IqxEREZHGSoFahGAGjzV5Rbw5f01FrzTAWdm9FKZFRESkRgrUkvTyi0q57vkvePnzVRVtPz16AOnNUjh7VK8IKxMREZGmQIFakt5NL8+tCNP/7/R96Ne5FaP6doi4KhEREWkqFKgl6X20aAPDurfhwQsOIKt9y6jLERERkSYmJeoCRKK0aVsxKzdvp2NmusK0iIiIfCvqoZak4+7c9N95LF6/jY8Wrgfgu/v2iLgqERERaaoUqCWpFJWWcdYDn/DZis1AML90z3YtOH3/rGgLExERkSZLgVoSkrsza/kmLnpkOsVl5aSElwnfXlIGQLc2Gfzn8oPp1UHDPERERKRuFKglId3yynwembIEgJNGdNspOGekpXLZEf111UMRERGpF0oUknCWrt9WEabvOXd/vrNP94grEhERkUSmQC0JoaSsnGdn5lBQXMYt4SXDzzggS2FaRERE4k6BWhLCIx8u4c+vflmxfMYBWfzp+3tHWJGIiIgkCwVqSQgvfhZc6fCja44hM6MZbTLSIq5IREREkoUCtTRZE6ev4PFPlgIwPzePkb3a0aNdi2iLEhERkaSjQC1Nxntfr+PXEz+jrNwxMzZuKwbg2CFd6DY0gx8d1i/iCkVERCQZKVBLo5ZfVMrqLdsBuPa52azPL+a8A3tXzCs9ZkQ3Dh3QKcoSRUREJMkpUEuj9dXqrZzyzw8oKfOKtkFdM7n1NJ1sKCIiIo2HArU0Clu2l/D6nNWUlgfhuai0jJv+G0x/t3fPtow7oj8A2X3bR1ajiIiISFXiFqjN7BHgFGCtu48I2zoAzwB9gaXAme6+KVx3LXAJUAZc5e6vx6s2aXx+8u+ZTFm4YZf2nx87kCuPHkB6s5QIqhIRERHZvXj2UI8H7gYej2m7Bnjb3W8zs2vC5avNbBhwNjAc6AG8ZWaD3L0sjvVJHH20aD0Tp6+o1bZlTkWYnnrdsRXtaakpdGiVHpf6REREROpL3AK1u79vZn0rNY8FjgrvPwZMBq4O25929yJgiZktBEYDH8erPomfjduKOfehqQD06dhyt9u7Q//OrfjzaXvTtU1GvMsTERERqVcNPYa6q7vnArh7rpl1Cdt7Ap/EbJcTtu3CzMYB4wB69+4dx1Ll27hv8iL+8lpwxcJLD+/H778zLOKKREREROKrsQxMtSravIo23P1Bd8929+zOnTvHuSypjZKycnI2FfDszJyKMD3uiP5cPWZIxJWJiIiIxF9D91CvMbPuYe90d2Bt2J4D9IrZLgtY1cC1yR4oLi1nds5mysqdq5+bzdINBRXrnr38YLL7doiwOhEREZGG09CB+mXgQuC28N+XYtonmNkdBCclDgSmNXBtUsnarYVMWbi+ynXjP1rG5ys2Vyx3ykznd2OG0Kt9S4VpERERSSrxnDbvKYITEDuZWQ5wA0GQnmhmlwDLgTMA3H2umU0E5gGlwJWa4SNa7s4pd33I2q1F1W6TmmI88aPRAIzIakubjLSGKk9ERESk0YjnLB/nVLPq2Koa3f1W4NZ41SO1s3Lzdu6fvIi5q7awdmsR3dtm8NSlB1W5befWzWnVXNcGEhERkeSmNCQVZi7bxOn3fQRAu5Zp9O3YkqfHHUy3tprKTkRERKQ6CtQCwIPvL+JPk4IZOs4e1YvbTt8n4opEREREmgYFamF2zuaKMH3TqcO58JC+0RYkIiIi0oQoUAv/eGsBAOMvHsVRg7vsZmsRERERidVYLuwiEXnh0xze+TKYDlxhWkRERGTPKVAnsYLiUn75zOcA/OvC7IirEREREWmaFKiTVHm5c/2LcwD4zQmDOHZo14grEhEREWmaFKiT1KJ1+Tw/ayUAY0Z0j7gaERERkaZLgTpJvff1OgAmXnYwA7pkRlyNiIiISNOlQJ2k1uQVArB3z7YRVyIiIiLStClQJ6kN+cX0bNeCFumpUZciIiIi0qQpUCepqUs20ql186jLEBEREWnydGGXJPP63NVMW7KRlZu306djy6jLEREREWnyFKiTyNTFG7jsiZkVy9edPDTCakREREQSgwJ1kthSUMJ5D08FYMKlBzKiZ1vaZKRFXJWIiIhI06dAnSTuemcBpeXOoQM6cshenaIuR0RERCRhKFAnuLJy55yHPmHako0AjL94dMQViYiIiCQWzfKR4P76+ldMW7KR9i3TeOiCbNJSdchFRERE6pN6qBPYrOWbuP+9RQC8/7ujaa0x0yIiIiL1Tt2VCWr5hgJ+NuFTAB66IFthWkRERCRO1EOdoE6950M2F5TQtU1zjh/WNepyRERERBKWeqgT0AcL1rG5oISh3dvw7m+OirocERERkYSmQJ2Afv/CHAD+ec5+tEzXlxAiIiIi8aRAnWBWbylk+cYCBndtzYAumVGXIyIiIpLwFKgTzHOzcgD45fGDIq5EREREJDkoUCeQ9flFLFizFYATdCKiiIiISIPQANsEsXBtPsfd8R4AQ7q1JiXFIq5IREREJDmohzoBrNhYUBGmDx/YiYcuyI64IhEREZHkoR7qJu7JT5Zx/YvBrB4XHNyHG747nFT1TouIiIg0GAXqJujDBet5/OOlFJaW8/7X6wD4/v49uXnsiIgrExEREUk+CtRNzPOzcvjVxM8BGNq9DftmteXW0/ZmRM+2EVcmIiIikpwUqJsId6eguIyXPlsFwF9O35uzRvWOuCoRERERUaBuhApLyli+sWCntt89O5vPVmwG4LihXRSmRURERBoJBepGZktBCafc/QErNm7fZV2LtFR+efxAjhrcJYLKRERERKQqCtSNyKJ1+Zxx/8ds3FZM1zbN+cMpw3daf1D/DnTMbB5RdSIiIiJSlUgCtZktBbYCZUCpu2ebWQfgGaAvsBQ40903RVFfQ3N3HpmylFtemQfAgC6ZvHTlobRqrr93RERERBq7KBPb0e6+Pmb5GuBtd7/NzK4Jl6+OprSG8/b8Ndz97kI+Xb4ZgPMO7M1Npw6nWaquuSMiIiLSFDSmLtCxwFHh/ceAySRwoH57/hrum7yIGcuCTvhh3dvwwPkH0KtDy4grExEREZE9EVWgduANM3PgAXd/EOjq7rkA7p5rZlWeeWdm44BxAL17N82ZLtydK/49i+LScg4b0IkzsrMYO7Jn1GWJiIiIyLcQVaA+1N1XhaH5TTP7srYPDMP3gwDZ2dkerwLjZUN+EWPvmUJxaTnHDe3CwxeOirokEREREamDSAK1u68K/11rZi8Ao4E1ZtY97J3uDqyNorZ4mr50I2fc/zEAPdu14M/f3yfiikRERESkrhr8zDcza2VmrXfcB04A5gAvAxeGm10IvNTQtcXT2q2FFWH6p0cP4NVfHE7n1poCT0RERKSpi6KHuivwgpnteP4J7v6amU0HJprZJcBy4IwIaoubix6ZDsBJI7rxmxMHR1yNiIiIiNSXBg/U7r4Y2LeK9g3AsQ1dT0NYuDafebl5tG2Rxj3n7h91OSIiIiJSjzTZcQO4/Y2vALj/hweQkmIRVyMiIiIi9UmBugFs3FZMi7RUDt6rY9SliIiIiEg9U6COs3Vbi5i6ZCPHDKlyWm0RERERaeIUqOPsy9V5AIzu1yHiSkREREQkHhSo4+z1uasBOGpw54grEREREZF4UKCOs5c+WwVA7w4tI65EREREROJBgTqO1m0tYmthKaP7dSCcd1tEREREEowCdRw9/OFiAC44uE/ElYiIiIhIvChQx8navEIeeC8I1EcN1gwfIiIiIolKgToOCkvKOPmuDwG45qQhZDaP4grvIiIiItIQlPTq2Zer8xjzjw8AyGrfgnGH94+4IhERERGJJwXqenbnWwsAuOiQvvzsmAG61LiIiIhIglOgrkf5RaW8OieYd/qG7w7TzB4iIiIiSUBjqOvRorX5AFz/naEK0yIiIiJJQoG6Hv338+AiLvtktYu2EBERERFpMArU9cTdefjDJQDs26ttxNWIiIiISENRoK4nG7cVA3DSiG40b5YacTUiIiIi0lAUqOvJrOWbAThlnx7RFiIiIiIiDUqBup6s3VoIwLAebSKuREREREQakgJ1PZm+ZCMQXMxFRERERJKHAnU92THkIy1Vb6mIiIhIMlH6qwdr8gpZvrGAMw7IiroUEREREWlgCtT14NrnvwDg0AGdIq5ERERERBqaAnUdrc8v4p0v15KWaowdqRk+RERERJKNAnUdzV2VB8A1J+ly4yIiIiLJSIG6jlZt3g7AsUO6RFyJiIiIiERBgbqOPlm8AYAubZpHXImIiIiIREGBuo5mLN1Ep8x0WqY3i7oUEREREYmAAnUdFJaUsXLzdrq31cVcRERERJKVAnUdLFm/DYDzD+4TcSUiIiIiEhUF6jpYuSk4IbFLa42fFhEREUlWCtR1cPMr8wAY2r1NxJWIiIiISFQUqL+l4tJylm8soEvr5uqhFhEREUliCtTf0sfhdHk/OqyfLugiIiIiksQUqL8Fd+fCR6YBcECf9hFXIyIiIiJRanSB2szGmNlXZrbQzK6Jup6q7Jjd47ihXclWoBYRERFJao0qUJtZKnAPcBIwDDjHzIZFW9WupixcD8CPD9dwDxEREZFk16gCNTAaWOjui929GHgaGBtxTbv4v5fmAprdQ0RERESgsV0vuyewImY5BzgwdgMzGweMA+jdu3fDVRbjyEGdye7TnrYt0iJ5fhERERFpPBpboK5q/ITvtOD+IPAgQHZ2tlexfdw99qPRUTytiIiIiDRCjW3IRw7QK2Y5C1gVUS0iIiIiIrvV2AL1dGCgmfUzs3TgbODliGsSEREREalWoxry4e6lZvZT4HUgFXjE3edGXJaIiIiISLUaVaAGcPdJwKSo6xARERERqY3GNuRDRERERKRJUaAWEREREakDBWoRERERkTpQoBYRERERqQMFahERERGROlCgFhERERGpAwVqEREREZE6UKAWEREREakDBWoRERERkTowd4+6hm/NzNYByyJ6+k7A+oieWxqOjnPi0zFODjrOyUHHOTlEdZz7uHvnqlY06UAdJTOb4e7ZUdch8aXjnPh0jJODjnNy0HFODo3xOGvIh4iIiIhIHShQi4iIiIjUgQL1t/dg1AVIg9BxTnw6xslBxzk56Dgnh0Z3nDWGWkRERESkDtRDLSIiIiJSBwrUe8jMxpjZV2a20Myuiboe2TNm1svM3jWz+WY218x+HrZ3MLM3zWxB+G/7mMdcGx7vr8zsxJj2A8zsi3DdXWZmUbwmqZqZpZrZp2b2SrisY5xgzKydmT1rZl+GP9MH6zgnHjP7Zfj/9Rwze8rMMnScmz4ze8TM1prZnJi2ejuuZtbczJ4J26eaWd94vh4F6j1gZqnAPcBJwDDgHDMbFm1VsodKgV+7+1DgIODK8BheA7zt7gOBt8NlwnVnA8OBMcC94ecA4D5gHDAwvI1pyBciu/VzYH7Mso5x4rkTeM3dhwD7EhxvHecEYmY9gauAbHcfAaQSHEcd56ZvPLseg/o8rpcAm9x9APB34C9xeyUoUO+p0cBCd1/s7sXA08DYiGuSPeDuue4+K7y/leAXcE+C4/hYuNljwPfC+2OBp929yN2XAAuB0WbWHWjj7h97cCLC4zGPkYiZWRbwHeDhmGYd4wRiZm2AI4B/Abh7sbtvRsc5ETUDWphZM6AlsAod5ybP3d8HNlZqrs/jGruvZ4Fj4/mthAL1nukJrIhZzgnbpAkKv/7ZD5gKdHX3XAhCN9Al3Ky6Y94zvF+5XRqHfwC/A8pj2nSME0t/YB3waDi052Eza4WOc0Jx95XA34DlQC6wxd3fQMc5UdXnca14jLuXAluAjvEqXIF6z1T1l42mSWmCzCwTeA74hbvn1bRpFW1eQ7tEzMxOAda6+8zaPqSKNh3jxq8ZsD9wn7vvB2wj/Hq4GjrOTVA4hnYs0A/oAbQysx/W9JAq2nScm75vc1wb9JgrUO+ZHKBXzHIWwVdP0oSYWRpBmP63uz8fNq8Jvzoi/Hdt2F7dMc8J71dul+gdCpxqZksJhmUdY2ZPomOcaHKAHHefGi4/SxCwdZwTy3HAEndf5+4lwPPAIeg4J6r6PK4VjwmHC7Vl1yEm9UaBes9MBwaaWT8zSycYIP9yxDXJHgjHT/0LmO/ud8Ssehm4MLx/IfBSTPvZ4dnC/QhOeJgWfhW11cwOCvd5QcxjJELufq27Z7l7X4Kf0Xfc/YfoGCcUd18NrDCzwWHTscA8dJwTzXLgIDNrGR6fYwnOfdFxTkz1eVxj9/UDgt8F8ftWwt1124MbcDLwNbAI+H3U9ei2x8fvMIKvfGYDn4W3kwnGVb0NLAj/7RDzmN+Hx/sr4KSY9mxgTrjubsILJenWeG7AUcAr4X0d4wS7ASOBGeHP84tAex3nxLsBNwFfhsfoCaC5jnPTvwFPEYyLLyHoTb6kPo8rkAH8h+AExmlA/3i+Hl0pUURERESkDjTkQ0RERESkDhSoRURERETqQIFaRERERKQOFKhFREREROpAgVpEREREpA4UqEVEImRm+VHXUJ3wct7Doq5DRKSx07R5IiIRMrN8d8+s5302c/fS+tyniIhUTz3UIiKNgJkdZWbvmdlEM/vazG4zs/PMbJqZfWFme4XbjTez+83sg3C7U8L2i8zsP2b2X+ANM2tlZo+Y2XQz+9TMxobbDQ/3+ZmZzTazgeG2/zOzz81sjpmdFW472cyyw/vnhHXMMbO/xNSdb2a3ho/9xMy6hu1nhNt+bmbvN/DbKSLSoBSoRUQaj32BnwN7A+cDg9x9NPAw8LOY7foCRwLfAe43s4yw/WDgQnc/huCqYu+4+yjgaOCvZtYKuBy4091HElxhLAcYA6xy933dfQTwWmxRZtYD+AtwDMHVCUeZ2ffC1a2AT9x9X+B94NKw/Q/AiWH7qXV7W0REGjcFahGRxmO6u+e6exHBZXTfCNu/IAjRO0x093J3XwAsBoaE7W+6+8bw/gnANWb2GTCZ4DK8vYGPgevM7Gqgj7tvD/d/nJn9xcwOd/ctleoaBUx293XhUJJ/A0eE64qBV8L7M2PqnAKMN7NLgdRv9W6IiDQRCtQiIo1HUcz98pjlcqBZzLrKJ7/sWN4W02bA6e4+Mrz1dvf57j6BoMd4O/C6mR3j7l8DBxAE6z+b2R8q7d9qqLnEvzkZp2xHne5+OXA90Av4zMw61rAPEZEmTYFaRKTpOcPMUsJx1f2Br6rY5nXgZ2ZmAGa2X/hvf2Cxu98FvAzsEw7pKHD3J4G/AftX2tdU4Egz62RmqcA5wHs1FWhme7n7VHf/A7CeIFiLiCSkZrvfREREGpmvCAJtV+Bydy8Mc3OsW4B/ALPDUL0UOAU4C/ihmZUAq4GbCYZ0/NXMyoES4IrYHbl7rpldC7xL0Fs9yd1f2k2NfzWzgeH2bwOff7uXKiLS+GnaPBGRJsTMxgOvuPuzUdciIiIBDfkQEREREakD9VCLiIiIiNSBeqhFREREROpAgVpEREREpA4UqEVERERE6kCBWkRERESkDhSoRURERETqQIFaRERERKQO/j+mPqp6r7MAqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12, 6))\n",
    "for e in eps:\n",
    "    ax.plot(range(len(regret_vec[e])), regret_vec[e])\n",
    "ax.set_ylabel('Total Regret')\n",
    "ax.set_xlabel('Impressions')\n",
    "ax.set_title('RNN: Total Regret of eps-greedy Exploration Strategies')\n",
    "ax.legend(eps)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:spyder-env-y]",
   "language": "python",
   "name": "conda-env-spyder-env-y-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}