CartPole_Approximate_Montecarlo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMD5cUgawEKASPFNPyqsCKJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ogunfool/Approximate-RL-Methods-for-CartPole-Environment-on-OpenAI-gym/blob/main/CartPole_Approximate_Montecarlo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "QS1lvLYKBadL"
      },
      "outputs": [],
      "source": [
        "# Monte Carlo Gradient Ascent (SGD)\n",
        "import gym\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.kernel_approximation import RBFSampler\n",
        "\n",
        "GAMMA = 0.99\n",
        "ALPHA = 0.1\n",
        "\n",
        "def epsilon_greedy(model, s, eps=0.1):\n",
        "  # we'll use epsilon-soft to ensure all states are visited\n",
        "  # what happens if you don't do this? i.e. eps=0\n",
        "  p = np.random.random()\n",
        "  if p < (1 - eps):\n",
        "    values = model.predict_all_actions(s)\n",
        "    return np.argmax(values)\n",
        "  else:\n",
        "    return model.env.action_space.sample()\n",
        "\n",
        "\n",
        "def gather_samples(env, n_episodes=10000):\n",
        "  samples = []\n",
        "  for _ in range(n_episodes):\n",
        "    s = env.reset()\n",
        "    done = False\n",
        "    truncated = False\n",
        "    while not (done or truncated):\n",
        "      a = env.action_space.sample()\n",
        "      sa = np.concatenate((s, [a]))\n",
        "      samples.append(sa)\n",
        "\n",
        "      s, r, done, truncated = env.step(a)\n",
        "  return samples\n",
        "\n",
        "def play_one_episode(env, model):\n",
        "  # Create lists for states, actions and rewards\n",
        "  states = []\n",
        "  rewards = [0]\n",
        "  actions = []\n",
        "  # Now play the game\n",
        "  s = env.reset()\n",
        "  episode_reward = 0\n",
        "  done = False\n",
        "  truncated = False\n",
        "  while not (done or truncated):\n",
        "    a = epsilon_greedy(model, s)\n",
        "    s2, r, done, truncated = env.step(a)\n",
        "    # Append to list\n",
        "    states.append(s)\n",
        "    rewards.append(r)\n",
        "    actions.append(a)\n",
        "\n",
        "    s =s2\n",
        "  # print(\"States:\", states, \"Rewards:\", rewards, \"Actions:\", actions)\n",
        "  return states, rewards, actions\n",
        "\n",
        "class Model:\n",
        "  def __init__(self, env):\n",
        "    # fit the featurizer to data\n",
        "    self.env = env\n",
        "    samples = gather_samples(env)\n",
        "    self.featurizer = RBFSampler()\n",
        "    self.featurizer.fit(samples)\n",
        "    dims = self.featurizer.n_components\n",
        "\n",
        "    # initialize linear model weights\n",
        "    self.w = np.zeros(dims)\n",
        "\n",
        "  def predict(self, s, a):\n",
        "    sa = np.concatenate((s, [a]))\n",
        "    x = self.featurizer.transform([sa])[0]\n",
        "    return x @ self.w\n",
        "\n",
        "  def predict_all_actions(self, s):\n",
        "    return [self.predict(s, a) for a in range(self.env.action_space.n)]\n",
        "\n",
        "  def transform(self, s, a):\n",
        "    sa = np.concatenate((s, [a]))\n",
        "    x = self.featurizer.transform([sa])[0]\n",
        "    return x\n",
        "\n",
        "\n",
        "def test_agent(model, env, n_episodes=20):\n",
        "  reward_per_episode = np.zeros(n_episodes)\n",
        "  for it in range(n_episodes):\n",
        "    done = False\n",
        "    truncated = False\n",
        "    episode_reward = 0\n",
        "    s = env.reset()\n",
        "    while not (done or truncated):\n",
        "      a = epsilon_greedy(model, s, eps=0)\n",
        "      s, r, done, truncated = env.step(a)\n",
        "      episode_reward += r\n",
        "    reward_per_episode[it] = episode_reward\n",
        "  return np.mean(reward_per_episode)\n",
        "\n",
        "\n",
        "def watch_agent(model, env, eps):\n",
        "  done = False\n",
        "  truncated = False\n",
        "  episode_reward = 0\n",
        "  s = env.reset()\n",
        "  while not (done or truncated):\n",
        "    a = epsilon_greedy(model, s, eps=eps)\n",
        "    s, r, done, truncated = env.step(a)\n",
        "    episode_reward += r\n",
        "  print(\"Episode reward:\", episode_reward)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# instantiate environment\n",
        "# env = gym.make(\"CartPole-v1\", render_mode=\"rgb_array\")\n",
        "env = gym.make(\"CartPole-v1\")\n",
        "model = Model(env)\n",
        "reward_per_episode = []\n",
        "\n",
        "# watch untrained agent\n",
        "watch_agent(model, env, eps=0)\n",
        "\n",
        "# repeat until convergence\n",
        "n_episodes = 5000\n",
        "for it in range(n_episodes):\n",
        "  # Collect states, rewards and actions for a single episode \n",
        "  states, rewards, actions = play_one_episode(env, model) \n",
        "\n",
        "  G = 0\n",
        "  T = len(states)\n",
        "  episode_reward = 0\n",
        "  for  t in range(T-2, -1, -1):\n",
        "    s = states[t]\n",
        "    r = rewards[t+1]\n",
        "    a = actions[t]\n",
        "    G = r + GAMMA*G # Smart Update rule (Did you see what I did there, HAHA)\n",
        "\n",
        "    # Let's update Q(s,a) using approximate function\n",
        "    # Constant learning rate since I don't use ample counts\n",
        "    # update the model\n",
        "    g = model.transform(s, a)\n",
        "    err = G - model.predict(s, a)\n",
        "    model.w += ALPHA * err * g\n",
        "    \n",
        "    # accumulate reward\n",
        "    episode_reward += r\n",
        "\n",
        "  if (it + 1) % 50 == 0:\n",
        "    print(f\"Episode: {it + 1}, Reward: {episode_reward}\")\n",
        "\n",
        "  # early exit\n",
        "  if it > 20 and np.mean(reward_per_episode[-20:]) == 200:\n",
        "    print(\"Early exit\")\n",
        "    break\n",
        "  \n",
        "  reward_per_episode.append(episode_reward)\n",
        "\n",
        "# test trained agent\n",
        "test_reward = test_agent(model, env)\n",
        "print(f\"Average test reward: {test_reward}\")\n",
        "\n",
        "plt.plot(reward_per_episode)\n",
        "plt.title(\"Reward per episode\")\n",
        "plt.show()\n",
        "\n",
        "# watch trained agent\n",
        "# env = gym.make(\"CartPole-v1\", render_mode=\"human\")\n",
        "watch_agent(model, env, eps=0)\n"
      ],
      "metadata": {
        "id": "pk_pS-4fwGKY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "624fcd5d-aaed-49d0-dcb6-b413abf80f3e"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.8/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode reward: 10.0\n",
            "Episode: 50, Reward: 12.0\n",
            "Episode: 100, Reward: 24.0\n",
            "Episode: 150, Reward: 31.0\n",
            "Episode: 200, Reward: 22.0\n",
            "Episode: 250, Reward: 27.0\n",
            "Episode: 300, Reward: 25.0\n",
            "Episode: 350, Reward: 28.0\n",
            "Episode: 400, Reward: 38.0\n",
            "Episode: 450, Reward: 66.0\n",
            "Episode: 500, Reward: 88.0\n",
            "Episode: 550, Reward: 26.0\n",
            "Episode: 600, Reward: 31.0\n",
            "Episode: 650, Reward: 31.0\n",
            "Episode: 700, Reward: 47.0\n",
            "Episode: 750, Reward: 40.0\n",
            "Episode: 800, Reward: 25.0\n",
            "Episode: 850, Reward: 34.0\n",
            "Episode: 900, Reward: 37.0\n",
            "Episode: 950, Reward: 15.0\n",
            "Episode: 1000, Reward: 15.0\n",
            "Episode: 1050, Reward: 13.0\n",
            "Episode: 1100, Reward: 17.0\n",
            "Episode: 1150, Reward: 18.0\n",
            "Episode: 1200, Reward: 23.0\n",
            "Episode: 1250, Reward: 20.0\n",
            "Episode: 1300, Reward: 20.0\n",
            "Episode: 1350, Reward: 67.0\n",
            "Episode: 1400, Reward: 46.0\n",
            "Episode: 1450, Reward: 60.0\n",
            "Episode: 1500, Reward: 29.0\n",
            "Episode: 1550, Reward: 102.0\n",
            "Episode: 1600, Reward: 65.0\n",
            "Episode: 1650, Reward: 81.0\n",
            "Episode: 1700, Reward: 79.0\n",
            "Episode: 1750, Reward: 85.0\n",
            "Episode: 1800, Reward: 83.0\n",
            "Episode: 1850, Reward: 89.0\n",
            "Episode: 1900, Reward: 74.0\n",
            "Episode: 1950, Reward: 93.0\n",
            "Episode: 2000, Reward: 79.0\n",
            "Episode: 2050, Reward: 84.0\n",
            "Episode: 2100, Reward: 78.0\n",
            "Episode: 2150, Reward: 107.0\n",
            "Episode: 2200, Reward: 78.0\n",
            "Episode: 2250, Reward: 85.0\n",
            "Episode: 2300, Reward: 91.0\n",
            "Episode: 2350, Reward: 81.0\n",
            "Episode: 2400, Reward: 93.0\n",
            "Episode: 2450, Reward: 100.0\n",
            "Episode: 2500, Reward: 93.0\n",
            "Episode: 2550, Reward: 108.0\n",
            "Episode: 2600, Reward: 87.0\n",
            "Episode: 2650, Reward: 78.0\n",
            "Episode: 2700, Reward: 93.0\n",
            "Episode: 2750, Reward: 89.0\n",
            "Episode: 2800, Reward: 93.0\n",
            "Episode: 2850, Reward: 86.0\n",
            "Episode: 2900, Reward: 79.0\n",
            "Episode: 2950, Reward: 81.0\n",
            "Episode: 3000, Reward: 86.0\n",
            "Episode: 3050, Reward: 68.0\n",
            "Episode: 3100, Reward: 79.0\n",
            "Episode: 3150, Reward: 73.0\n",
            "Episode: 3200, Reward: 82.0\n",
            "Episode: 3250, Reward: 84.0\n",
            "Episode: 3300, Reward: 108.0\n",
            "Episode: 3350, Reward: 107.0\n",
            "Episode: 3400, Reward: 83.0\n",
            "Episode: 3450, Reward: 93.0\n",
            "Episode: 3500, Reward: 147.0\n",
            "Episode: 3550, Reward: 102.0\n",
            "Episode: 3600, Reward: 121.0\n",
            "Episode: 3650, Reward: 49.0\n",
            "Episode: 3700, Reward: 91.0\n",
            "Episode: 3750, Reward: 99.0\n",
            "Episode: 3800, Reward: 93.0\n",
            "Episode: 3850, Reward: 98.0\n",
            "Episode: 3900, Reward: 92.0\n",
            "Episode: 3950, Reward: 90.0\n",
            "Episode: 4000, Reward: 30.0\n",
            "Episode: 4050, Reward: 27.0\n",
            "Episode: 4100, Reward: 18.0\n",
            "Episode: 4150, Reward: 23.0\n",
            "Episode: 4200, Reward: 28.0\n",
            "Episode: 4250, Reward: 24.0\n",
            "Episode: 4300, Reward: 11.0\n",
            "Episode: 4350, Reward: 19.0\n",
            "Episode: 4400, Reward: 34.0\n",
            "Episode: 4450, Reward: 24.0\n",
            "Episode: 4500, Reward: 63.0\n",
            "Episode: 4550, Reward: 74.0\n",
            "Episode: 4600, Reward: 115.0\n",
            "Episode: 4650, Reward: 146.0\n",
            "Episode: 4700, Reward: 163.0\n",
            "Episode: 4750, Reward: 207.0\n",
            "Episode: 4800, Reward: 55.0\n",
            "Episode: 4850, Reward: 24.0\n",
            "Episode: 4900, Reward: 37.0\n",
            "Episode: 4950, Reward: 68.0\n",
            "Episode: 5000, Reward: 139.0\n",
            "Average test reward: 138.3\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZgUxfnHPy+w3DcsiIAuCop4oiiKNygexB+axMQjCRoTE4+oMVHRxGgSD0w8k5h4R41BxRMjKiIiKKeAHHIvsNzHcizXsiy7W78/umfomZ17emZ6et7P8+yzPdU13VU91d9++62qt8QYg6IoiuIvGuS6AIqiKIr7qLgriqL4EBV3RVEUH6LiriiK4kNU3BVFUXyIiruiKIoPUXFXfIuIXCMiX+W6HMkiImeKyJIMHNeISE+3j6t4ExV3JSVEpExE9orIbhHZKCIvi0jLXJfLDxhjvjTGHJnrcij5jYq7kg6XGGNaAicAfYG7c1UQEWlUiOdWlGiouCtpY4zZCIzFEnkARORUEZkiIhUiMldEzrHTzxWR+Y5840Tka8fnL0XkUnt7uIgsF5FdIrJQRC5z5LtGRCaLyBMishW4X0Q6iMgHIrJTRGYAh0crs4iU2G6K60VkvYhsEJHfOvY3cJx/q4iMEpH2Yd+9TkRWA59HOcd3RGSOfQ2miMhxjn1lInK3Xa/tIvJvEWlq7ztHRNY68t4lIuvs67BERAbZ6U1E5Em7/Ovt7SaO791h12u9iPw0rGxNRORREVktIptE5BkRaRbteil5iDFG//Qv6T+gDDjP3u4GzAeesj93BbYCF2MZEOfbn4uBZkAV0BEoAjYB64BW9r69QAf7OJcDB9vH+CGwB+hi77sGqAF+BTSyv/sGMApoARxjH/erKOUvAQzwup3/WKDcUadbgWl23ZoAzwKvh333Vfu7zSIcvy+wGegPNASG2desieP6fQt0B9oDk4EH7H3nAGvt7SOBNcDBjnMfbm//yS5jJ/vaTgH+bO+70L62x9hlHGmXuae9/wngA/vcrYD/AQ/nul3pn4v3aK4LoH/5+WeL025gly0a44G29r67gP+E5R8LDLO3vwS+C5wKfGoL8oXAucC8GOecAwy1t68BVjv2NQT2A70daQ8lIO7O/H8BXrS3FwGDHPu62Mdv5PjuYTHK+q+A0DrSlgBnO67fLx37LgaW29tOce9pPyTOA4rCjrccuNjx+QKgzN5+CRjh2HdEQNwBwXpQHu7YfxqwMtftSv/c+1NfoZIOlxpjPhORs7Esw45ABXAocLmIXOLIWwRMsLcnYguYvb0dOBvYZ38GQER+AtyOJaYALe1zBFjj2C7GEl5n2qoE6hCe/1h7+1DgPRGpc+yvBTpH+W44hwLDRORXjrTGWG8i0c7t3AeAMaZURG4D7geOFpGxwO3GmPV2/lVRjnEwMCtsX4BioDkwS0QCaYL1gFR8gvrclbQxxkwEXgYetZPWYFnubR1/LYwxI+z9AXE/y96eiCXuZ9vbiMihwPPAzVhumrZYboygGmFZogHKsdw03R1phyRQ/PD86x11uCisDk2NMeuinD+cNcCDYd9vbox5PYFzh2CMGWmMOQPrgWGAR+xd6+20SMfYEOH4AbZgub+OdpStjbE6xxWfoOKuuMWTwPkicjzwGnCJiFwgIg1FpKndSdjNzjsFy5d8CjDDGLMAS6T6A5PsPC2whKwcQESuxfIfR8QYUwu8i9Wx2lxE+mD5ueNxr53/aOBa4E07/RngQfshg4gUi8jQxC4FYD2Yfiki/cWihYgMEZFWjjw3iUg3u6P2d45zBxGRI0VkoN1RWoUlyoG3ideB39tl6wj8Aevag+XqukZE+ohIc+C+wDGNMXV2+Z4QkU72ebqKyAVJ1E/xOCruiisYY8qxOhj/YIxZAwwF7sES5zXAHdjtzRizB5gNLDDGVNuHmAqsMsZstvMsBB6z0zdhuUsmxynGzVium41YbxL/TqDoE4FSrD6DR40xn9rpT2F1OH4qIruwOi77J3A87PLPBH4O/APL7VSK1U/gZCRWn8MKLP/5AxEO1QQYgWVtb8TqPA0MOX0AmAnMw+rQnh04hjHmY6wH7uf2ucNH9Nxlp08TkZ3AZ1gPXMUniDG6WIdSeIhICbASq5OyJgfnLwN+Zoz5LNvnVgoDtdwVRVF8iIq7oiiKD1G3jKIoig9Ry11RFMWHeGISU8eOHU1JSUmui6EoipJXzJo1a4sxpjjSPk+Ie0lJCTNnzsx1MRRFUfIKEYk6C1vdMoqiKD5ExV1RFMWHqLgriqL4EBV3RVEUH6LiriiK4kNU3BVFUXyIiruiKIoPUXFPk51V+/lgbsQ1FhRFUXKGJyYx5TN3vDWXsQs20fugVhzRuVX8LyiKomQBtdzTZMOOKgD2VtfmuCSKoigHUHFXFEXxISruiqIoPkTF3SU0Kr6iKF5CxT1NJNcFUBRFiYCKe5qoxa4oihdRcXcJteAVRfESKu4uoRa8oiheQsU9TdRiVxTFi6i4K4qi+BAV9zRJxR3zv7nrmbB4s+tlURRFCaCxZXLAr17/BoCyEUNyXBJFUfyKWu5poj53RVG8iIq7oiiKD1FxVxRF8SEq7i5hjI50VxTFO6i4K3HZVbWfm0fOZvue6lwXRVGUBFFxdwkR/3atjpy+mg/nbeBfE5fnuiiKoiSIiruiKIoPUXF3CfW5K4riJVTc08XH7phw9AGmKPmDirsSlwJ6fimKb1BxTxe1ZhVF8SAq7krC6HNMUfIHFfd0KQCfhWgEHUXJO1TclbgYXWdKUfIOFXclYQrgJUVRfIOKu5Iw6nNXlPwhYXEXkYYi8o2IfGh/7iEi00WkVETeFJHGdnoT+3Opvb8kM0VXsoX63BUl/0jGcr8VWOT4/AjwhDGmJ7AduM5Ovw7Ybqc/YedTFEVRskhC4i4i3YAhwAv2ZwEGAm/bWV4BLrW3h9qfsfcPEj9H1Sog1CujKPlDopb7k8CdQJ39uQNQYYypsT+vBbra212BNQD2/h12/hBE5HoRmSkiM8vLy1MsvpIN9NGsKPlHXHEXke8Am40xs9w8sTHmOWNMP2NMv+LiYjcPnRPUqlUUxUs0SiDP6cD/icjFQFOgNfAU0FZEGtnWeTdgnZ1/HdAdWCsijYA2wFbXS+4RCsGo1VEyipJ/xLXcjTF3G2O6GWNKgCuAz40xVwMTgO/b2YYBo+3tD+zP2Ps/Nz4OJ+jbiimKktekM879LuB2ESnF8qm/aKe/CHSw028HhqdXxPzAzxa8+twVJf9IxC0TxBjzBfCFvb0COCVCnirgchfKlleoBa8oipdIStyVA2zbU82Jfx6X62JkFf861xTFf2j4gRRZs60y10VQFEWJioq7oiiKD1FxVxRF8SEq7krCaFx3RckfVNwVRVF8iIp7ihTi2G8N/aso+YOKu0sUwjBBdcsoSv6g4q7ERSM2K0r+oeLuEqp/iqJ4CRV3RVEUH6Li7hIF4XMvgDoqil9QcVfioh4nRck/VNyVuKjBrij5h4q7oiiKD1FxT5FCmtBTODVVFP+g4q4oiuJDVNwVRVF8iIp7ihTSVHydoKUo+YevxX1fTS2Pjl1C1f7aLJytcMReURTv42txf3XKKv4xoZRnJ65w/diF1KGqKEr+4Wtx31djWezVtdmw3FXsFUXxDr4W9wA6bV5RlELD1+KuoWoVRSlUfC3umaT+c8P/rwdGX4EUJW8oCHFXSUoPff9RlPyjIMRdURSl0FBxdw21bxVF8Q4q7q7hf+eP/2uoKP5BxV2Ji446UpT8oyDEXQd5pIeOklGU/MPX4q4Gp6IohYqvxV0NTndQt4yi5B9xxV1EmorIDBGZKyILROSPdnoPEZkuIqUi8qaINLbTm9ifS+39JZmtgqIoSu5YXr7bk67LRCz3fcBAY8zxwAnAhSJyKvAI8IQxpiewHbjOzn8dsN1Of8LOlxPyyeA0xvDDZ6cybuGmXBdFUZQEmVm2jUGPTeS1aatyXZR6xBV3Y7Hb/lhk/xlgIPC2nf4KcKm9PdT+jL1/kOT4vT4fFtaoqTNMX7mNG16bleuiRMWDxomi5JRVWysBmL26IsclqU9CPncRaSgic4DNwDhgOVBhjKmxs6wFutrbXYE1APb+HUCHCMe8XkRmisjM8vLy9GoRrdxZnFjkZ+HLpzcgRckmRY0sCa2uqctxSeqTkLgbY2qNMScA3YBTgN7pntgY85wxpp8xpl9xcXG6h1MURck6DW3Lp86D1l1So2WMMRXABOA0oK2INLJ3dQPW2dvrgO4A9v42wFZXSpsqWbjublm33msiiqLkI4mMlikWkbb2djPgfGARlsh/3842DBhtb39gf8be/7nxYleyx1DPh6IobtIofha6AK+ISEOsh8EoY8yHIrIQeENEHgC+AV60878I/EdESoFtwBUZKHdyZEE503186dNPURQ3iSvuxph5QN8I6Suw/O/h6VXA5a6Uzi1UORVFyQBeHmzg6xmqXr7w4eRDUfNhSKmiKBa+FvcAmZCkfHpwpEsBVVVRfIOvxT0fRUn7nhVFcQNfi7vKpLvoc0dR8gdfi3smcVvoPK2bheSDUpQU8KLh42txz0dJ8mR4XS+2XEVRYuJrcQ+QCT92uAa7dQYv+9y9+NxRFCUyvhZ3FSN38fBzR1EyRsnwMfxm1NxcFyNpfC3u2cTXzxF9SioFzjuz1+a6CElTEOKeDxZnoIx5UFRFUWy8bPYUhLhngvBY8YUgyoVQR0XxCwUh7vnkVfBiUb1YJkXxEl4MzVEQ4p4PbpkAeVRURVE8jK/FPZPL7HnxSa0oihLA1+IeIBsyrK4LRSk8vOzy9bW4Z/LCu92hqm8CiqK4ia/FPZ987V5l084qVm+rzHUxFEVJkkSW2VMKmP4PjQ9u68NSUSLjxXvD15a7l/1hiqIomcTX4p6PeNECUBQl/ygIcc+EYGb6raB0825Kho9h4fqdmT1RUuiTR1HyhYIQ93wg/AE0dsFGAP43b30OSqMoSr6j4p4ihek+0U4MRQnFu/eEirtLFKbYK0p2GbtgIyXDx7B5V1Wui+J5VNxTpDBH4ugTTMktr01bBcCiDbtyXBLvUxDino3Zn4Up9opS2FRW1+S6CFHxtbhncrHp2/Nw2S1FUdzlD6MXADB/3Y4cl6Q+vhb3AJnwhy/aEDpEsRB87oVQRyU/8MpC8rv3WZb7rirvWfC+Fnf1lChK/lBX5w3BToVaD5bd1+LuvcudPPqAUgqB0XPWcdg9H7Fq656E8mfS5ZoKtR55k3Dia3H3A95rMoriPh/O2wAkPgrGK26ZAF586/C1uHvr2a4oSjQa2DdrPNH2msUeQC13JSoebBuKkjUa2KLtNQM40TcEL96/ccVdRLqLyAQRWSgiC0TkVju9vYiME5Fl9v92drqIyN9EpFRE5onIiZmuhKIo+U1A3OPNScmkO2bS0nK27t4XN9/+2jrGzNuAMYZTStoDcFy3NhkrV6okYrnXAL8xxvQBTgVuEpE+wHBgvDGmFzDe/gxwEdDL/rse+JfrpU4Qj77B5S1etE4UfxC4V3M16mR/bR0/eWkGP3pxRkh6pDb/ry+Wc9PI2Xzy7UbOOqIjAGf07JiNYiZFXHE3xmwwxsy2t3cBi4CuwFDgFTvbK8Cl9vZQ4FVjMQ1oKyJdXC+5x/BaB4+i5BNByz3ObZQpn3udfeLlm3eHpEcqzoYdVlybrXuqg+X1oiGZlM9dREqAvsB0oLMxZoO9ayPQ2d7uCqxxfG2tnRZ+rOtFZKaIzCwvL0+y2Iqi+Ilgh2qOxodFe6hEMtqcQh7YKx4cvpGwuItIS+Ad4DZjTMj0TGNdgaR+FWPMc8aYfsaYfsXFxcl81ZN4tRdfUfKBwP1TV5dYfq+8J+e95S4iRVjC/l9jzLt28qaAu8X+v9lOXwd0d3y9m52WM/LBZRLNYvFgm1EU1wmIY53H7tV4pQnct168TxMZLSPAi8AiY8zjjl0fAMPs7WHAaEf6T+xRM6cCOxzum6ySzQueqQeIt5q6UujU1RmOuW8sT08odfW4DRP0uQfI2L0ddmBnecLv8ZraugP7PWi6N0ogz+nAj4H5IjLHTrsHGAGMEpHrgFXAD+x9HwEXA6VAJXCtqyVWFCVnXPHcNHbvq+GvY5dw07k9XTuudy33A+UxJlTD7//fQho3tOxj70l7AuJujPmK6GUfFCG/AW5Ks1yu4q3mkr/kqrNL8Q4zyrZl5LjJTmLKRUusM4YGYV2n1bVWJ4EHDXedoep1PNhmFKVgcL5I1MXoPM3r0TKKoiiZQpIcCpkpKY113FguI7Xcs0w+DU+MOs42u8WIicfcoYqvSO5ezVZTDO1QjZ7Pi0rja3FXFMVfZFNEt+zeR41j4H2+We6JjJbJe/LZ4vRgm1GUjJGrezX8vLuq9tPvgc/4Qb9uwTSvjeSJh68t93hP0y+WbKbH3WPYWbU/OwVSlDxm8cad8TOlyAGfuzcIrIn6ybcbg2mBss1fV/86eNEF7Gtxj8ffPy/FGFi6MbHVXxSlkLnwyS8zduxcS2N4R27wYeNInmkPA527piJbxUqLghZ3L+EVi8Uv/Patubw9a22ui6EkS45dH+EGuNMVM7l0a9TveTHEiYq74kvenrWW3741N9fF8D3GGP4ztYwdlem5NnPt1QjX5sC4defyeQ1y/XqRJAUh7jqz0h30KhYGVftrufvd+VRUVsfNO2dNBfeOXsCd77jzIE24jWW4MQbDIdQ50ySqS8YYqKyuYdXWPZktWBL4WtwTfdC60U5U+BS/8Pastbw+YzWPfrokbt59NZb6bU/Xcs+x1z3a/eu03AWYtiK6a+anL3/N2X/9wtVypYOvxV1RlOQJyFku3MgJnzNDz4LAQyZw+JDhjwLz1u2I+D0DTFuRmbg7qaLijjvtJM/ccTFZu72STTurcl0MJcckt/pOeuc6MDolN5HDop03VNuFMfPcjV4+Z01FxtaNVXHHGy4VL/W2n/HIBPo/ND7XxVAKiESNo6x1vEY4zyHtm0fNnsrtO3dNBZc+PZm/jV+W/JcToCDEPRu66R1pVpQckOevroncv/e8N9/Vc66r2AvA0k2ZmWfjb3HP9fiqNPCSJa8UFindNS4113iH2bbHGsFTvmufOycMIyAZyXbwpjIir8Z2xzRqmBkZ9re4xyF/pd99Nu2sYvSc2Evd6vMm/9lfWxfTcPhw3np+//63CR/PrXtIElxmb95aq0Pz5SllLp2ZhM6bCWrtcZYNMyREBSHu0X43L2tVtmNV/OiF6dz6xhx276vJ6nmV7LGzaj+9fvcx//xiedQ8j3+6NGL6f6atylSxUsLte7fOtqIrq2uB5F/6U3k41NRaX2rYQC33pFHLPHE27rBGx+Rb5Dslcbbttlwao2auiZ4p7KaprK5h0tJy7o1izeeqtbjttpyyPPr49URIpTSbbddSUYZMd1+Le7wL7iXxDy9rtn3uu2yL3W/a/tOXv851ETxD4KeN1e6d+4yBu96Zz09empHBUoWSq+YXbtSkow23vvFNQvn+OtaaJNYwQ3ENfC3ufiDbfcIrt0SfPp2PYRw+X7w56e/U1NaxaEPmwttmmxXlu9mzryYoYLFcfs59SzftYlmckRzu+dyt/498spj5ayNPFHKSaSMk6cM7CjR6zvqkvtqySWaW1fC1uHvJMk+VbFvSyzfvzu4JPchfP13CRU99SalPrsXAxyby05e/DralRC33Wau2s2FH9MlsE5eWu1I+67zWmatr6rj0n5NdO27C509SLEo6HBjznq4BduUph6R3gCj4WtwDZGWcu8vnyFXw/1jVyHX8j2wxZ7UVHCpTw+1ywfSV2wj8usk0rR17o8eMGfbSDDI0uTLrhLftePfz2F+fZX0vxUVGtu4+0LaKGmmHatIk2oi96Gf24jj3PJ42kBL56IaKRUCIY7tlkjumW9co2fPmuuO/SaOGvHvjAL66a2BKJs9d78wLbjfM0I1VEGuoZoNMCV+hCaoXCF5zH2i700hIzC2TtLrnBLdP2zjMek7koXXiIe0O5E+yQHv21Qa3M3WP+9pyzyc8aKhz/B8/zXURckKDwISaHJfDDZxuk4BgNXDRcnfLLZNrG6Zbu2YJ520S9iBIxYXq/Eqm6l7Q4u7mEzNT4pxt0XdaeuH+1lzfgNkiuFBDChd/zbZKT3XEOn/PG1+bDbjc7tN4BO6vreNHL0znm9Xbkz+vyzdGvd86xuEjXb9kr0OIuGfIdC8QcY8fzjMX1NTWxc+UZWJ2qBaIugdcE8bAcfeP5YlxkWdtRuLMv0zgvMcnZqpoSeP8PVfYw1yra+uoqzNU7a8NyTvq6zVJdyIHLPdURH55+W6+Kt3CXe/Mi9q2Nu+qYtTX9SdduX3vJnO8+kvywStTkpvB63R/qVsmBQIX8MtlW3JWhsrqGu55bz67quqPOrjgyUlxv591QfWDLyJNnCMgdlbV8FSGQrJmg0iitaJ8D0fe+zG97/2E0s3WOPb1FXu58515bN0Tf2m90OMn1mCMMTw4ZmHIPIrAEnax3ES//M8s7nxnHuvtCIqZIilxj5CWbNgOZ5XbNCtK6ruJ4mtxD7B2e+SGkQ3h/PfkMkZOX82zE1fU27e83DvrLQaIZYHl61DIr5Zt4cInJ1Fdk9ybUqQHcr4R7ffcb8c1Oe/xSYyes44VKbbFA5200dvGV8u2MHt1Bc9/uZLrX50JwLuz13L1C9MAWLW1koVRJo1V2Mv37Q17y3Cb8OuUjI2Tjo4c2bkVRRmKCqmjZTJMwLKJ+9oaZXf2fe7R9+WrW+ae9+azelsl6yv2UtKxRdz8AR/obW/MCaY9N2k5hxe3ZNBRnTNWToAdlftp09w9Sy6R9nOro55JH5/Y7btqfy0/enE6rexZmIE1SW8fdWBB7b37a5lcGjm2SyM77sr+MBem27dF+HWK1d/SrW1o52vgQZkMgVjuSzIUyx0KxHIH+Gi+u8tjJUvVfu/51yPhR6+M083y7bod9YQiwPLy3eyo3M+OSss1UeMYCvLQR4u57pWZMc8zd01FwmV6dOwSZqwMXXPzw3nrOf5PnzIniePEYummXRmP8hnv4RG4hoHYRcm+IQRcNnVhP1kmO1Q/XbAxZj/LQ989Nu3zpfqmlAy+FnenNXHjf2fnpAyBEScvfrUype9n21rOZ8s92g0fEIjlm3fznb9/xcMfLY6Yb9BjE7nsX5OZGyO2SV2UsX+zV29n6NOJT5v/x4RSfvDs1JC0yaVW39CC9fFjq8TDGMPgJybR74HP0j5WLMLX/5y3toIHxyzk3Ee/iPp7XPnctISPv2prJeDuhLLqmjrenb02dA6AY//UFVv5MMZaqa2bpvdmla0JinHFXUReEpHNIvKtI629iIwTkWX2/3Z2uojI30SkVETmiciJmSx8PLwwNToQHzpfiH0TuavuG3ak30m2t7qWRz5ZzM0jZ9Pj7o8i5gmU+t7RVhN+aXL0B208i+qNCCM3ILbV/tbMNUxZvoUV5bvZVxPaHkZOXx2hvAeu85bd+3h83NKoD5WNUWK/ZKvtByzer8us4Yz/94/JPP/lSlZu2cPUFVsjCtnUFYmH1w342uu7TVIsMPDkZ0u5fdRcPl24KZjmLOe+mrqY923jRundB84BHpccf3Bax4pFIj73l4F/AK860oYD440xI0RkuP35LuAioJf91x/4l/0/N2TRYR1NFL1u7YYTq+c+lbq8+fVq1ldU8evzjwhJn7VqO9/71xSe+OHxXNa3W/IHtnl1ahn/Clt84uGPFoVmssvtDIJljGHq8q18unATpx7WnnbNGyd0vm176g8VnLVqG3/838Ko37nj7QNTzVs0bsiCP10Y/HzPe/O5qn/9wFEvfrWSujrDtBVbGb94M40aCDecc3hI59vv35/Pa9NW8+6NA0JmS05ZviXkcyZxiux3/v5lyL6rnp/O74cclfQxIzWz+sPQU7+3A3HUnfM4nMeP9MB1kmhbCWft9kpq60zIeW8d1CulYyVCXHE3xkwSkZKw5KHAOfb2K8AXWOI+FHjVWI/BaSLSVkS6GGNy4vCeHubTjEYmX5Nemxa7ocQj2x2qnVo1jZtn6+59nPTAZ4z8WX8G9OwYM+9d71iLCoeL+/Jya6LP5NKt9cT9wicncc6RnRh+Ue+4ZakJM+FGTl/Ns5NCRyZFGmo3auaaYNmSWbIt0u+xZGPik5b2VNfW87WHU1tXx58/tB4W/Xu0B+DxcUvZU13DHYOPRETYu7822LbmralgycZdCNC6WRE3/nd2RkUjtKwHLsi36+qPeHlgzKJ6aakQLubpjNwK/IYVldW8O3st3z2xW8KPimsGlNChZZOUznvGIxMA+NuVfYNpB7eNf7+lSqqjZTo7BHsjEBhC0BVwvreutdNyIu6x/GZewytBqmKNEgjcTnPXWi6I579cEVfcoxEIlrR6ayX7a+tCLNLFG3exeOMurji5O93bN09qMYNIK9RHmjEaEPZkeWzcUm445/DgosZ79tWwaWd918jijTtp26xxvanqQD1f+/0fLOA3g48Iis69oxcE9zl/jWcnrggOqe19UKsD34/w1pCtsfnZCuDlpuX+zuy1gNVJDnBySfuobq9wkglTEA2nMRlrjH+6pD0U0hhjRCTpKy0i1wPXAxxyiLvxjCcs2czphycuOrkKr+tky+7IMwP92qEaEOwZZdsY9NhEHvvB8fQsbkm7Fgdeec959AtuGdiT2wcfGUybuLSc/j3a07SooXuFSZKJS8tZva2S2asr+N/cyAszXPjklxHTI/HylDJemVoW8dpHs/IXb8zcELpkcPqt3SJSEwyknXtkMROWlDPkWPd81ftq6hJ+VHhBKxIl1dEym0SkC4D9P7DczTqguyNfNzutHsaY54wx/Ywx/YqLi1MsRn2mr9jKtf/+msc+XZLwd7wQXveipxIXg0ySqWuxt7qW5yYtD77GN3BY46u3VXL5M1O56oXp/Duss/Nvn5eyvmIvldU1fLtuB8NemsGfPlzI3upaamrrctKnUWfgj/9bGFXYU8EDTTAlxmTg7ThSGwy8IUxYYi0QEh7FMV2yGULYeSovWu4fAMOAEfb/0Y70m0XkDayO1GBjbtgAABhySURBVB3Z9rdvt8col23N8uzPNNtGKhMhMkHssTKpN8THxy3h+S9XsnNvDT89owe3vF5/nclFG3ZG7JgcMOLzkM/LN+/mqD98wsDenTjp0Ox0HDoZ7ojFrbhPRMs9fLSMq8OBTFYneDhdShlaPtU6drwMIvI6MBU4UkTWish1WKJ+vogsA86zPwN8BKwASoHngRszUuoEWOahyHz5REyfe1hDDM+5s2o/943+Nhh86rdvHZiFOGmpNfzrHxNKOfHP49Iq4zf2SkmprI/qBsnGX/ELLRpnxxVmjPWgD13H1jDi48WOT+6qcaJHO65bG1fPm0k3TyKjZa6MsmtQhLwGuCndQrlBIjPAUrVEU531N3FpOcMSXEn+kwUbAdi4I7tLvaXydmqMCRlj/srUVSx78CLenrU2mObmNOtqxwxTL7jUCoVZ957PiI8XJzW6KFXC3ZTGwDMTl4d8dpNE3TInl7RP+1zOUyUzWCBZfD1DNVGSbSfz1qY2Pfz9byJ2P0QkMKzMjdmKyZDIaJlwwocjAvT63cculSg2j36aeDheJT2aFjXkujN6pHWMVBeDDm9itS6rezZthGxNMPOhuCfxJEzxoXnV89PrpSXye6VyumyPiojpcw97hQzEQKnxSH+Bknm6t2+e1vedvvKubRMfVhj+huamGBsT26iZPHygeyfD7f6C6PhQ3BMn8EaUqZ7ysi1hrqE8GEWVSMMbOd2ayhAIx1oTHtVJ8RRuxAu/ZkBJ0t9pHCGU7SEdDjwcOrVOfDJQeKtMxx03sHenemkPxphslcxDKMDrM6JPXrwzSx3yvhP3ZPonnCvuZIKrXwi18JMZ9nR897ZuFycqOyoPTIeON2pn255qPlsUOrbZL5b7Mz/KaSikjFA2Yghz7xuc1HfG/fqsemnJ+obvu6QPU+4eyN+v7BucyDXkuC788uzDue08a/ZsOqsfpWP8htfE4P4AjFEzI8cgyia+E/dkaGDXPlPiHh4kKqnbI4tOwA/mHRivHc8KjzTSJVYgrnzghZ/0o2zEEM7vcxC/PPvwjJzj6INbZ+S4sejZqWXS3ykbMYRenQ/Mfu3QIvk4Kgv+eAHXnt6Dji2bcMnxB/PxrWcC8J1ju9CwgXDOkZblnIz1HZ43PBplMmRjboQX+vl9J+7J/G7BWNFRfom5ayp4ekJpQseKdIgtu0OHzCXTqHLVNmJZ4ZHKv7e6lr9/ntg1yjW3DupF2Ygh3HdJn2DaCd3bcl4fK3pGwwYSEs/m+rMOc+W8PTq24MNfnRFx38qHL3blHJGIZIGP/Hl/xtxSvyz/vPpEvvjtOfXSbzgn+Yddiyahg/AOK25J2YghXHRsF+DAPVpnYM4fzufJH54Q95huumWy4R/1wigu34l7KkQT96FPT+avYxOf6RqPZIZert5W6dp5kyHaQhbRGBnDt5gIT/zw+IjxVzKJ08Xwzg0Doua744Ij+U1YwLNE+Zk9qqRPl9ZM+O05Ecczf3zrmYgIo35xWkrnuP6sw7jujB5R3woinXPA4R05+uD6Y7UH9u6U0CpVsZh//+CoDzEnTqOqbfPGXNq3a9zvhLs403LLJHAbntkrtZhJASKNIAsnUp+Em/humb1kJgUEAkpl6xmbjOVeUZmb9TtjNcpID6d0e/6HHt+VC44+iDMfmRB1ctB1Z/RIarGTIzq3ZOmm+j7Ujq2sDjxnLaK5kq84uTtFDRvwq0G9eCzKqjw9OrYILvh80qHtWFG+m6l3D2Jn1X7mrtnBC1+tpEub+lH/5t43mJraumB0wZNLIs+yfXFYP/ZU1wZn8/556NGcdnjH4DEDFnLV/lp63/tJSP2fuqJv/QPGILw/6OmrTqS4VZOEh/02bCC0alrEMV3jT/I5xB5x8+NTD02qjE7SGQQR/pMPfqL+QvWPXX48pzw0PuVzLFgfeU1YJ09fndk+noK23APxvScs3syzE5fHzFtTW8fd785jTRoWdbSFHr5cVh7yeUV5lmfXOm6UmhiWe6SHU7SAZ04u69uVf197cvD7z/74pOC+Bg2E5o0bMeve8ykbMYRJd5xb7/v3fqcP439zdvDz547tcBoI9Szhj289k6euOIGr7THWpzmCykUyBspGDGHE946LW68JDjfGOzcM4Js/DKZpUUM6tWoafGg4Jei9Gwfw2e1n0aZZUUjYWBFhyQMXcr/DXQQw6KjOFDvy/fi0Enp2akmLJo1CXB9NixpSNmIITYus2/nN60/jqC6h1vwP+nWLOeIl/CE35LgunNLjwISdeFr61+/Hv14B2jQvomzEEK5Iccw7ZN5y79Q6c6F4A3ROYrRQKvjPck/hO69OXQXAL2J0ps1atZ3XZ6zh9RmRBTrV6dCbdlbx4xdDZ60OfGxiSsdKlT2OVWdijZaJdG3DY6eH87uLj+Lntu/6oNZN2bCjKqZ1d0iH5hzVpTWLNuzkpWv6BWcaH17ckpIOzSnbWomI8MClx9ChRWNuCFs+MZJYH9WldYjYtbc7CY/onFiHY89OLSndvJvLT+rGW45ZtwDT7h4U7Jh3cnixdWznsLu+MRbQaNKoIUOOO7he+N5Au+rfI/7MyMCbVaRRWX/5/vERv9OhRWO27qlOe6Zksu68dEnnjTGVmem3ndeL01MMbx2NTAYNAx+Kezzmrqlg6NOTeffG6L7WSDRIsvEnYtECjMtAyNRkcboO3B6z/nNHp+SvBvbinvfmxx2BMfqm06kzhqZFDRnoWK8jcDsL8KMor/RHdWkV9+Zt36IxD152TMTxzpH46JYzqTOGCYs3B8V9wOEdADgogtsFoKRjC+bdP5hWTRK/xZydcI//ILIYxyKgFZLA+/iHvzqDdRV7Oeqg1kxbuTWvQtlCdqM4Atx2Xmp9L7HI9CUvOHEPuEA+iyOqpZsPzAw1xrC+IvH1PiuraxJemDj3feqhk1xiWu4JNsbzjurEZ4vqB/W6qv8hwSXlRt90OsWtIr+WRgvnGriho5Xj+O5tefmakxMq49X9E/f3BsrTrZ3lK77n4t5cf1b8USTJLqQcMEa7tWvGd0+0Vqc6oXtbjj64Nb9LYLk6Cfsfi2O6tgm+QTknFoUz9ISuvPH1Gq49vSTi/o4tmyRsyLhJOjNl47Xjn5+ZXoiFRFHL3WUCK+jEGic7s2wb33/mwGo5xsCtb8xJ+BxuLoqdyuy4ZDDGhFyLaWGLFzeQA6KTqHX3wrCTKd+1j6KG0fOnM0krmmV+Skk72rVoHLJGpZsc260N439zNoelOaokGp1bN+Hmc3vy3RMPjB5p3rgRY245M6HvB34fNw2G4lZN+Oz26H0cA3sXM2rm2qj7M0WjGG0rGnV1hne/WcfH326Mme+8ozrH3O8WarknSbwLFljerabORB1V4RR2SOxmcb4lJvPGOH5R7DeIi489KPGDpcAx940N8bk7V2YHOKJzq5Ti20SzytPh6C5tWLNtL82ihJ5NZEhdugR86ZlARPjtBUfGz6gkPUnonVlr+Y0jBLUXUMvdZQLX05jEL+4Nr81K6hzJdK5+saQ85v5Muxb3xHnLcA6NjHe1bhnYk25pBpaKxeM/PJ6fre8R9cERGL+dZ+5j13hxWD9emVpGy8bZua1zeZ2TnST0QRKrZmWi/2HK8i310jK5UAf4UNwT/V2SEeBk14l0NWKde4dKiZDFpSNc28CalkDIWqeZoHnjRvRLIp62CMz6/fkZLJG36H9YB/of1iFr58vlJMxMBlZMJWxDPCKtPJbpTuyCG+ce9Eu63DhWOca/e2DmcWaIUK+AZf/wd4/NcmGiE7hlmhc1DA57VNyneZZWZopEMvdYXZ1h4tLYb8hOMtFmIk0OzLRbxnfiHm8YXGCvMcbVWOn3vv9tcNvNhQS89KCIVJSAjz7VRRiU/KV7u8y54OKRzJt3LpbcvPDo0L6ySAM4Mu2W8Z24xyPwsAyfjOIm2QrGn23ypV75NmY7Xxl+ce/4mTJEMk0xmebw8rWJDaWNh3NimzGGXVX1l+ZUyz1Z4lyvwG43hyuGs9TF9ULdXgg4HSYsyc2C1Io3OffIxCaBZYQkXmmTkdDmGeiMnr16e8T0TL+V+65Dde6a2IGOKmKMgV5RvptFG9IT5gfHLOT5L92Lb55I3OqZZdto1bSIIw9qFTevk/lrk1ufdXkCi44rSjZIVBcrKqv5aH7sce1OTjr0QIiIEd89lnYu+N+rayKX1u11YMPxnbg/+dmylPenG9NlRfluV4Ud4ocF3bp7X3BcftmIIRHzVNfUUVFZHQyGVLW/lj37anh8XPxwxtFmmypKLkk0/MAtb8xhUhKdqc4YO+kENmvk8MtEe/uuzfDylP5zy8Rg0Yb4YTjT4WevznT9mPEM95MSCHNw59tzOeWh8eyvrWPumgoGPTaRkx74LDiEMRYXHdMl0aIqStZI1OjdvLMqswUJ40Z7cZNJjkiv0TQ807HWCkrcV4YvWO0yKzLgtohkodTU1vHKlDKqa0JbR/hnYwwjp6/m/TnWBI7aOsPQpyezLsE4Ob0PauUhj39khp5wcK6LoOSA16atomT4GEqGjwGsFcGcHf51dYZXp5ZlPVrlnRdancwVlfspGT6GUx8az49enB4xb6cMzOJ24ju3TCyyHUkuWYYc14Ux8zaEpDmjNNbWGR7+aBGNGzXgn18sZ+/+0E7hHXv388JXKxh4ZCf++cVyaurqmFx6IFbM9srIi2FEY/HGXZ5YLiwWT/7wBEbPSXz2oeIPdjpGn5Tv2sfJD1pvsA9ddiyV1TU8MGZR0sf8nh2szU02RnlzWPHQxUlHmk0WX4n7nW/Hjh1x88j6s8Sywcif9Wfv/lp2Vu3n9J4dOeXByCu8DD3+4HriPn/dAVfS7NXbecGxItHusOFVyzbv4tmJK3h2YuQY66c9/HnSZfe2tOuwR4WgsAPc8978lI/zWAphllMl08IOPnPLhEenG5RgvO5UmHvf4ITy/eS0QxnQsyODjurMZX270alV9BVe9jncKkPsxYTnrqlg4tJybn9zDpeHBTQLf+W86vnIr3/p4HXLXVG8wlu/jL0W7hkuL/YRD1+Jezj3XXI0v85AkP2GDSQkBnosIsXhDixo/N2+Xbmq/yEc1Lop8+8fHBJq1Lm+4rCXZvDuN+vqHSfeKkjpclnfrp6aIasomaJV0/SdGCfHiXvUx77vO7bMTkgMX4v7IR2ac+t5vVw/7tIHLgJg8vCBMfOVjRhCk0b1428EpiZ3aduUhy47lmn3DKJV0yKaNbbWwow2pDHbtG/ROCG3zOA+2Yl/rSgL/3SBK8f5y/eOo2PLxpxvt92RPzvVlePGommURWgyha987k4W/enCtI/x7I9P4qb/zg4J+rPkgQuDY2G7tm3GJ7edyfvfrOeZOAtsOxl0VGceG7fU88MMmxY1iGi53zKoF38bf2C+wLEJrHivKMky6Y5zOeuvE4KfVz58cVp9LM/9+CTOOqIYsBYV/8HJ3amrM+yprqFVkqtmpUSw7NnpJ/KNuH80/0BH5Pl9Oocs6NCogUSMyhagbMQQnhi3lKfGL+PWQb3YXlnNq1NX0bd7W7644xy27K7mhCgrB/U+qDXDL2odFPdv7j2fK5+fFuJWCafPwa09Y51H47DiFlzV/1BWb7WiXT502bF0a9eMvoe0pVXTIjq3bsLv3rOCpV3er3sui6r4lEM6NHf1Phl8dP2Fbxo0kOwIO45lELM0BiDvxX3llj28N3stf/u8NJj29yv7huT56Rk9eM72Tw84vANTlocuJQfQpMh6ZaqurePe7/Th2tN7BGd0dksg+t2s358HQLsWjfnktrNSq0yOeOKHx7O7qoau7Zrx05dnMvve84NhT7u2bcZnt5/F4cUtQ6ymq/sfGhT3aItEK/5nYO/OjJq5lmO7pr5sIljT/metOhCD5fsnuTsssUsW22jPTi1D10GwCS5gnqVy5LW4v//NOm57M3Rt01G/OI2mRaF+7rsv6s3t5x9BAxEeHLOQKcu38r+bz8BggsMJD21vrYvZpU1Tiho2oEeS62R2aJnZCQnJcMvAnpR0bMHto6yhoZf17UoDEd6ZXT8S5nM/PinEoolkKfXsFDlmzTM/OolenTO37JziTd69cQD79lsjtS485iCWPHBhxL6lZBj1i9MY/MRETj2sA/ddcjSNYgwVfP4n/fh5krPB375hQFrli8X7N53OXkcgwpE/789fPlnC22GRZ4/qYnWo3pGlpRTFC0Pd+vXrZ2bOTH7q/i//M4tPFoQGBYr3GrevppZZq7Yz4PDQYUnGGL5ctoUzenbMyhjURLj/gwW8PKUs+HnlwxfT4+6PuOiYgxh6QlfaNCuiuFUTmjRqQPf2zRkzbwMHtWkaEvwoQOnmXZz3+CR+cdZhPDtpBdcMKGHQUZ04s1dxFmuUGUqGj6Fd8yK++YM1PLWuzvCL12Zx7YASBmR5+JmSHdZX7OX5L1fwddk23r/xdBqI8FXpFs7s1dEzcx9mr97Ouu17ef+bdWzZU83om053/RwiMssY0y/ivkyIu4hcCDwFNAReMMaMiJU/VXEHuOOtucHY7ON+fRa9OicXGdHrGGNYuGEnh7RvTqumRWzeWUWb5kUpWUrrKvZycJumbNxZRXHLJjSKE5QsX9i+p5qiRg1o2SSvX0QVJWmyKu4i0hBYCpwPrAW+Bq40xiyM9p10xF1RFKVQiSXumTDdTgFKjTErjDHVwBvA0AycR1EURYlCJsS9K7DG8XmtnRaCiFwvIjNFZGZ5eeLxlhVFUZT45Mzpaox5zhjTzxjTr7g4/zv1FEVRvEQmxH0d4JzV0s1OUxRFUbJEJsT9a6CXiPQQkcbAFcAHGTiPoiiKEgXXx44ZY2pE5GZgLNZQyJeMMQvcPo+iKIoSnYwMDDbGfAR8lIljK4qiKPHxxywWRVEUJQRPhB8QkXJgVYpf7whscbE4+YDWuTDQOhcG6dT5UGNMxOGGnhD3dBCRmdFmaPkVrXNhoHUuDDJVZ3XLKIqi+BAVd0VRFB/iB3F/LtcFyAFa58JA61wYZKTOee9zVxRFUerjB8tdURRFCUPFXVEUxYfktbiLyIUiskRESkVkeK7Lkw4i8pKIbBaRbx1p7UVknIgss/+3s9NFRP5m13ueiJzo+M4wO/8yERmWi7okgoh0F5EJIrJQRBaIyK12up/r3FREZojIXLvOf7TTe4jIdLtub9oxmRCRJvbnUnt/ieNYd9vpS0TkgtzUKHFEpKGIfCMiH9qffV1nESkTkfkiMkdEZtpp2W3bxpi8/MOKW7McOAxoDMwF+uS6XGnU5yzgROBbR9pfgOH29nDgEXv7YuBjrIXUTwWm2+ntgRX2/3b2drtc1y1KfbsAJ9rbrbBW7+rj8zoL0NLeLgKm23UZBVxhpz8D3GBv3wg8Y29fAbxpb/ex23sToId9HzTMdf3i1P12YCTwof3Z13UGyoCOYWlZbds5vwhpXLzTgLGOz3cDd+e6XGnWqSRM3JcAXeztLsASe/tZrKULQ/IBVwLPOtJD8nn5DxiNtTRjQdQZaA7MBvpjzU5sZKcH2zVW8L3T7O1Gdj4Jb+vOfF78wwr7PR4YCHxo18HvdY4k7llt2/nslkloxac8p7MxZoO9vRHobG9Hq3teXhP71bsvliXr6zrb7ok5wGZgHJYFWmGMqbGzOMsfrJu9fwfQgTyrM/AkcCdQZ3/ugP/rbIBPRWSWiFxvp2W1bety8XmCMcaIiO/GrYpIS+Ad4DZjzE4RCe7zY52NMbXACSLSFngP6J3jImUUEfkOsNkYM0tEzsl1ebLIGcaYdSLSCRgnIoudO7PRtvPZci+EFZ82iUgXAPv/Zjs9Wt3z6pqISBGWsP/XGPOunezrOgcwxlQAE7BcEm1FJGBoOcsfrJu9vw2wlfyq8+nA/4lIGfAGlmvmKfxdZ4wx6+z/m7Ee4qeQ5badz+JeCCs+fQAEesiHYfmlA+k/sXvZTwV22K97Y4HBItLO7okfbKd5DrFM9BeBRcaYxx27/FznYttiR0SaYfUxLMIS+e/b2cLrHLgW3wc+N5bz9QPgCntkSQ+gFzAjO7VIDmPM3caYbsaYEqx79HNjzNX4uM4i0kJEWgW2sdrkt2S7bee64yHNTouLsUZZLAd+l+vypFmX14ENwH4s39p1WL7G8cAy4DOgvZ1XgKftes8H+jmO81Og1P67Ntf1ilHfM7D8kvOAOfbfxT6v83HAN3advwX+YKcfhiVUpcBbQBM7van9udTef5jjWL+zr8US4KJc1y3B+p/DgdEyvq2zXbe59t+CgDZlu21r+AFFURQfks9uGUVRFCUKKu6Koig+RMVdURTFh6i4K4qi+BAVd0VRFB+i4q4oiuJDVNwVRVF8yP8DllMujXCpwh0AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode reward: 142.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Some tests.."
      ],
      "metadata": {
        "id": "n_MZPaEwiCoT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "a = model.predict(s,a)"
      ],
      "metadata": {
        "id": "xnQ6XUc9HS1e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def play_one_episode(env, model):\n",
        "  # Create lists for states, actions and rewards\n",
        "  states = []\n",
        "  rewards = [0]\n",
        "  actions = []\n",
        "  # Now play the game\n",
        "  s = env.reset()\n",
        "  episode_reward = 0\n",
        "  done = False\n",
        "  truncated = False\n",
        "  while not (done or truncated):\n",
        "    a = epsilon_greedy(model, s)\n",
        "    s2, r, done, truncated = env.step(a)\n",
        "    # Append to list\n",
        "    states.append(s)\n",
        "    rewards.append(r)\n",
        "    actions.append(a)\n",
        "\n",
        "    s =s2\n",
        "  print(\"States:\", states, \"Rewards:\", rewards, \"Actions:\", actions)\n",
        "  return states, rewards, actions\n"
      ],
      "metadata": {
        "id": "o-lvyUTrR5-4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "states, rewards, actions = play_one_episode(env, model)\n",
        "len(states)"
      ],
      "metadata": {
        "id": "lAwHYlydT7Gu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}