predibase_lorax.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "095c2c45",
   "metadata": {},
   "source": [
    "## Lesson 7 - LoRAX\n",
    "\n",
    "In this lesson, we explore how to efficiently Serve LLMs using LoRAX!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c1b1de77",
   "metadata": {
    "height": 183
   },
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import json\n",
    "import time\n",
    "from typing import List\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "from pydantic import BaseModel, constr\n",
    "\n",
    "from lorax import AsyncClient, Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4686ef7f",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "from utils import endpoint_url, headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aeceb612",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "client = Client(endpoint_url, headers=headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a838efbc",
   "metadata": {},
   "source": [
    "## Prefill vs Decode (KV Cache)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e6100dc",
   "metadata": {
    "height": 132
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Deep learning is a subset of machine learning that uses artificial neural networks to learn from data. It is a powerful tool that can be used to solve complex\n",
      "\n",
      "\n",
      "----------\n",
      "Request duration (s): 0.7536032199859619\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "resp = client.generate(\"What is deep learning?\", max_new_tokens=32)\n",
    "duration_s = time.time() - t0\n",
    "\n",
    "print(resp.generated_text)\n",
    "print(\"\\n\\n----------\")\n",
    "print(\"Request duration (s):\", duration_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa960b0d",
   "metadata": {
    "height": 236
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Deep learning is a subset of machine learning that uses artificial neural networks to learn from data. It is a powerful tool that can be used to solve complex\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.32323384284973145\n",
      "Throughput (tok / s): 63.82977603712064\n"
     ]
    }
   ],
   "source": [
    "durations_s = []\n",
    "\n",
    "t0 = time.time()\n",
    "for resp in client.generate_stream(\"What is deep learning?\", max_new_tokens=32):\n",
    "    durations_s.append(time.time() - t0)\n",
    "    if not resp.token.special:\n",
    "        print(resp.token.text, sep=\"\", end=\"\", flush=True)\n",
    "    t0 = time.time()\n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", durations_s[0])\n",
    "print(\"Throughput (tok / s):\", (len(durations_s) - 1) / sum(durations_s[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc2ec81c",
   "metadata": {
    "height": 47
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQUklEQVR4nO3de1xc9Z0//teZYS7AwAAhMEAIl1zNDRQSRI3pNlRiu9bUtptYdxOzrrZe+tWitsauiTbdJV7qN7WmZmvX1Xa9pPZX7dafm2rRpLWSxJCkuV8lIeEaSJiBAWaGmfP9YzgHSLjM5cycYeb1fDzmAQyH4cxkwrzm83l/3h9BFEURRERERBFMo/YJEBEREY2HgYWIiIgiHgMLERERRTwGFiIiIop4DCxEREQU8RhYiIiIKOIxsBAREVHEY2AhIiKiiBen9gkowePxoKmpCUlJSRAEQe3TISIiIh+Iooiuri5kZ2dDoxl7DCUqAktTUxNyc3PVPg0iIiIKwLlz5zBlypQxj4mKwJKUlATAe4eTk5NVPhsiIiLyhc1mQ25urvw6PpaoCCzSNFBycjIDCxER0QTjSzkHi26JiIgo4jGwEBERUcRjYCEiIqKIx8BCREREEY+BhYiIiCIeAwsRERFFPAYWIiIiingMLERERBTxGFiIiIgo4jGwEBERUcRjYCEiIqKIx8BCREREEY+BZQx9Ljf+/f2jePydg+h3e9Q+HSIiopjFwDIGQQB+8efP8cauBtidbrVPh4iIKGYxsIzBEKeFTuvd8tru6Ff5bIiIiGIXA8s4Eg1xAIBuBhYiIiLVMLCMw8TAQkREpDoGlnFIgYVTQkREROphYBmHPMLSx8BCRESkFgaWcbCGhYiISH0MLONgDQsREZH6GFjGwRoWIiIi9TGwjGNwSoiN44iIiNTCwDIOk1EKLC6Vz4SIiCh2MbCMw2TQAgDsHGEhIiJSDQPLOLhKiIiISH0MLONgHxYiIiL1MbCMQ14l5GRgISIiUgsDyzg4wkJERKQ+BpZxsIaFiIhIfQws42DjOCIiIvUxsIxD6sNid7rh8Ygqnw0REVFsYmAZhzTCArDwloiISC0BBZbNmzcjPz8fRqMRZWVl2L1796jH/u53v0NpaSlSUlKQmJiI4uJi/PrXvx52jCiKWLduHbKyshAfH4+KigqcPHkykFNTnCFOgziNAIB1LERERGrxO7Bs3boVVVVVWL9+Pfbu3YuioiJUVlaira1txOPT0tLwwx/+ELW1tThw4ADWrFmDNWvW4I9//KN8zDPPPIMXXngBW7Zswa5du5CYmIjKykr09fUFfs8UIgiCXHjLOhYiIiJ1CKIo+lWYUVZWhoULF+LFF18EAHg8HuTm5uK73/0uHnvsMZ9u45prrsFXvvIVbNiwAaIoIjs7Gw8//DAeeeQRAIDVakVmZiZeffVVrFy5ctzbs9lsMJvNsFqtSE5O9ufu+OT6jR+hsbMX795/PYpzUxS/fSIioljkz+u3XyMsTqcTdXV1qKioGLwBjQYVFRWora0d9+dFUURNTQ2OHz+OG2+8EQBQX1+PlpaWYbdpNptRVlbm022GA3uxEBERqStu/EMGtbe3w+12IzMzc9j1mZmZOHbs2Kg/Z7VakZOTA4fDAa1Wi5///Of40pe+BABoaWmRb+Py25S+dzmHwwGHwyF/bbPZ/Lkbfksc2ACRNSxERETq8CuwBCopKQn79+9Hd3c3ampqUFVVhcLCQnzhC18I6Paqq6vx1FNPKXuSYzAZdQAYWIiIiNTi15RQeno6tFotWltbh13f2toKi8Uy+i/RaDB9+nQUFxfj4Ycfxje+8Q1UV1cDgPxz/tzm2rVrYbVa5cu5c+f8uRt+Mw2MsLDoloiISB1+BRa9Xo+SkhLU1NTI13k8HtTU1KC8vNzn2/F4PPKUTkFBASwWy7DbtNls2LVr16i3aTAYkJycPOwSSol6tucnIiJSk99TQlVVVVi9ejVKS0uxaNEibNq0CXa7HWvWrAEArFq1Cjk5OfIISnV1NUpLSzFt2jQ4HA68//77+PWvf42XXnoJgHfZ8EMPPYQf//jHmDFjBgoKCvDEE08gOzsby5cvV+6eBkHqdsvAQkREpA6/A8uKFStw4cIFrFu3Di0tLSguLsa2bdvkotmGhgZoNIMDN3a7Hffddx/Onz+P+Ph4zJ49G//93/+NFStWyMd8//vfh91uxz333IPOzk7ccMMN2LZtG4xGowJ3MXjcT4iIiEhdfvdhiUSh7sOyZcdpbPzfY7jt6hw8v6JY8dsnIiKKRSHrwxKr5D4sHGEhIiJSBQOLD+QpIW5+SEREpAoGFh+w0y0REZG6GFh8kMgpISIiIlUxsPhgcJWQW+UzISIiik0MLD5gHxYiIiJ1MbD4QNr80O7sh8cz4VeBExERTTgMLD5IMng3PxRFoMfFaSEiIqJwY2DxgVGngUbwfs5ut0REROHHwOIDQRC4UoiIiEhFDCw+SmIvFiIiItUwsPgokRsgEhERqYaBxUfS0uYuBhYiIqKwY2DxkYkjLERERKphYPFRop6BhYiISC0MLD7ilBAREZF6GFh8xCkhIiIi9TCw+MjEZc1ERESqYWDx0WDjOLbmJyIiCjcGFh+ZpA0QOSVEREQUdgwsPpKKbtman4iIKPwYWHwkLWtmYCEiIgo/BhYfcYSFiIhIPQwsPuKyZiIiIvUwsPhocJUQAwsREVG4MbD4KGnICIsoiiqfDRERUWxhYPGRNMLiEYFeF3uxEBERhRMDi48S9FoIgvdzTgsRERGFFwOLjwRBgEnP9vxERERqYGDxQ6Jcx8IpISIionBiYPGD1Iuly+FS+UyIiIhiCwOLHzjCQkREpA4GFj9wA0QiIiJ1MLD4Qep228XAQkREFFYMLH5IZHt+IiIiVTCw+EHqdstlzUREROHFwOIH7idERESkDgYWP3BKiIiISB0MLH5IMnKEhYiISA0MLH5I1DOwEBERqYGBxQ8mjrAQERGpIqDAsnnzZuTn58NoNKKsrAy7d+8e9diXX34ZixcvRmpqKlJTU1FRUXHF8XfeeScEQRh2WbZsWSCnFlIm1rAQERGpwu/AsnXrVlRVVWH9+vXYu3cvioqKUFlZiba2thGP3759O26//XZ8/PHHqK2tRW5uLm666SY0NjYOO27ZsmVobm6WL2+++WZg9yiE2JqfiIhIHX4Hlueffx5333031qxZgzlz5mDLli1ISEjAK6+8MuLxr7/+Ou677z4UFxdj9uzZ+OUvfwmPx4OampphxxkMBlgsFvmSmpoa2D0KIbnTbR83PyQiIgonvwKL0+lEXV0dKioqBm9Ao0FFRQVqa2t9uo2enh64XC6kpaUNu3779u3IyMjArFmzcO+996Kjo8OfUwsLeUrI6YYoiiqfDRERUeyI8+fg9vZ2uN1uZGZmDrs+MzMTx44d8+k2fvCDHyA7O3tY6Fm2bBluu+02FBQU4PTp03j88cdx8803o7a2Flqt9orbcDgccDgc8tc2m82fuxEwqejW7RHR5/IgXn/luREREZHy/Aoswdq4cSPeeustbN++HUajUb5+5cqV8ufz58/HggULMG3aNGzfvh1Lly694naqq6vx1FNPheWch0rQDQaUbkc/AwsREVGY+DUllJ6eDq1Wi9bW1mHXt7a2wmKxjPmzzz33HDZu3IgPPvgACxYsGPPYwsJCpKen49SpUyN+f+3atbBarfLl3Llz/tyNgGk0AhIHQgpXChEREYWPX4FFr9ejpKRkWMGsVEBbXl4+6s8988wz2LBhA7Zt24bS0tJxf8/58+fR0dGBrKysEb9vMBiQnJw87BIu7MVCREQUfn6vEqqqqsLLL7+M1157DUePHsW9994Lu92ONWvWAABWrVqFtWvXysc//fTTeOKJJ/DKK68gPz8fLS0taGlpQXd3NwCgu7sbjz76KHbu3IkzZ86gpqYGt956K6ZPn47KykqF7qZyuAEiERFR+Pldw7JixQpcuHAB69atQ0tLC4qLi7Ft2za5ELehoQEazWAOeumll+B0OvGNb3xj2O2sX78eTz75JLRaLQ4cOIDXXnsNnZ2dyM7Oxk033YQNGzbAYDAEefeUl8TmcURERGEniFGwPtdms8FsNsNqtYZ8euhbL+/Ep6c78NOVxbi1OCekv4uIiCia+fP6zb2E/MQpISIiovBjYPGTNCXU3cfAQkREFC4MLH5KZA0LERFR2DGw+GlwWTM3QCQiIgoXBhY/meQaFm6ASEREFC4MLH4a7HTLERYiIqJwYWDxk8moAwB0sYaFiIgobBhY/GQycC8hIiKicGNg8RNXCREREYUfA4ufpKLbLvZhISIiChsGFj9JgcXuZGAhIiIKFwYWP8l9WPr6EQXbMBEREU0IDCx+kmpY+j0iHP0elc+GiIgoNjCw+ClRHyd/zsJbIiKi8GBg8ZNWIyBhoHkcd2wmIiIKDwaWACTK7fkZWIiIiMKBgSUASYbBwlsiIiIKPQaWACRyaTMREVFYMbAEINEg1bBwA0QiIqJwYGAJgMng3QCRU0JEREThwcASAG6ASEREFF4MLAGQu90ysBAREYUFA0sAuKyZiIgovBhYAmAa6HbLKSEiIqLwYGAJgDQl1MXAQkREFBYMLAGQ+7AwsBAREYUFA0sAkhhYiIiIwoqBJQDSCEsX+7AQERGFBQNLANian4iIKLwYWAKQZOTmh0REROHEwBKAwaJb7iVEREQUDgwsATANBBan2wNHP0MLERFRqDGwBCBRr5U/5ygLERFR6DGwBCBOq4FR533ouLSZiIgo9BhYAmQy6ABwaTMREVE4MLAEyGTwTgtxaTMREVHoMbAESNpPiDs2ExERhR4DS4AS9ezFQkREFC4MLAEycT8hIiKisGFgCRCnhIiIiMKHgSVAUrdbBhYiIqLQCyiwbN68Gfn5+TAajSgrK8Pu3btHPfbll1/G4sWLkZqaitTUVFRUVFxxvCiKWLduHbKyshAfH4+KigqcPHkykFMLG04JERERhY/fgWXr1q2oqqrC+vXrsXfvXhQVFaGyshJtbW0jHr99+3bcfvvt+Pjjj1FbW4vc3FzcdNNNaGxslI955pln8MILL2DLli3YtWsXEhMTUVlZib6+vsDvWYiZOMJCREQUNoIoiqI/P1BWVoaFCxfixRdfBAB4PB7k5ubiu9/9Lh577LFxf97tdiM1NRUvvvgiVq1aBVEUkZ2djYcffhiPPPIIAMBqtSIzMxOvvvoqVq5cOe5t2mw2mM1mWK1WJCcn+3N3Avafn9Rjw3tHcEtRNn52+9Vh+Z1ERETRxJ/Xb79GWJxOJ+rq6lBRUTF4AxoNKioqUFtb69Nt9PT0wOVyIS0tDQBQX1+PlpaWYbdpNptRVlbm822qIUkaYelzqXwmRERE0S/On4Pb29vhdruRmZk57PrMzEwcO3bMp9v4wQ9+gOzsbDmgtLS0yLdx+W1K37ucw+GAw+GQv7bZbD7fB6UkyjUs3PyQiIgo1MK6Smjjxo1466238M4778BoNAZ8O9XV1TCbzfIlNzdXwbP0TeJAa37WsBAREYWeX4ElPT0dWq0Wra2tw65vbW2FxWIZ82efe+45bNy4ER988AEWLFggXy/9nD+3uXbtWlitVvly7tw5f+6GIpLYh4WIiChs/Aoser0eJSUlqKmpka/zeDyoqalBeXn5qD/3zDPPYMOGDdi2bRtKS0uHfa+goAAWi2XYbdpsNuzatWvU2zQYDEhOTh52CbdELmsmIiIKG79qWACgqqoKq1evRmlpKRYtWoRNmzbBbrdjzZo1AIBVq1YhJycH1dXVAICnn34a69atwxtvvIH8/Hy5LsVkMsFkMkEQBDz00EP48Y9/jBkzZqCgoABPPPEEsrOzsXz5cuXuqcK4rJmIiCh8/A4sK1aswIULF7Bu3Tq0tLSguLgY27Ztk4tmGxoaoNEMDty89NJLcDqd+MY3vjHsdtavX48nn3wSAPD9738fdrsd99xzDzo7O3HDDTdg27ZtQdW5hJoUWBz9HrjcHui0bBpMREQUKn73YYlEavRhcbk9mPHD/wUA7F/3JaQk6MPye4mIiKJFyPqw0CCdVgNDnPfh6+rjtBAREVEoMbAEQd5PyMnAQkREFEoMLEEwGblSiIiIKBwYWIKQqPcGFk4JERERhRYDSxBMbM9PREQUFgwsQTDJ3W65ASIREVEoMbAEIVFuHscRFiIiolBiYAmCie35iYiIwoKBJQgm7thMREQUFgwsQUjkfkJERERhwcASBHkDRC5rJiIiCikGliCwhoWIiCg8GFiCMLismYGFiIgolBhYgsAaFiIiovBgYAkCp4SIiIjCg4ElCCaOsBAREYUFA0sQGFiIiIjCg4ElCFINS5/Lg363R+WzISIiil4MLEFIHOh0C3DHZiIiolBiYAmCIU4Lvdb7EHY7OS1EREQUKgwsQZJ6sXClEBERUegwsARJmhbqYnt+IiKikGFgCVKiniMsREREocbAEqQktucnIiIKOQaWILE9PxERUegxsASJ7fmJiIhCj4ElSHK3WxbdEhERhQwDS5DkKSH2YSEiIgoZBpYgcYSFiIgo9BhYgsQaFiIiotBjYAmSSV7WzL2EiIiIQoWBJUiDy5pdKp8JERFR9GJgCZJpoDU/d2smIiIKHQaWIJkMOgBsHEdERBRKDCxBkjY/ZGAhIiIKHQaWICUNjLBwlRAREVHoMLAESRph6XG64faIKp8NERFRdGJgCZK0SggA7Ox2S0REFBIMLEEyxGmg0woA2O2WiIgoVBhYgiQIgjzKwjoWIiKi0GBgUYC8nxADCxERUUgEFFg2b96M/Px8GI1GlJWVYffu3aMee/jwYXz9619Hfn4+BEHApk2brjjmySefhCAIwy6zZ88O5NRUwcBCREQUWn4Hlq1bt6Kqqgrr16/H3r17UVRUhMrKSrS1tY14fE9PDwoLC7Fx40ZYLJZRb3fu3Llobm6WL5988om/p6YaTgkRERGFlt+B5fnnn8fdd9+NNWvWYM6cOdiyZQsSEhLwyiuvjHj8woUL8eyzz2LlypUwGAyj3m5cXBwsFot8SU9P9/fUVDM4wsL2/ERERKHgV2BxOp2oq6tDRUXF4A1oNKioqEBtbW1QJ3Ly5ElkZ2ejsLAQd9xxBxoaGoK6vXCSA0sfN0AkIiIKBb8CS3t7O9xuNzIzM4ddn5mZiZaWloBPoqysDK+++iq2bduGl156CfX19Vi8eDG6urpGPN7hcMBmsw27qElqHmd3coSFiIgoFOLGPyT0br75ZvnzBQsWoKysDHl5efjNb36Du+6664rjq6ur8dRTT4XzFMckbYDYxT4sREREIeHXCEt6ejq0Wi1aW1uHXd/a2jpmQa2/UlJSMHPmTJw6dWrE769duxZWq1W+nDt3TrHfHQiTNMLColsiIqKQ8Cuw6PV6lJSUoKamRr7O4/GgpqYG5eXlip1Ud3c3Tp8+jaysrBG/bzAYkJycPOyiJpORq4SIiIhCye8poaqqKqxevRqlpaVYtGgRNm3aBLvdjjVr1gAAVq1ahZycHFRXVwPwFuoeOXJE/ryxsRH79++HyWTC9OnTAQCPPPIIbrnlFuTl5aGpqQnr16+HVqvF7bffrtT9DClpWXMXAwsREVFI+B1YVqxYgQsXLmDdunVoaWlBcXExtm3bJhfiNjQ0QKMZHLhpamrC1VdfLX/93HPP4bnnnsOSJUuwfft2AMD58+dx++23o6OjA5MnT8YNN9yAnTt3YvLkyUHevfAwsQ8LERFRSAmiKIpqn0SwbDYbzGYzrFarKtNDNUdbcddre7Bgihn/88ANYf/9REREE5E/r9/cS0gBiWzNT0REFFIMLArglBAREVFoMbAoYLDTLQMLERFRKDCwKEDe/NDphscz4UuCiIiIIg4DiwKSjIOLrexOjrIQEREpjYFFAYY4DbQaAQBg547NREREimNgUYAgCIN1LCy8JSIiUhwDi0IYWIiIiEKHgUUhidwAkYiIKGQYWBQijbB0cWkzERGR4hhYFJLI5nFEREQhw8CiEGlpM5c1ExERKY+BRSGJek4JERERhQoDi0I4JURERBQ6DCwKkaeEGFiIiIgUx8CiEGmEpYuBhYiISHEMLAoxcUqIiIgoZBhYFMJOt0RERKHDwKKQRDmwcPNDIiIipTGwKIRTQkRERKHDwKIQeUqIfViIiIgUx8CiEBOXNRMREYUMA4tCpN2au539EEVR5bMhIiKKLgwsCpGmhEQR6HGy8JaIiEhJDCwKiddpoRG8n3NaiIiISFkMLAoRBIHdbomIiEKEgUVBXNpMREQUGgwsCuLSZiIiotBgYFFQItvzExERhQQDi4KSpF4sTgYWIiIiJTGwKChRzykhIiKiUGBgURA3QCQiIgoNBhYFSVNC3Q6XymdCREQUXRhYFCS157dzhIWIiEhRDCwKMhl0ALhKiIiISGkMLAoySRsgsuiWiIhIUQwsCpKKbrmsmYiISFkMLAoysXEcERFRSDCwKIit+YmIiEKDgUVBJiM3PyQiIgoFBhYFSTUsXQwsREREigoosGzevBn5+fkwGo0oKyvD7t27Rz328OHD+PrXv478/HwIgoBNmzYFfZuRSpoSsjv6IYqiymdDREQUPfwOLFu3bkVVVRXWr1+PvXv3oqioCJWVlWhraxvx+J6eHhQWFmLjxo2wWCyK3GakkgKLRwT6XB6Vz4aIiCh6+B1Ynn/+edx9991Ys2YN5syZgy1btiAhIQGvvPLKiMcvXLgQzz77LFauXAmDwaDIbUaqBL0WguD9vIvt+YmIiBTjV2BxOp2oq6tDRUXF4A1oNKioqEBtbW1AJxCK21SLIAgw6aVpIbbnJyIiUopfgaW9vR1utxuZmZnDrs/MzERLS0tAJxDIbTocDthstmGXSJHIpc1ERESKm5CrhKqrq2E2m+VLbm6u2qckkzZAZPM4IiIi5fgVWNLT06HVatHa2jrs+tbW1lELakNxm2vXroXVapUv586dC+h3h4LJ6N0Akb1YiIiIlONXYNHr9SgpKUFNTY18ncfjQU1NDcrLywM6gUBu02AwIDk5edglUpg4wkJERKS4OH9/oKqqCqtXr0ZpaSkWLVqETZs2wW63Y82aNQCAVatWIScnB9XV1QC8RbVHjhyRP29sbMT+/fthMpkwffp0n25zIuF+QkRERMrzO7CsWLECFy5cwLp169DS0oLi4mJs27ZNLpptaGiARjM4cNPU1ISrr75a/vq5557Dc889hyVLlmD79u0+3eZEksjAQkREpDhBjIKWrDabDWazGVarVfXpoXW/P4Rf1Z7Fd784HQ/fNEvVcyEiIopk/rx+T8hVQpGMU0JERETKY2BRGPuwEBERKY+BRWHyBohOBhYiIiKlMLAoTAosXRxhISIiUgwDi8KkKSE2jiMiIlIOA4vCkozc/JCIiEhpDCwKYx8WIiIi5TGwKIyt+YmIiJTHwKIwk2Fw88Mo6MlHREQUERhYFJY4MMLS7xHh6PeofDZERETRgYFFYYn6we2ZOC1ERESkDAYWhWk0AhL1A3Us7MVCRESkCAaWEOBKISIiImUxsISAycjmcUREREpiYAkB7thMRESkLAaWEGBgISIiUhYDSwiwhoWIiEhZDCwhYOIGiERERIpiYAmBwSkhboBIRESkBAaWEJCnhNiHhYiISBEMLCGQxGXNREREimJgCQG50y0DCwWpsbMX5y72qH0aEev8pR6UbPgQ//7+UbVPhYhCjIElBLhKiJTg6Hfj1hf/iq+++Al6nHwujaTmaBs67E78bu957o5OFOUYWEKAU0KkhOMtXWjvduBSjwtHmmxqn05EOtRoBQC0dzvR1uVQ+WyIKJQYWEKAIyykhAPnrfLnBxutYxwZuw4NCXKHm/gYEUUzBpYQYKdbUsKhISHlUCNHWC7X53LjZGuX/DUfI6LoxsASAgwspISDwwILRw8ud7ylC/2ewboVjrAQRTcGlhBIZKdbCpKj340TQ0YPTrZ1odfJRoRDHRoIKFLN2GHW+RBFNQaWEDAN/AF1uUU4+vkiQ/473tIFl1tESoIO6SYDPCJwrIUvyENJU0BfLcoGAJy/1IvOHqeap0REIcTAEgKJ+jj5c3a7pUBI00Hzc8yYl5MMgNNCl5OmgK6blo7ctHgA4GoqoijGwBICWo2AhIHmcXbuJ0QBkMLJvBwz5ueYB67ji7HE5fbgWLN3ymxeTjLmZXsfI04LEUUvBpYQkepYuhwulc+EJqKhIyxzB16MubR50MnWbjjdHiQZ4zA1LQFzswdGoVh4q6j//KQe/7HjtNqnQQQAiBv/EAqEyRCHC10OjrCQ3xz9bhxv8Y4ezM8xQ6MRAAAnWrvg6HfDEKdV8/QigjwClW2GIAhyqOMIi3LabH3Y8N4RAMCyeRbkTUpU+Ywo1nGEJURMXCk0rv/74QmUbPgQ9e12tU8logwtuJ2SGo9ssxGpCTr0e0ScaOlW+/QigjSSItX3SCMsn1/o5jYGCqk7e0n+fNfnF1U8EyIvBpYQSTR43wV3MbCMSBRFvLG7AR12J94/2Kz26USUodNBgiBAEATMy+G00FBDa3wAICPZiMlJ3tVUR5u7xvpR8tHQwLKzvkPFM4lOl+xOfHaGQdAfDCwhwhGWsTVc7MGFgb1f+J92uMtfjId+zhoNwO0RcaTZO/UjTQV5P/eOshzhY6SIPRxhCalHf3sA39xSi+3H29Q+lQmDgSVEGFjGtrt+8A9g3dlLcHu4065k6AiLRFoFw6XN3mmfPpcHCXotCtIH6yqkwMI6luD1udzDOgc3dvbi/KUeFc8oujj7Pfjk1AUAwB8Pt6p8NhMHA0uIyKuE2IdlRHvODL576+rrH9bVNZZdXnArkT4/1twFl9ujyrlFCmmUaU5WMrQDBckAuLRZQQcbrXC5RWQkGVCcmwKAoyxKOtRkRZ/L+//4zycuQBT5hs0XDCwhInW75QjLyD476/3jJ/Wr2cNpIQDAiZZuuNwizPHegltJblo8koxxcLo9ONka24W3Uj+aoVNmwOD0kLdoObZDXbCkNxQleakoK0wDAOxiHYtiPhsywtzY2YszHRy98gUDS4gkG3UAgNaBOg0a1N7twOcXvCuDVizMBQB8NmTEJZZdXnArEQSB00IDDo5Q4wMw1ClJKrgtyUvFtQWTAAC76vmmQimX/7375OQFlc5kYmFgCZGrp6YAAGpPt8PD+oxhpHdvszKTUHFV5sB1/GMIDAksU8xXfE+6LpYLbz0eUW6/Ly1plgiCgDlZUh1L7D5GwRJFEXsbBgNLaX4qNAJwtqMHLdY+lc9u4vN4ROwZGGG+aY7379+fT7areUoTRkCBZfPmzcjPz4fRaERZWRl279495vFvv/02Zs+eDaPRiPnz5+P9998f9v0777xTXr4pXZYtWxbIqUWM0rw0JOi1aO92yisayEsKJwsLUlGcmwKtRkCTtQ+Nnb0qn5n6DjZ2AhhevyKRikpjeWnz2Ys96Hb0wxCnwfTJpiu+L426sI4lcPXtdly0O2GI02ButhlJRp083cZpoeCdutCNzh4X4nVafOcL0wAAtac7OI3pA78Dy9atW1FVVYX169dj7969KCoqQmVlJdraRl6a9emnn+L222/HXXfdhX379mH58uVYvnw5Dh06NOy4ZcuWobm5Wb68+eabgd2jCKGP0+C6ad6h1B0nONw3lLSMeWF+GhINcfILcayPsoxWcCuRrjvabEN/jP5xk6bDZmclI0575Z+vwZVCsRvqgiVNBxVNSYE+zvsYlxV461h2svA2aNIKyaunpqB4SgpSE3TodvRj/7lOdU9sAvA7sDz//PO4++67sWbNGsyZMwdbtmxBQkICXnnllRGP/+lPf4ply5bh0UcfxVVXXYUNGzbgmmuuwYsvvjjsOIPBAIvFIl9SU1MDu0cR5MaZkwF4q8DJq8fZj0MD735L871/BEvzvB/3xHgdy2gFt5L8SYlI1GvR5/Lg8xjtDix3uM1OHvH70kjAkSYbp2IDJAWWa/IG/waXFUp1LBxhCdbQN2wajYDrp6cDAP7C14lx+RVYnE4n6urqUFFRMXgDGg0qKipQW1s74s/U1tYOOx4AKisrrzh++/btyMjIwKxZs3Dvvfeio2Pi/8dYMhBY6s5eQjdXCwEA9jV0wu0RkZMSj5wU74vywnzvH8ZYbyA3WsGtRKMZ3DPn4PnYHEE4PMoKIcm0yYkwxGlgd7pxpiM2Q12whhbcShblp0EQgM8v2NHWxTqWYEgrhBYNjFrdOGPgjS3rWMblV2Bpb2+H2+1GZmbmsOszMzPR0tIy4s+0tLSMe/yyZcvwq1/9CjU1NXj66aexY8cO3HzzzXC7R9440OFwwGazDbtEorxJiciblIB+j4hPT/HJCAyGktL8wT+GJQOfH2/tgrU3dne3Hm31y1Cx3PFWFEX5fo80ZQYAcVoNZmexgVygOnucONnmXWE1NLCYE3SYbfE+rru5WihgjZ29aLL2IU4jyAszbpjhHWE5cL4TnT1OFc8u8kXEKqGVK1fiq1/9KubPn4/ly5fjvffew2effYbt27ePeHx1dTXMZrN8yc3NDe8J+0EaZfkzl60BGJz2kaaDACAjyYi8SQkQRcirE2LRoRE63F5OWhkTi0ubz1/qRWePCzqtgBmZVxbcStjxNnD7GjoBAIXpiUhL1A/7nlTHwgZygZNGV+bmmJGg9/bqyk6Jx/QMEzwi8OnpiT+zEEp+BZb09HRotVq0tg5vJdza2gqLxTLiz1gsFr+OB4DCwkKkp6fj1KlTI35/7dq1sFqt8uXcuXP+3I2wkob7drCbIfrdHjmQLBoSWIChdSyx+cfQ2e8Zs+BWMn/IKphYq9GQCmlnZibBEKcd9bjBjrexF+qCNdJ0kORaNpAL2m6pfuWyx3fxwCjLX/jGdkx+BRa9Xo+SkhLU1NTI13k8HtTU1KC8vHzEnykvLx92PAB8+OGHox4PAOfPn0dHRweysrJG/L7BYEBycvKwS6QqnzYJOq2AcxfZzfBIsw09TjeSjXGYkTH8HbJUxxKrhbcnWrvgdHtgjtchN+3KgltJ4WQTjDoNepxu1MdYjYbc4TZ79EAHDB9hifU3Cf6S+oOMFFgWDTSQO9HajYt2Tl0EQhphWVgw/A2bXMdyop3P2TH4PSVUVVWFl19+Ga+99hqOHj2Ke++9F3a7HWvWrAEArFq1CmvXrpWPf/DBB7Ft2zb85Cc/wbFjx/Dkk09iz549eOCBBwAA3d3dePTRR7Fz506cOXMGNTU1uPXWWzF9+nRUVlYqdDfVk2iIk0cPYn210GdDpoM0muFFpdIU0f5znXD2x96S3fEKbiVazWBztFibFpJXCOWM/QZlliUJWo2Ai3YnWmwsEPWVy+3B3855H+ORAktaoh4zB6bidnOUxW+X7IP1QQsvG2EuK0yDTiuwTf84/A4sK1aswHPPPYd169ahuLgY+/fvx7Zt2+TC2oaGBjQ3N8vHX3fddXjjjTfwi1/8AkVFRfjtb3+Ld999F/PmzQMAaLVaHDhwAF/96lcxc+ZM3HXXXSgpKcFf/vIXGAwGhe6muqTlzbHej0V+d3HZf1bAu7ojNUEHR78nJgtKD5wfv+BWIk0LxVJgEUVRvr9zx3mMjDqt3FROWlVE4zvabEOvyw1zvA7TRmjKBwBlA6Ms7Mfivz0D023TM0xX1Acl6Aff2HJaaHRxgfzQAw88II+QXG6kQtlvfvOb+OY3vzni8fHx8fjjH/8YyGlMGDfOTMfT27zdDB397jHn36OVKA62o16Yf+W7N0EQUJqfhg+PtGLPmYu4ZurE78PjD18KbiXSC3Ysdbxt63KgvdsJjQBcZRl/CnhuTjKOt3bhUJMVFXMyxz2ehvRfmZpyxQiopKwwDb/eeZb7CgVgaP+VkSyemY7azzvw5xPtWFWeH8YzmzgiYpVQtJuTlYzJSQb0utyoi9EajTMdPWjvdkIfpxlxnxxgaD+W2HqMfC24lciFt42xU3grBboZGUmI148f+OdmDxYnk2+kwFI6ygsqMNg75FiLDdae2G1BEIjd9aO/YQMG61hqT7ezTf8oGFjCQBAEuQo8VqeFpOmg4ikpo44wlQxZKRRLhWe+FtxKpmeYoI/ToMvRj3OXYmO++6A8HeRbgb1UeHuEgcVngyMso49uZiQZUTg5EaI4uOKFxtfj7JdD92gjLHOykpGWqIfd6ZaXl9NwDCxhsiTG61hGahh3uXk5yTDEaXCpx4XTF2JnBcxgw7jkMQtuJTqtBldZkob9bLTzdYWQZM5AYGns7MUlrmgZV1NnL5qtfdBqBBTljv0YS3Usuz5n4a2v9jd0ot8jIstsHHHbDQDD2/SzjmVEDCxhsnjGZAgCcKylC60xuHJBKjgb7d0FABjitCjKTQEA1J2NnXdvvnS4vZzc8TZGikoPN/n3GCUbdciblDDws7HxGAVD+v85NztZbmg2msF+LLHzfzRYu4fUr4z1pkQaiWeb/pExsIRJWqJerj2IteXNbV19qG+3QxCGb6g2klisY/Gn4FYyL4ZWCrV3O9Bs9Yb8OaNsejgS7tzsu70+TAdJpBGWw01W2PpYx+ILqb/U5f1XLreYbfrHxMASRoNt+mMrPUuFxrMyk2CO1415rFTwFysdb539Hhxr9hbcLshJ8fnn5g/ZUyja632kEZLC9ESYDL4vbGThre/G6nB7OYvZu5WGR0TMLiLwx1gdvi+XZY7HjAwTRBH46ylOuV2OgSWMpH4sfzl5Ae4YWd0BDI6WjDUdJLlmaioEwbuqKBZ2hfW34FYyI9MEnVZAZ48LjZ29ITxD9fnaf+Vy0ghLLPb18Yfd0Y8jzd5QN1aN2VDSvkI72UBuXIebvB2+zfG6Kzp8j2TxwGqhT07F1ki8LxhYwqg4NwVJhjh09rhiplgSGNJ/YJzhUAAwx+swK9NbUBoL7978LbiVGOK0mDnwOEX7tNBheYdm/7bgkEZY6tvtsDv6FT+vaPG3851we0Rkm43IMvsWmgcLb2NjJDQY8oKDvNRR+9sMtXjmQB0L2/RfgYEljHRajVwFHit1LN2OfvkFZ7T+A5crjaE6lkAKbiXzY6TwVn6MfFwhJJmcZEBGkgGi6O0bQiOT3hiU+DACKikbKLw92GhlGBzH7lH2DxpNWUEa9FoNGjt7Ud8eO6slfcHAEmax1qZ/X8MleERgSmq8z+/epKmjPTGwUiiQgltJLHS8tfa4cO6id8prrp+BBYi91VSBqBuoryiZmuLzz0xJTUBOSjzcHlGuf6EreTt8+z4lDgy06R940/aXGKt3HA8DS5jdODDct6/hUkx0ivSnfkUiFd56536j993b0ILbQALL0D2FonXoWBqdy02Lhzlh7ILtkXCl0Ng8HlFeISQ1bvRVmby8mXUsozl9wY6LdieMOo1f/8elOhb2YxmOgSXMpqQmYNrkRHhE4K+noz897/GhYdzlclLikW02wu0RsT+KOz5KBbfJxjhMTUvw++dnD+xK3GF3otXmCMEZqk/eoTmA0RVgaGDhCMtITl3ohq2vH/E6La7KSvLrZ69lHcu4pPqV4twU6ON8f7mVljfXnu6Iyd3rR8PAogJpWija61hcbo/cYnq85XyXk0ZZormO5dCQ+hV/Cm4lRp1WXnUQrdNCcofbAEaggMFppBOtXfzDPwJpOqc4NwVxWv9eDqQRlr+d70Sv0634uUWDsXaoH8ucrGRMktv0R+/fQH8xsKhgyZDAEq1D+YD3XW2vy42UhNG3qx+NVKAbzXUsUsgYbTNIX0R7AzlphGWuHw3jhpqSGg9zvA4ut4gTrV1KnlpUkBqa+TMCKpmalgBLshEut8gX1VHsHmeH5tFoNAJumCG16Y/+kXhfMbCooKxgEvRxGjRZ+3CqrVvt0wmZPX4u5xtKGmHZe/YS+qN059JgCm4l86ReI1EYWLod/fIqiUBHWARBwJwsboQ4Gqmh2XgdqEciCII8yrKTbfqv0GztxflLvdD40OF7JKxjuRIDiwri9Vq58VI0rxbaHeBwKADMzExCkiEOdqcbx1qi752xs9+Do0EU3Eqk0ZlobI52pMkGUQSyzEakmwwB3868HDaQG0lHt0MOhNfk+v+CCnAjxLFI09lzs81+dWiW3DDQAuNAo5Vt+gcwsKgk2tv0D13OVxpAYNFqBPldyWdR2KY/2IJbyVVZyRAEoNXmiLrOwHKH2wALbiVs0T8yqX5lZqYpoBVYwGAdy75znehzsY5lqEDrVyQWsxEzM9mmfygGFpVIhbe7Pu+Iyv/on7d7l/MZ4vxbzjfUYB1L9M2PB1twK0nQx8n1QYejrNeIvELIzw63l5PqX44222JqS4zxyP1XApiukBSmJyLdZICz34O/netU6Myig9zhO4D6IAmnhYZjYFHJjAwTssxGOPo9UblNu/Tuwt/lfEMN3Qgx2oqTDypQvyKZH6WFt1IAC3RJs6RwsglGnQY9TjfOdLBzqETqcOvLDs2jGVrHEo1/xwJl7XHh+ECRdyAjzJLFQwpvo+1vYCAYWFQiCAJuHEjPO45HX3oOpGHc5YqmpECnFdBqc+D8peja4O9QEC35LyeNIETT0uZepxsn27x/8IN9jLQaAVdlRW9xciAc/W4cGHgsgnlBBYBrC9hA7nJ7zl6EKHpHoCYnBV5/VVYwSW7T/znb9DOwqEnuxxKFw33ScuRAlktK4vVa+cUqmupYXG4PjrYEX3ArkW4jmmo0jrXY4BGBdJMemcmB/8GXSKGOK4W8DjfZ4Oz3YFKiHvmTAq+hAoCyQm/hbd3ZS+x1M0CJN2yA92/gwoKBNv0qLtAQRRGfnm7H/1d3XrVzABhYVHXD9HRoBOBUWzcaO6NnBKHN1oezHT3QCMHNjwOD/+GjqYGc1MQsyRiHvCBfLABgzsCLcWNnLy7ao2M1waGmwYZxwdT4SFh4O5w8HZSXGvTjOyPDhLREPfpcHhxs7FTg7CY+f3aoH89gHUv4F2h09bnw2qdn8KX/+2d86+VdeOoPh1VtEsjAoiJzgg7FuSkAoqvrrRQuZluSkWQMbPWBRAo8e6JohGVo/xUlXoyTjDoUpCcOu+2J7nCAOzSPRrqdQ03Ru++SP+rOBl9wKxEEQe5kvZNt+tHncuPA+U4A/nf4Honcpv/z8LXpP9Ziww/fOYiyf6/B+v85jFNt3UjQa3FLUbaq+7sxsKhsycwMANEWWIKvjpeUDvxBPdnWHTW9CA6cV67gViJ3vI2SXiMHG5VZISSZaTEhTiOgs8eFJmt0Lf/2lyiKiqwQGoqFt4P2n+uEyy0iI8mA3DTfdqgfy1WWZKSb9OhxuuVGf6Hg7Pfgf/7WhH/YUotlm/6C13c1oMfpxvQME35061zsenwp/u1r8zEpiJ5IwWJgUZm0e/Mnp9qjpqOrksOhk0wGFE72jh5Eyzb2ShbcSqKp462j3y230Q+2B4vEEKfF9Axp+ffEf4yCce5iLy50OaDTCoqFZqmBXN2Zi1HzdyxQcv+VgjRFRlA1GkFuIvdJCKaFmjp78ZMPjuO6jR/h/7y5D7vPXIRWI+DL8y148+5r8eH3bsSq8vygR8uVwMCisgVTUpCSoENXXz/2R0Efg64+F442e+sESv3crn40C/Oip45F6YJbyeDS5olfo3GytRsutwhzvA5TUoN/hyoZHIWa+I9RMOoavC+o83LMMOq0itzmbEsSzPE62J3umH98pf2DlJgOkijdj0UURXxysh3f/vUe3PD0R/jZR6fQ3u1ARpIBDy6dgb/+4Iv4+R0lKJ82SZHQpRQGFpVph6TnaJgW2tvQCY8I5KbFw2I2KnKb0kqjaKhjUbrgViKNRDRc7IG1x6XY7arh0JDpICX/WA6uFIrtERZ5w0OFpoMA7yiAVCAfy236+90e7D2rzAqhoaQ6lgONVlwKorDe7ujHK5/UY+lPduAf/3MX/ni4FR4RuLYwDT+/4xr89bEv4ntfmqnY326lMbBEAGl5czTsK7QnwN1JxyLd1oHz1gnfFfjQkGJSJV+MzQk6eb788AR/QZY73Co0HSThSiEvJQtuh7qWdSw41tIFu9ONJGMcZlmSFLvdjGQjZmUmedv0nw5sWuhPR1rxped34EfvHcHn7XaYDHFYVZ6HD793I966pxxfnp8FnTayI0Fkn12MkBrIHWi0TvhlqZ+FILDkTUrwtv92eyZ8czS5w+0UZV+MgSHTQhM9sDQOLmlW0lVZ3heQZmsfOrodit72RNHVN9iBNZAdhMci1bF8Vn8xZrdAkDZ8LclLhdbPHerHI3e9PeFfYGmz9eG+1+vwL7/agyZrH6akxmPD8nnY+fhS/OjWeZiRqVywCjUGlghgMRsx2+JNz5+cmribITr7PdjX0AlA2cAiCII8fD3RG8gdHHgxVrJ+RSKNIBycwHUs/W6PXAOldGAZuvw7VkdZ9jV0QhSBqWkJyEhSdth/TnYykgxx6HL0y/+GsSYUb9gki2cO1rH4sjTf4xHx3zvPYunzO/D+wRZoNQK+vaQQH35vCf7p2ryAdpBWGwNLhJCnhSZwm/5DTVY4+j1ITdBh2sDKHqUM1rFM3MJb15AX41AEFrnj7QQehTp1oRuOfg9MhjjkBbGL9WikJnuxGlhCNR0EeOvxpP+nO2OwjkUURTmwLFJgheTlFuWnQR+nQZO1D6cvjN2m/0RrF775H7X413cPoauvH0VTzPifB67H2puvQrxemUJrNTCwRIglfqbnSCTVr5TmK7OcbyjpHUvd2UvwTNDh5pOt3SEpuJVIRaWft9vR1TcxC2+l6aA52cnQKDykDgw+RhO9zidQUmBRejpIIrXpj8U6ljMdPWjvdkIfp8GCEEz5xuu18sqj0VYL9bnceP6D4/jKC39B3dlLSNRrsf6WOfjdfdcr1iJATQwsEaI0PxXxOi3auhw4NrDsdaLZXe/9Y6jkcj7JnOxkxOu0sPa6cOpCt+K3Hw5S23KlC24lk0wGZA9U90/UPXMOKdzh9nLzYrjw1u0Rsa9B+RVCQ5UVSC0ILk7YNxaBkneon5ICQ1xoRjGG7t58udrTHfjyT/+CFz46BZdbRMVVGfiwagnWXF+geD2NWhhYIoQhTitX2U/E1UIej4g6BTY8HI1Oq8HVU1MATNw6llAW3Eomeq8RaeRDqQ63l5NGWOrb7eh2qNdiXA3HpRUshjjMDFGh5bwcMxL0WnT2DBb3xordcsPM0IRBYLAfy84hbfo7e5z4/m//httf3onP2+3ISDLgpTuuwcurSpGdolwfo0jAwBJBpGmhidiP5fP2blzqccGo04Rs6LF0YORmotaxHAzR6peh5MAyAetYPB5RHvkIRY0P4B2FsiR7R6FirTBUekNRPDUlZO+4dVqNXB8Ta/1YPhsyJR4qsy1JSDcZ0ON0o+7sJfx+fyOW/mQHfrPHu4vyP147FX96eAlunp8VUQ3flMLAEkGkwtvPzlyEfYK9+5Omg67OTYU+LjRPq4m8UijUBbeS+RM4sNR32NHjdMOo06Bwsilkv0cavQn0MXL0u/H6rrOoOdo6oerNQllwO9S1MVjHIu1QLyiwQ/1YNBpBnhZ64I29ePCt/eiwOzEjw4TffqccP14+H8kR0EI/VCbeuqYoVpCeiCmp8Th/qRc7P+/A0qsy1T4ln+1RcMPD0Vw9NQUaATh/qRfN1l5kmSfOcKdccBui1S+SuQMvxqcvdKPH2Y8E/cT5Ly4FiDlZySGdc5+TbcafjrYFVMfyycl2PPH7Q6hv967SmG1Jwne/OAM3z7OEpEhYSXvCFFikOpbd9RchimJUvtO/nLRtyFWW5JAHhsUz0vHOvkZ02L0Fvv/ni9Nxz43TQvZGMZJE/z2cQARBmLDTQrvDMByaZNThqizvC/JEmxYauuFhKF/YMpKMyEgywCNOvCmPUGwKOZK5ASxtbuvqw/95cx/+8T93ob7djnSTHol6LY61dOH+N/bipk1/xrv7GiN2479WWx/OX+qFRgCKc1NC+rsWTEmBUadBh92JU20Ts0DeX6Fczny5ijmZuHpqCv5u1mRse3AxHvjijJgIKwBHWCLOjTMn4/VdDfj4+AVU9bpgjo/84b1ma6/8xzBUyyUlC/PTcLjJhrqzl3BLUXZIf5eSwlFwK5mfY0bNsTYcarShRKENKMNB7nAb4uWXUmA52doFR797zBUdbo+I13edxbN/PI6uvn4IArDq2jw8XDkLHo+IV/56Bv/113qcauvGQ1v3Y9OfTuC+v5uOr12dE1FtzqX9bWZZkkO+664+ToNrpqbi09MdWPu7g8hMNqLf44HbA7g9HvR7RHhEEf1uEW6PeMXXblFEWqIeN8+z4Cvzs5CRHJn72gwldbgNRcO4yyUbdXjnvutD/nsiEQNLhLlu2iTEaQQ0XOxB8Y8+wMyMJJTkp6JkaipK8lKRNykh4oZYpdGOOdnJIe+eWJqfilc/PTPh6lgOhmn0AADmDgSWibSNgSiK8pYCc0O0QkiSkxKPlAQdOntcONHSPWqIPHjeih++exAHzg+EzRwz/u1r87BgSop8TNWXZuJfFhfg17Vn8cu/fI4zHT34/m8P4Kd/Ool7vzAN3yydErIlrv6QpoNCtZz5cotnTManpzvk3xuI3fUX8aP3juDagkm4pSgby+ZZkJaoV/AslWHrc+Foizdsh3JKnBhYIk6SUYfHbp6NX+88i7MdPTje2oXjrV14Y1cDACDdpMc1U1NRmu8NMPNyzEH/QexzudHZ44LL7YHFbPT7naFcHR+Gd/PS7zjabENXnyvk7xaV4HJ7cCQMBbeSiVh4e+5iL7r6+qHXajAjI7R7mwiCgLnZyfjrqQ4cbrJeEVhsfS48/8EJ/Kr2DDwikGSIw6PLZuGOsrwRa2uSjTrc/3fTced1+Xh911n84s/1aOzsxb++ewgvfnQK315SiNsXTYVRp15wCVfBreTO6/KRkqBDr9ONOK0AjSAgTiNAqxGGfK3xfj1wvfS5RiPgaLMNf/hbE/Y2dKL28w7Uft6BJ35/CDdMT8ctRdm4aW6m4rUigdbb1J29BFH07nk2EUaDJrKAAsvmzZvx7LPPoqWlBUVFRfjZz36GRYsWjXr822+/jSeeeAJnzpzBjBkz8PTTT+PLX/6y/H1RFLF+/Xq8/PLL6OzsxPXXX4+XXnoJM2bMCOT0Jrx/WVyIf1lciAtdDtSdvYS9DZew58xFHGq0ob3biQ+OtOKDI60AAL1Wg/lTzCjJ8waYq3NTAAG4ZHfhot2JSz1O70e7Exd7vB8v9biGXW93Du6ArBEAS7IRU1ITMCU1HlNS45GTGi9/nWWOv2K+VCo4C8f8rcVsRG5aPM5d7MW+hk55ZVUkC1fBrURaBXOyrRt9LreqL5S+kkZXZmclhWU+fm62eSCwDNaxiKKI9w40Y8N7R9DW5d0c8atF2fjXr1zl0wtRoiEO99w4DavK8/Hm7gb8x47P0WLrw1N/OILNH5/GPTcW4I6yPCSGeQ+XPpdb7m8TrsASr9fi9kVTA/75awsnYc31BTh/qQf//4Fm/OFAEw412rDjxAXsOHEB+t9psGTWZNxSlI2KqzJ8Li7vcfbjTHsP6tvtqG/vxuft9oHP7eh1unFVVjLm5SRjfo4Zc7PNmJk5/vPxszBOB8U6v//nbN26FVVVVdiyZQvKysqwadMmVFZW4vjx48jIyLji+E8//RS33347qqur8fd///d44403sHz5cuzduxfz5s0DADzzzDN44YUX8Nprr6GgoABPPPEEKisrceTIERiNsZtYJycZsGyeBcvmWQAM/uHZc+YS6s56Lx12p/x5MLQaAVpBgNPtQZO1D03WPuw+c+VxggBkJhnlMJOdEo9jA8Oh4RpuLs1Lw7mLjdhz5mJEBhZHvxvt3U5c6HKgvcuBj4+3AfBOdYRjJYkl2YhJiXp02J041tIV8iJLJUijQeFqHy7VsUhB6Uy7HU/8/pDcQbQgPREbbp2HGwaWkPrDqNNizfUF+FbZVLy95zxe2n4ajZ29+Pf3j+Gl7afxT+X5KExPRHJ8HJKMOiQZ45A88DFRH6f4c+TAeStcbhEZSQZMSZ04K+sAYEpqAr69ZBq+vWQa6tvteO9vTfifvzXhZFs3PjzSig+PtCJep8XSqzJwS1E2lsycDK1GwPlLvd5AcmEwkNS329Fs7Rvz9+0/14n95zrlr/VaDWZZkjAvJxnzcsyYl23GLEvSsDcB0pR4KDp803CC6GcjgbKyMixcuBAvvvgiAMDj8SA3Nxff/e538dhjj11x/IoVK2C32/Hee+/J11177bUoLi7Gli1bIIoisrOz8fDDD+ORRx4BAFitVmRmZuLVV1/FypUrxz0nm80Gs9kMq9WK5OTQzn9HElEUcbajB3VnL2HP2UvYe/YSTrR5u0umJuiRmqBDWqIeqQl6pCXqkZKgR1qiTv46NVGPtATvx2RjHEQRaO924Hynt4j2/KUenL/Ui8Yhnzv6R14FkTcpATse/buw3O/Xd53FD985JO/Jo9dqoNNqoI/TwBDn/agf+FofN+R7Q66L02ig0wrQaTWI0w75XCNAF6eBbuD7Q7+n0wqwO9xo73Z4w0i3Qw4mF7od8vVdfSP30Pn2jYVY++WrwvIYrX5lN3acuIDlxdkozk2BRiNAEARoBEAreIfkBQHQCAI0moGP8gVwiyJ6nG70udzocbrR63Sj1+VGj7MfvU4Pel396HUOfM/llj/vc3lH66TfJd2eMPT3DXwc/FpAk9U7JfRvX5uHO8ryQv74nGrrRsXzO2DUafCdJdPw8+2n4ez3QB+nwf1fmI5vLylUbGTK5fbgnX2N+PnHp3Cmo2fMYzUCYDJ4g0xy/GCYSTbGIcnoDTPOfg8c/Z6Bj+7LvvZ+dLo9cLjccLo96Hb0o8/lwc3zLHjpH0sUuU9qO97ShT/8rQl/ONCEs0Me03idFi63t7B3NOZ4HQonJ6IgPRGF6YkoSDehID0R+jgNjjTbcKjRKl9sI/xfjtMImJGZhPk5yZibbca/vX8Uzn4PPn7kC/Ju4OQ7f16//RphcTqdqKurw9q1a+XrNBoNKioqUFtbO+LP1NbWoqqqath1lZWVePfddwEA9fX1aGlpQUVFhfx9s9mMsrIy1NbW+hRYYpUgCMhPT0R+eiK+XjIFgHcURqfVBNTHQhCAjGQjMpKNuGbqlaMloiiiw+68Isy02vrwD6W5Qd8fX904YzJ0WgFdff3yypJIo9MKmGwyID3JgHST953tXTcUhO33L5hixo4TF/Du/ia8u78pbL83GBoBKCuYFJbfVZCeiHidFr0uNzb96SQAb3+LDbfOQ77CLzo6rQb/UJqL267OwXsHmvHhkVZ09jrR1dcPW6/L+7HPBZdbhEcEbH39sPX1o7GzV9HzuHl+lqK3p6ZZliTMsszCwzfNxKFGG/5woAnv/a0JTQMjKEadBgXppoFA4v0bKQWU1DEKd6dnmPDVgdWHoiji3MVeHGqy4uCQEHOpx4WjzbaBtgHeDrPpJgPyQ7ChKQ3nV2Bpb2+H2+1GZubwhmaZmZk4duzYiD/T0tIy4vEtLS3y96XrRjvmcg6HAw6HQ/7aZovMFy01hLJeQRAEpJu8L8BqTjPkpiXgo4e/gIaLPXC6B95RDnln6XJf+W7TNfBROs7lEdE/cKzLLaLf44GrX4TL40G/Wxy4fuB7bu/xLrcH8TotJg+EkHSTfsjnBvnzySYDkuPjVF3N9U/lebjU44Sttx8eUYQoAh7Ru3zU7fH+MfZ+PXi9xwP5WEEAEvRaxOu1iNfFDfnc+zFBr4VR5/04+HkcjDoNNIJwxe15hvw+URQhwtuKX/raI3rrk6ZnhK7D7VBajYD5U8zYXX8RGUkGPPH3c/D3C0LbzjxOq8Hyq3Ow/OqcK74niiIc/R7Y+lyw9fajq88FW5/3oxRsbH0ueETIo4iGOK08qmgYMro49Hrp62RjHCaZDCG7b2oRBO+/4/wpZjy2bDZOtnUjOT4OmUnGoKfWBEHA1EkJmDopAV8eCHuiKKLJ2jdsFObUhW7807V5Ebd6MxpNyFVC1dXVeOqpp9Q+DVJRbloCcsNQwDpRZSQZ8ePl89U+jYj271+bh09OtuO2kimqtzMXBAFGnTf4hXiRVNTSaATMsoR+hVlOSjxyUuJROdcS0t9FV/KrHD89PR1arRatra3Drm9tbYXFMvI/nsViGfN46aM/t7l27VpYrVb5cu7cOX/uBhERpmck4c7rC1QPK0TkG78Ci16vR0lJCWpqauTrPB4PampqUF5ePuLPlJeXDzseAD788EP5+IKCAlgslmHH2Gw27Nq1a9TbNBgMSE5OHnYhIiKi6OX3lFBVVRVWr16N0tJSLFq0CJs2bYLdbseaNWsAAKtWrUJOTg6qq6sBAA8++CCWLFmCn/zkJ/jKV76Ct956C3v27MEvfvELAN4htoceegg//vGPMWPGDHlZc3Z2NpYvX67cPSUiIqIJy+/AsmLFCly4cAHr1q1DS0sLiouLsW3bNrlotqGhARrN4MDNddddhzfeeAP/+q//iscffxwzZszAu+++K/dgAYDvf//7sNvtuOeee9DZ2YkbbrgB27Zti+keLERERDTI7z4skShW+7AQERFNZP68fkfOdqJEREREo2BgISIioojHwEJEREQRj4GFiIiIIh4DCxEREUU8BhYiIiKKeAwsREREFPEYWIiIiCjiMbAQERFRxPO7NX8kkpr12mw2lc+EiIiIfCW9bvvSdD8qAktXVxcAIDc3V+UzISIiIn91dXXBbDaPeUxU7CXk8XjQ1NSEpKQkCIKg6G3bbDbk5ubi3Llz3KdoFHyMxsbHZ3x8jMbHx2h8fIzGFomPjyiK6OrqQnZ29rCNk0cSFSMsGo0GU6ZMCenvSE5Ojph/4EjFx2hsfHzGx8dofHyMxsfHaGyR9viMN7IiYdEtERERRTwGFiIiIop4DCzjMBgMWL9+PQwGg9qnErH4GI2Nj8/4+BiNj4/R+PgYjW2iPz5RUXRLRERE0Y0jLERERBTxGFiIiIgo4jGwEBERUcRjYCEiIqKIx8Ayjs2bNyM/Px9GoxFlZWXYvXu32qcUEZ588kkIgjDsMnv2bLVPS1V//vOfccsttyA7OxuCIODdd98d9n1RFLFu3TpkZWUhPj4eFRUVOHnypDonq5LxHqM777zziufVsmXL1DlZFVRXV2PhwoVISkpCRkYGli9fjuPHjw87pq+vD/fffz8mTZoEk8mEr3/962htbVXpjMPPl8foC1/4whXPo+985zsqnXH4vfTSS1iwYIHcIK68vBz/+7//K39/oj6HGFjGsHXrVlRVVWH9+vXYu3cvioqKUFlZiba2NrVPLSLMnTsXzc3N8uWTTz5R+5RUZbfbUVRUhM2bN4/4/WeeeQYvvPACtmzZgl27diExMRGVlZXo6+sL85mqZ7zHCACWLVs27Hn15ptvhvEM1bVjxw7cf//92LlzJz788EO4XC7cdNNNsNvt8jHf+9738Ic//AFvv/02duzYgaamJtx2220qnnV4+fIYAcDdd9897Hn0zDPPqHTG4TdlyhRs3LgRdXV12LNnD774xS/i1ltvxeHDhwFM4OeQSKNatGiReP/998tfu91uMTs7W6yurlbxrCLD+vXrxaKiIrVPI2IBEN955x35a4/HI1osFvHZZ5+Vr+vs7BQNBoP45ptvqnCG6rv8MRJFUVy9erV46623qnI+kaitrU0EIO7YsUMURe9zRqfTiW+//bZ8zNGjR0UAYm1trVqnqarLHyNRFMUlS5aIDz74oHonFYFSU1PFX/7ylxP6OcQRllE4nU7U1dWhoqJCvk6j0aCiogK1tbUqnlnkOHnyJLKzs1FYWIg77rgDDQ0Nap9SxKqvr0dLS8uw55PZbEZZWRmfT5fZvn07MjIyMGvWLNx7773o6OhQ+5RUY7VaAQBpaWkAgLq6OrhcrmHPo9mzZ2Pq1Kkx+zy6/DGSvP7660hPT8e8efOwdu1a9PT0qHF6qnO73Xjrrbdgt9tRXl4+oZ9DUbH5YSi0t7fD7XYjMzNz2PWZmZk4duyYSmcVOcrKyvDqq69i1qxZaG5uxlNPPYXFixfj0KFDSEpKUvv0Ik5LSwsAjPh8kr5H3umg2267DQUFBTh9+jQef/xx3HzzzaitrYVWq1X79MLK4/HgoYcewvXXX4958+YB8D6P9Ho9UlJShh0bq8+jkR4jAPjWt76FvLw8ZGdn48CBA/jBD36A48eP43e/+52KZxteBw8eRHl5Ofr6+mAymfDOO+9gzpw52L9//4R9DjGwUEBuvvlm+fMFCxagrKwMeXl5+M1vfoO77rpLxTOjiWzlypXy5/Pnz8eCBQswbdo0bN++HUuXLlXxzMLv/vvvx6FDh2K+Nmwsoz1G99xzj/z5/PnzkZWVhaVLl+L06dOYNm1auE9TFbNmzcL+/fthtVrx29/+FqtXr8aOHTvUPq2gcEpoFOnp6dBqtVdUTre2tsJisah0VpErJSUFM2fOxKlTp9Q+lYgkPWf4fPJPYWEh0tPTY+559cADD+C9997Dxx9/jClTpsjXWywWOJ1OdHZ2Djs+Fp9Hoz1GIykrKwOAmHoe6fV6TJ8+HSUlJaiurkZRURF++tOfTujnEAPLKPR6PUpKSlBTUyNf5/F4UFNTg/LychXPLDJ1d3fj9OnTyMrKUvtUIlJBQQEsFsuw55PNZsOuXbv4fBrD+fPn0dHRETPPK1EU8cADD+Cdd97BRx99hIKCgmHfLykpgU6nG/Y8On78OBoaGmLmeTTeYzSS/fv3A0DMPI9G4vF44HA4JvZzSO2q30j21ltviQaDQXz11VfFI0eOiPfcc4+YkpIitrS0qH1qqnv44YfF7du3i/X19eJf//pXsaKiQkxPTxfb2trUPjXVdHV1ifv27RP37dsnAhCff/55cd++feLZs2dFURTFjRs3iikpKeLvf/978cCBA+Ktt94qFhQUiL29vSqfefiM9Rh1dXWJjzzyiFhbWyvW19eLf/rTn8RrrrlGnDFjhtjX16f2qYfFvffeK5rNZnH79u1ic3OzfOnp6ZGP+c53viNOnTpV/Oijj8Q9e/aI5eXlYnl5uYpnHV7jPUanTp0Sf/SjH4l79uwR6+vrxd///vdiYWGheOONN6p85uHz2GOPiTt27BDr6+vFAwcOiI899pgoCIL4wQcfiKI4cZ9DDCzj+NnPfiZOnTpV1Ov14qJFi8SdO3eqfUoRYcWKFWJWVpao1+vFnJwcccWKFeKpU6fUPi1VffzxxyKAKy6rV68WRdG7tPmJJ54QMzMzRYPBIC5dulQ8fvy4uicdZmM9Rj09PeJNN90kTp48WdTpdGJeXp549913x9QbhJEeGwDif/3Xf8nH9Pb2ivfdd5+YmpoqJiQkiF/72tfE5uZm9U46zMZ7jBoaGsQbb7xRTEtLEw0Ggzh9+nTx0UcfFa1Wq7onHkb//M//LObl5Yl6vV6cPHmyuHTpUjmsiOLEfQ4JoiiK4RvPISIiIvIfa1iIiIgo4jGwEBERUcRjYCEiIqKIx8BCREREEY+BhYiIiCIeAwsRERFFPAYWIiIiingMLERERBTxGFiIiIgo4jGwEBERUcRjYCEiIqKIx8BCREREEe//AWz09Q+1payLAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(durations_s)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af60d44b",
   "metadata": {},
   "source": [
    "## Continuous Batching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9477d603",
   "metadata": {
    "height": 166
   },
   "outputs": [],
   "source": [
    "color_codes = [\n",
    "    \"31\",  # red\n",
    "    \"32\",  # green\n",
    "    \"34\",  # blue\n",
    "]\n",
    "\n",
    "\n",
    "def format_text(text, i):\n",
    "    return f\"\\x1b[{color_codes[i]}m{text}\\x1b[0m\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a9cdf83c",
   "metadata": {
    "height": 406
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m\n",
      "\u001b[0m\u001b[34m\n",
      "\u001b[0m\u001b[32m\n",
      "\u001b[0m\u001b[34mDeep\u001b[0m\u001b[32m\n",
      "\u001b[0m\u001b[32mDeep\u001b[0m\u001b[34m learning\u001b[0m\u001b[32m learning\u001b[0m\u001b[34m is\u001b[0m\u001b[32m is\u001b[0m\u001b[32m a\u001b[0m\u001b[32m subset\u001b[0m\u001b[32m of\u001b[0m\u001b[32m machine\u001b[0m\u001b[32m learning\u001b[0m\u001b[34m a\u001b[0m\u001b[34m subset\u001b[0m\u001b[34m of\u001b[0m\u001b[34m machine\u001b[0m\u001b[34m learning\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mDeep\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m is\u001b[0m\u001b[31m a\u001b[0m\u001b[31m subset\u001b[0m\u001b[31m of\u001b[0m\u001b[31m machine\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m that\u001b[0m\u001b[31m uses\u001b[0m\u001b[31m artificial\u001b[0m\u001b[31m neural\u001b[0m\u001b[31m networks\u001b[0m\u001b[31m to\u001b[0m\u001b[31m learn\u001b[0m\u001b[31m from\u001b[0m\u001b[31m data\u001b[0m\u001b[31m.\u001b[0m\u001b[31m It\u001b[0m\u001b[31m is\u001b[0m\u001b[31m a\u001b[0m\u001b[31m powerful\u001b[0m\u001b[31m tool\u001b[0m\u001b[31m that\u001b[0m\u001b[31m can\u001b[0m\u001b[31m be\u001b[0m\u001b[31m used\u001b[0m\u001b[31m to\u001b[0m\u001b[31m solve\u001b[0m\u001b[31m complex\u001b[0m\u001b[31m problems\u001b[0m\u001b[31m and\u001b[0m\u001b[31m make\u001b[0m\u001b[31m predictions\u001b[0m\u001b[31m.\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mWhat\u001b[0m\u001b[31m are\u001b[0m\u001b[31m the\u001b[0m\u001b[31m benefits\u001b[0m\u001b[31m of\u001b[0m\u001b[31m deep\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m?\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mDeep\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m has\u001b[0m\u001b[31m many\u001b[0m\u001b[31m benefits\u001b[0m\u001b[31m,\u001b[0m\u001b[31m including\u001b[0m\u001b[31m the\u001b[0m\u001b[31m ability\u001b[0m\u001b[31m to\u001b[0m\u001b[31m:\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mMake\u001b[0m\u001b[31m predictions\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mS\u001b[0m\u001b[31molve\u001b[0m\u001b[31m complex\u001b[0m\u001b[31m problems\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mL\u001b[0m\u001b[31mear\u001b[0m\u001b[31mn\u001b[0m\u001b[31m from\u001b[0m\u001b[31m data\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mIm\u001b[0m\u001b[31mpro\u001b[0m\u001b[31mve\u001b[0m\u001b[31m accuracy\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mRed\u001b[0m\u001b[31muce\u001b[0m\u001b[31m bias\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mIn\u001b[0m\u001b[31mcre\u001b[0m\u001b[31mase\u001b[0m\u001b[31m efficiency\u001b[0m\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): [0.4239628314971924, 0.3092198371887207, 0.2458968162536621]\n",
      "Throughput (tok / s): [73.73928609101748, 122.94883854241307, 59.56887418159094]\n",
      "Total duration (s): 1.8057630062103271\n"
     ]
    }
   ],
   "source": [
    "async_client = AsyncClient(endpoint_url, headers=headers)\n",
    "\n",
    "\n",
    "durations_s = [[], [], []]\n",
    "\n",
    "\n",
    "async def run(max_new_tokens, i):\n",
    "    t0 = time.time()\n",
    "    async for resp in async_client.generate_stream(\"What is deep learning?\", max_new_tokens=max_new_tokens):\n",
    "        durations_s[i].append(time.time() - t0)\n",
    "        print(format_text(resp.token.text, i), sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "\n",
    "t0 = time.time()\n",
    "all_max_new_tokens = [100, 10, 10]\n",
    "await asyncio.gather(*[run(max_new_tokens, i) for i, max_new_tokens in enumerate(all_max_new_tokens)]) \n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", [s[0] for s in durations_s])\n",
    "print(\"Throughput (tok / s):\", [(len(s) - 1) / sum(s[1:]) for s in durations_s])\n",
    "print(\"Total duration (s):\", time.time() - t0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e2f37bf",
   "metadata": {},
   "source": [
    "## Multi-LoRA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f09a9304",
   "metadata": {
    "height": 338
   },
   "outputs": [],
   "source": [
    "def run_with_adapter(prompt, adapter_id):\n",
    "    durations_s = []\n",
    "\n",
    "    t0 = time.time()\n",
    "    for resp in client.generate_stream(\n",
    "        prompt, \n",
    "        adapter_id=adapter_id,\n",
    "        adapter_source=\"hub\",\n",
    "        max_new_tokens=64,\n",
    "    ):\n",
    "        durations_s.append(time.time() - t0)\n",
    "        if not resp.token.special:\n",
    "            print(resp.token.text, sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "    print(\"\\n\\n\\n----------\")\n",
    "    print(\"Time to first token (TTFT) (s):\", durations_s[0])\n",
    "    print(\"Throughput (tok / s):\", (len(durations_s) - 1) / sum(durations_s[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e191cbf",
   "metadata": {
    "height": 321
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " are running around the field.\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.2438361644744873\n",
      "Throughput (tok / s): 47.09352418745579\n"
     ]
    }
   ],
   "source": [
    "pt_hellaswag_processed = \\\n",
    "\"\"\"You are provided with an incomplete passage below. Please read the passage and then finish it with an appropriate response. For example:\n",
    "\n",
    "### Passage: My friend and I think alike. We\n",
    "\n",
    "### Ending: often finish each other's sentences.\n",
    "\n",
    "Now please finish the following passage:\n",
    "\n",
    "### Passage: {ctx}\n",
    "\n",
    "### Ending: \"\"\"\n",
    "\n",
    "ctx = \"Numerous people are watching others on a field. Trainers are playing frisbee with their dogs. the dogs\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_hellaswag_processed.format(ctx=ctx), adapter_id=\"predibase/hellaswag_processed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bb0ad965",
   "metadata": {
    "height": 219
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Mondale was admitted to the Mayo Clinic on Friday . He was diagnosed with influenza after a routine checkup . He is expected to return home soon .\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.23698735237121582\n",
      "Throughput (tok / s): 54.630049559435506\n"
     ]
    }
   ],
   "source": [
    "pt_cnn = \\\n",
    "\"\"\"You are given a news article below. Please summarize the article, including only its highlights.\n",
    "\n",
    "### Article: {article}\n",
    "\n",
    "### Summary: \"\"\"\n",
    "\n",
    "article = \"(CNN)Former Vice President Walter Mondale was released from the Mayo Clinic on Saturday after being admitted with influenza, hospital spokeswoman Kelley Luckstein said. \\\"He's doing well. We treated him for flu and cold symptoms and he was released today,\\\" she said. Mondale, 87, was diagnosed after he went to the hospital for a routine checkup following a fever, former President Jimmy Carter said Friday. \\\"He is in the bed right this moment, but looking forward to come back home,\\\" Carter said during a speech at a Nobel Peace Prize Forum in Minneapolis. \\\"He said tell everybody he is doing well.\\\" Mondale underwent treatment at the Mayo Clinic in Rochester, Minnesota. The 42nd vice president served under Carter between 1977 and 1981, and later ran for President, but lost to Ronald Reagan. But not before he made history by naming a woman, U.S. Rep. Geraldine A. Ferraro of New York, as his running mate. Before that, the former lawyer was  a U.S. senator from Minnesota. His wife, Joan Mondale, died last year.\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_cnn.format(article=article), adapter_id=\"predibase/cnn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c908562c",
   "metadata": {
    "height": 321
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " {\"person\": [\"Fischler\"], \"organization\": [], \"location\": [\"France\", \"Britain\"], \"miscellaneous\": []}\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.23740363121032715\n",
      "Throughput (tok / s): 54.16591277092673\n"
     ]
    }
   ],
   "source": [
    "pt_conllpp = \"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Predict the category of\n",
    "each entity, then place the entity into the list associated with the \n",
    "category in an output JSON payload. Below is an example:\n",
    "\n",
    "Input: EU rejects German call to boycott British lamb . Output: {{\"person\":\n",
    "[], \"organization\": [\"EU\"], \"location\": [], \"miscellaneous\": [\"German\",\n",
    "\"British\"]}}\n",
    "\n",
    "Now, complete the task.\n",
    "\n",
    "Input: {inpt} Output:\"\"\"\n",
    "\n",
    "inpt = \"Only France and Britain backed Fischler 's proposal .\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_conllpp.format(inpt=inpt), adapter_id=\"predibase/conllpp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e4cc6add",
   "metadata": {
    "height": 559
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m Mond\u001b[0m\u001b[32male\u001b[0m\u001b[31m are\u001b[0m\u001b[31m running\u001b[0m\u001b[31m around\u001b[0m\u001b[32m was\u001b[0m\u001b[32m admitted\u001b[0m\u001b[32m to\u001b[0m\u001b[32m the\u001b[0m\u001b[31m the\u001b[0m\u001b[31m field\u001b[0m\u001b[31m.\u001b[0m\u001b[32m May\u001b[0m\u001b[32mo\u001b[0m\u001b[34m {\"\u001b[0m\u001b[34mperson\u001b[0m\u001b[32m Clin\u001b[0m\u001b[32mic\u001b[0m\u001b[32m on\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [\"\u001b[0m\u001b[34mF\u001b[0m\u001b[34misch\u001b[0m\u001b[32m Friday\u001b[0m\u001b[32m .\u001b[0m\u001b[32m He\u001b[0m\u001b[34mler\u001b[0m\u001b[34m\"],\u001b[0m\u001b[34m \"\u001b[0m\u001b[32m was\u001b[0m\u001b[32m diagnosed\u001b[0m\u001b[32m with\u001b[0m\u001b[32m influ\u001b[0m\u001b[34morgan\u001b[0m\u001b[34mization\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [],\u001b[0m\u001b[34m \"\u001b[0m\u001b[32menza\u001b[0m\u001b[32m after\u001b[0m\u001b[32m a\u001b[0m\u001b[32m routine\u001b[0m\u001b[34mlocation\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [\"\u001b[0m\u001b[34mFr\u001b[0m\u001b[32m check\u001b[0m\u001b[32mup\u001b[0m\u001b[32m .\u001b[0m\u001b[32m He\u001b[0m\u001b[32m is\u001b[0m\u001b[34mance\u001b[0m\u001b[34m\",\u001b[0m\u001b[34m \"\u001b[0m\u001b[34mB\u001b[0m\u001b[32m expected\u001b[0m\u001b[32m to\u001b[0m\u001b[32m return\u001b[0m\u001b[32m home\u001b[0m\u001b[32m soon\u001b[0m\u001b[34mrit\u001b[0m\u001b[34main\u001b[0m\u001b[34m\"],\u001b[0m\u001b[34m \"\u001b[0m\u001b[34mm\u001b[0m\u001b[32m .\u001b[0m\u001b[34mis\u001b[0m\u001b[34mcell\u001b[0m\u001b[34maneous\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m []\u001b[0m\u001b[34m}\u001b[0m\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): [0.2720015048980713, 0.24322223663330078, 0.4646892547607422]\n",
      "Throughput (tok / s): [34.63259413041427, 43.471444858343034, 50.4683102466998]\n",
      "Total duration (s): 1.1557486057281494\n"
     ]
    }
   ],
   "source": [
    "durations_s = [[], [], []]\n",
    "\n",
    "\n",
    "async def run(prompt, adapter_id, i):\n",
    "    t0 = time.time()\n",
    "    async for resp in async_client.generate_stream(\n",
    "        prompt, \n",
    "        adapter_id=adapter_id,\n",
    "        adapter_source=\"hub\",\n",
    "        max_new_tokens=64,\n",
    "    ):\n",
    "        durations_s[i].append(time.time() - t0)\n",
    "        if not resp.token.special:\n",
    "            print(format_text(resp.token.text, i), sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "\n",
    "t0 = time.time()\n",
    "prompts = [\n",
    "    pt_hellaswag_processed.format(ctx=ctx),\n",
    "    pt_cnn.format(article=article),\n",
    "    pt_conllpp.format(inpt=inpt),\n",
    "]\n",
    "adapter_ids = [\"predibase/hellaswag_processed\", \"predibase/cnn\", \"predibase/conllpp\"]\n",
    "await asyncio.gather(*[run(prompt, adapter_id, i) \n",
    "                       for i, (prompt, adapter_id) in enumerate(zip(prompts, adapter_ids))]) \n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", [s[0] for s in durations_s])\n",
    "print(\"Throughput (tok / s):\", [(len(s) - 1) / sum(s[1:]) for s in durations_s])\n",
    "print(\"Total duration (s):\", time.time() - t0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed936723",
   "metadata": {},
   "source": [
    "## Bonus: Structured Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00431323",
   "metadata": {
    "height": 149
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Person',\n",
       " 'type': 'object',\n",
       " 'properties': {'name': {'title': 'Name', 'maxLength': 10, 'type': 'string'},\n",
       "  'age': {'title': 'Age', 'type': 'integer'}},\n",
       " 'required': ['name', 'age']}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pydantic import BaseModel, constr\n",
    "\n",
    "class Person(BaseModel):\n",
    "    name: constr(max_length=10)\n",
    "    age: int\n",
    "\n",
    "schema = Person.schema()\n",
    "schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "08b706aa",
   "metadata": {
    "height": 98
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'John Doe', 'age': 30}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    \"Create a person description for me\", \n",
    "    response_format={\"type\": \"json_object\", \"schema\": schema}\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15870540",
   "metadata": {
    "height": 355
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"person\":\\n[\"Fischler\"], \"organization\": [\"France\", \"Britain\"], \"location\": [],\\n\"miscellaneous\": []}\\n\\nInput: The EU has agreed to a 10-year extension of the ban on the use of\\nhormones in beef . Output: {\"person\": [], \"organization\": [\"EU\"],\\n\"location\": [], \"miscellaneous\": [\"hormones\", \"beef\"]}\\n\\nInput: The EU has agreed to a 10-year extension of the ban on the use of\\nhormones in'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt_template = \"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Predict the category of\n",
    "each entity, then place the entity into the list associated with the \n",
    "category in an output JSON payload. Below is an example:\n",
    "\n",
    "Input: EU rejects German call to boycott British lamb . Output: {{\"person\":\n",
    "[], \"organization\": [\"EU\"], \"location\": [], \"miscellaneous\": [\"German\",\n",
    "\"British\"]}}\n",
    "\n",
    "Now, complete the task.\n",
    "\n",
    "Input: {input} Output:\"\"\"\n",
    "\n",
    "# Base Mistral-7B\n",
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),  \n",
    "    max_new_tokens=128,\n",
    ")\n",
    "resp.generated_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "65923d86",
   "metadata": {
    "height": 183
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Output',\n",
       " 'type': 'object',\n",
       " 'properties': {'person': {'title': 'Person',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'organization': {'title': 'Organization',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'location': {'title': 'Location',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'miscellaneous': {'title': 'Miscellaneous',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}}},\n",
       " 'required': ['person', 'organization', 'location', 'miscellaneous']}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from typing import List\n",
    "\n",
    "class Output(BaseModel):\n",
    "    person: List[str]\n",
    "    organization: List[str]\n",
    "    location: List[str]\n",
    "    miscellaneous: List[str]\n",
    "\n",
    "schema = Output.schema()\n",
    "schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a29a1eb2",
   "metadata": {
    "height": 185
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'person': ['Fischler'],\n",
       " 'organization': ['France', 'Britain'],\n",
       " 'location': [],\n",
       " 'miscellaneous': []}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),\n",
    "    response_format={\n",
    "        \"type\": \"json_object\",\n",
    "        \"schema\": schema,\n",
    "    },\n",
    "    max_new_tokens=128,\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c56fca8",
   "metadata": {
    "height": 151
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'person': ['Fischler'],\n",
       " 'organization': [],\n",
       " 'location': ['France', 'Britain'],\n",
       " 'miscellaneous': []}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),\n",
    "    adapter_id=\"predibase/conllpp\",\n",
    "    adapter_source=\"hub\",\n",
    "    max_new_tokens=128,\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee30357c",
   "metadata": {},
   "source": [
    "## Additional Predibase Resources: \n",
    "\n",
    "- LoRA Land demo environment: https://predibase.com/lora-land \n",
    "- Predibase blog articles: https://predibase.com/blog \n",
    "- LoRAX Github repo: https://github.com/predibase/lorax \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}