run_a_simple_testcase.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sddGmEyxMegT"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/carlini/yet-another-applied-llm-benchmark/blob/master/run_a_simple_testcase.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ocOUT0jTNTS6"
      },
      "source": [
        "# Run a happy test case"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "35QuPBmFNBL2"
      },
      "source": [
        "## Install Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-A_XJDB0NAEb",
        "outputId": "76c23689-306b-4fe8-c9d5-998a1380914f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'yet-another-applied-llm-benchmark'...\n",
            "remote: Enumerating objects: 467, done.\u001b[K\n",
            "remote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "remote: Total 467 (delta 0), reused 1 (delta 0), pack-reused 462\u001b[K\n",
            "Receiving objects: 100% (467/467), 170.46 KiB | 1.23 MiB/s, done.\n",
            "Resolving deltas: 100% (319/319), done.\n",
            "/content/yet-another-applied-llm-benchmark\n",
            "Collecting docker (from -r requirements.txt (line 1))\n",
            "  Downloading docker-7.0.0-py3-none-any.whl (147 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.6/147.6 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (1.25.2)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (9.4.0)\n",
            "Collecting openai (from -r requirements.txt (line 4))\n",
            "  Downloading openai-1.12.0-py3-none-any.whl (226 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.7/226.7 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (3.5.2)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (2.16.1)\n",
            "Collecting selenium (from -r requirements.txt (line 7))\n",
            "  Downloading selenium-4.18.1-py3-none-any.whl (10.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.0/10.0 MB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=14.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (23.2)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (2.31.0)\n",
            "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (2.0.7)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai->-r requirements.txt (line 4)) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai->-r requirements.txt (line 4))\n",
            "  Downloading httpx-0.26.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (2.6.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (1.3.0)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (4.66.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (4.9.0)\n",
            "Collecting trio~=0.17 (from selenium->-r requirements.txt (line 7))\n",
            "  Downloading trio-0.24.0-py3-none-any.whl (460 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.2/460.2 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium->-r requirements.txt (line 7))\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 7)) (2024.2.2)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 4)) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 4)) (1.2.0)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 4))\n",
            "  Downloading httpcore-1.0.3-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai->-r requirements.txt (line 4))\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 4)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 4)) (2.16.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->docker->-r requirements.txt (line 1)) (3.3.2)\n",
            "Requirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 7)) (23.2.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 7)) (2.4.0)\n",
            "Collecting outcome (from trio~=0.17->selenium->-r requirements.txt (line 7))\n",
            "  Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium->-r requirements.txt (line 7))\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3>=1.26.0->docker->-r requirements.txt (line 1)) (1.7.1)\n",
            "Installing collected packages: outcome, h11, wsproto, trio, httpcore, docker, trio-websocket, httpx, selenium, openai\n",
            "Successfully installed docker-7.0.0 h11-0.14.0 httpcore-1.0.3 httpx-0.26.0 openai-1.12.0 outcome-1.3.0.post0 selenium-4.18.1 trio-0.24.0 trio-websocket-0.11.1 wsproto-1.2.0\n",
            "Collecting llama-cpp-python (from -r requirements-extra.txt (line 1))\n",
            "  Downloading llama_cpp_python-0.2.44.tar.gz (36.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.6/36.6 MB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting anthropic (from -r requirements-extra.txt (line 2))\n",
            "  Downloading anthropic-0.16.0-py3-none-any.whl (846 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m846.4/846.4 kB\u001b[0m \u001b[31m57.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: google-cloud-aiplatform in /usr/local/lib/python3.10/dist-packages (from -r requirements-extra.txt (line 3)) (1.42.1)\n",
            "Collecting cohere (from -r requirements-extra.txt (line 4))\n",
            "  Downloading cohere-4.48-py3-none-any.whl (52 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.2/52.2 kB\u001b[0m \u001b[31m180.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (4.9.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (1.25.2)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python->-r requirements-extra.txt (line 1))\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (3.1.3)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (0.26.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (2.6.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (1.3.0)\n",
            "Requirement already satisfied: tokenizers>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (0.15.2)\n",
            "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.11.1)\n",
            "Requirement already satisfied: google-auth<3.0.0dev,>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.27.0)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.23.0)\n",
            "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (3.20.3)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (23.2)\n",
            "Requirement already satisfied: google-cloud-storage<3.0.0dev,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.8.0)\n",
            "Requirement already satisfied: google-cloud-bigquery<4.0.0dev,>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (3.12.0)\n",
            "Requirement already satisfied: google-cloud-resource-manager<3.0.0dev,>=1.3.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.12.1)\n",
            "Requirement already satisfied: shapely<3.0.0dev in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.0.3)\n",
            "Requirement already satisfied: aiohttp<4.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (3.9.3)\n",
            "Collecting backoff<3.0,>=2.0 (from cohere->-r requirements-extra.txt (line 4))\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting fastavro<2.0,>=1.8 (from cohere->-r requirements-extra.txt (line 4))\n",
            "  Downloading fastavro-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m110.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting importlib_metadata<7.0,>=6.0 (from cohere->-r requirements-extra.txt (line 4))\n",
            "  Downloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.25.0 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (2.31.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (2.0.7)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (4.0.3)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->anthropic->-r requirements-extra.txt (line 2)) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->anthropic->-r requirements-extra.txt (line 2)) (1.2.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.62.0)\n",
            "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.60.1)\n",
            "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.48.2)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (4.9)\n",
            "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.3.3)\n",
            "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.7.0)\n",
            "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.8.2)\n",
            "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.4 in /usr/local/lib/python3.10/dist-packages (from google-cloud-resource-manager<3.0.0dev,>=1.3.3->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.13.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (2024.2.2)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (1.0.3)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (0.14.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib_metadata<7.0,>=6.0->cohere->-r requirements-extra.txt (line 4)) (3.17.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python->-r requirements-extra.txt (line 1)) (2.1.5)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->anthropic->-r requirements-extra.txt (line 2)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->anthropic->-r requirements-extra.txt (line 2)) (2.16.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.25.0->cohere->-r requirements-extra.txt (line 4)) (3.3.2)\n",
            "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (0.20.3)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (2023.6.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (4.66.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (6.0.1)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.5.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0dev,>=2.7.2->google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.16.0)\n",
            "Building wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.44-cp310-cp310-manylinux_2_35_x86_64.whl size=2590691 sha256=d5974cc437ebeb180349ab448a922c8065407c9c25d347831042ded8973e9a9d\n",
            "  Stored in directory: /root/.cache/pip/wheels/6e/f0/52/1716aa7fefc7eb2a9b76775b0a61fc131b7dcc961e310a048a\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: importlib_metadata, fastavro, diskcache, backoff, llama-cpp-python, cohere, anthropic\n",
            "  Attempting uninstall: importlib_metadata\n",
            "    Found existing installation: importlib-metadata 7.0.1\n",
            "    Uninstalling importlib-metadata-7.0.1:\n",
            "      Successfully uninstalled importlib-metadata-7.0.1\n",
            "Successfully installed anthropic-0.16.0 backoff-2.2.1 cohere-4.48 diskcache-5.6.3 fastavro-1.9.4 importlib_metadata-6.11.0 llama-cpp-python-0.2.44\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/yet-another-applied-llm-benchmark/tests/print_hello.py\", line 1, in <module>\n",
            "    from evaluator import *\n",
            "  File \"/content/yet-another-applied-llm-benchmark/evaluator.py\", line 31, in <module>\n",
            "    import docker_controller\n",
            "  File \"/content/yet-another-applied-llm-benchmark/docker_controller.py\", line 41, in <module>\n",
            "    BACKEND = json.load(open(\"config.json\"))['container']\n",
            "FileNotFoundError: [Errno 2] No such file or directory: 'config.json'\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/carlini/yet-another-applied-llm-benchmark.git\n",
        "# !git clone https://github.com/ViswanathaReddyGajjala/yet-another-applied-llm-benchmark.git\n",
        "\n",
        "%cd yet-another-applied-llm-benchmark\n",
        "!pip install -r requirements.txt\n",
        "!pip install -r requirements-extra.txt\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CJ44Re4nSoTd",
        "outputId": "d880a595-79eb-42d9-d0f5-00dd2459c44e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libfuse2\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  buildah catatonit conmon containernetworking-plugins crun dns-root-data\n",
            "  dnsmasq-base fuse-overlayfs fuse3 glib-networking glib-networking-common\n",
            "  glib-networking-services golang-github-containernetworking-plugin-dnsname\n",
            "  golang-github-containers-common golang-github-containers-image\n",
            "  gsettings-desktop-schemas iptables libavahi-glib1 libfuse3-3 libgpgme11\n",
            "  libip6tc2 libnetfilter-conntrack3 libnfnetlink0 libnftnl11 libostree-1-1\n",
            "  libproxy1v5 libslirp0 libsoup2.4-1 libsoup2.4-common libyajl2 netbase\n",
            "  session-migration slirp4netns uidmap\n",
            "Suggested packages:\n",
            "  containers-storage firewalld nftables docker-compose\n",
            "The following packages will be REMOVED:\n",
            "  fuse\n",
            "The following NEW packages will be installed:\n",
            "  buildah catatonit conmon containernetworking-plugins crun dns-root-data\n",
            "  dnsmasq-base fuse-overlayfs fuse3 glib-networking glib-networking-common\n",
            "  glib-networking-services golang-github-containernetworking-plugin-dnsname\n",
            "  golang-github-containers-common golang-github-containers-image\n",
            "  gsettings-desktop-schemas iptables libavahi-glib1 libfuse3-3 libgpgme11\n",
            "  libip6tc2 libnetfilter-conntrack3 libnfnetlink0 libnftnl11 libostree-1-1\n",
            "  libproxy1v5 libslirp0 libsoup2.4-1 libsoup2.4-common libyajl2 netbase podman\n",
            "  session-migration slirp4netns uidmap\n",
            "0 upgraded, 35 newly installed, 1 to remove and 35 not upgraded.\n",
            "Need to get 27.1 MB of archives.\n",
            "After this operation, 116 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 netbase all 6.3 [12.9 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfuse3-3 amd64 3.10.5-1build1 [81.2 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 fuse3 amd64 3.10.5-1build1 [24.7 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libip6tc2 amd64 1.8.7-1ubuntu5.2 [20.3 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnfnetlink0 amd64 1.0.1-3build3 [14.6 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnetfilter-conntrack3 amd64 1.0.9-1 [45.3 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnftnl11 amd64 1.2.1-1build1 [65.5 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 iptables amd64 1.8.7-1ubuntu5.2 [455 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 uidmap amd64 1:4.8.1-2ubuntu2.2 [22.4 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containers-image all 5.16.0-3 [29.3 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containers-common all 0.44.4+ds1-1 [28.1 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgpgme11 amd64 1.16.0-1.2ubuntu4.1 [136 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libavahi-glib1 amd64 0.8-5ubuntu5.2 [8,296 B]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 libproxy1v5 amd64 0.4.17-2 [51.9 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking-common all 2.72.0-1 [3,718 B]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking-services amd64 2.72.0-1 [9,982 B]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 session-migration amd64 0.3.6 [9,774 B]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 gsettings-desktop-schemas all 42.0-1ubuntu1 [31.1 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking amd64 2.72.0-1 [69.8 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsoup2.4-common all 2.74.2-3 [4,008 B]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsoup2.4-1 amd64 2.74.2-3 [287 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libostree-1-1 amd64 2022.2-3 [333 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 buildah amd64 1.23.1+ds1-2 [6,094 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 catatonit amd64 0.1.7-1 [307 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 conmon amd64 2.0.25+ds1-1.1 [35.1 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 containernetworking-plugins amd64 0.9.1+ds1-1 [6,422 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libyajl2 amd64 2.1.0-3ubuntu0.22.04.1 [21.0 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu jammy/universe amd64 crun amd64 0.17+dfsg-1.1 [300 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dns-root-data all 2023112702~ubuntu0.22.04.1 [5,136 B]\n",
            "Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dnsmasq-base amd64 2.86-1.1ubuntu0.5 [355 kB]\n",
            "Get:31 http://archive.ubuntu.com/ubuntu jammy/universe amd64 fuse-overlayfs amd64 1.7.1-1 [44.7 kB]\n",
            "Get:32 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containernetworking-plugin-dnsname amd64 1.3.1+ds1-2 [1,083 kB]\n",
            "Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 libslirp0 amd64 4.6.1-1build1 [61.5 kB]\n",
            "Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 podman amd64 3.4.4+ds1-1ubuntu1.22.04.2 [10.6 MB]\n",
            "Get:35 http://archive.ubuntu.com/ubuntu jammy/universe amd64 slirp4netns amd64 1.0.1-2 [28.2 kB]\n",
            "Fetched 27.1 MB in 2s (17.8 MB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 35.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Removing fuse (2.9.9-5ubuntu3) ...\n",
            "Selecting previously unselected package netbase.\n",
            "(Reading database ... 121739 files and directories currently installed.)\n",
            "Preparing to unpack .../00-netbase_6.3_all.deb ...\n",
            "Unpacking netbase (6.3) ...\n",
            "Selecting previously unselected package libfuse3-3:amd64.\n",
            "Preparing to unpack .../01-libfuse3-3_3.10.5-1build1_amd64.deb ...\n",
            "Unpacking libfuse3-3:amd64 (3.10.5-1build1) ...\n",
            "Selecting previously unselected package fuse3.\n",
            "Preparing to unpack .../02-fuse3_3.10.5-1build1_amd64.deb ...\n",
            "Unpacking fuse3 (3.10.5-1build1) ...\n",
            "Selecting previously unselected package libip6tc2:amd64.\n",
            "Preparing to unpack .../03-libip6tc2_1.8.7-1ubuntu5.2_amd64.deb ...\n",
            "Unpacking libip6tc2:amd64 (1.8.7-1ubuntu5.2) ...\n",
            "Selecting previously unselected package libnfnetlink0:amd64.\n",
            "Preparing to unpack .../04-libnfnetlink0_1.0.1-3build3_amd64.deb ...\n",
            "Unpacking libnfnetlink0:amd64 (1.0.1-3build3) ...\n",
            "Selecting previously unselected package libnetfilter-conntrack3:amd64.\n",
            "Preparing to unpack .../05-libnetfilter-conntrack3_1.0.9-1_amd64.deb ...\n",
            "Unpacking libnetfilter-conntrack3:amd64 (1.0.9-1) ...\n",
            "Selecting previously unselected package libnftnl11:amd64.\n",
            "Preparing to unpack .../06-libnftnl11_1.2.1-1build1_amd64.deb ...\n",
            "Unpacking libnftnl11:amd64 (1.2.1-1build1) ...\n",
            "Selecting previously unselected package iptables.\n",
            "Preparing to unpack .../07-iptables_1.8.7-1ubuntu5.2_amd64.deb ...\n",
            "Unpacking iptables (1.8.7-1ubuntu5.2) ...\n",
            "Selecting previously unselected package uidmap.\n",
            "Preparing to unpack .../08-uidmap_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking uidmap (1:4.8.1-2ubuntu2.2) ...\n",
            "Selecting previously unselected package golang-github-containers-image.\n",
            "Preparing to unpack .../09-golang-github-containers-image_5.16.0-3_all.deb ...\n",
            "Unpacking golang-github-containers-image (5.16.0-3) ...\n",
            "Selecting previously unselected package golang-github-containers-common.\n",
            "Preparing to unpack .../10-golang-github-containers-common_0.44.4+ds1-1_all.deb ...\n",
            "Unpacking golang-github-containers-common (0.44.4+ds1-1) ...\n",
            "Selecting previously unselected package libgpgme11:amd64.\n",
            "Preparing to unpack .../11-libgpgme11_1.16.0-1.2ubuntu4.1_amd64.deb ...\n",
            "Unpacking libgpgme11:amd64 (1.16.0-1.2ubuntu4.1) ...\n",
            "Selecting previously unselected package libavahi-glib1:amd64.\n",
            "Preparing to unpack .../12-libavahi-glib1_0.8-5ubuntu5.2_amd64.deb ...\n",
            "Unpacking libavahi-glib1:amd64 (0.8-5ubuntu5.2) ...\n",
            "Selecting previously unselected package libproxy1v5:amd64.\n",
            "Preparing to unpack .../13-libproxy1v5_0.4.17-2_amd64.deb ...\n",
            "Unpacking libproxy1v5:amd64 (0.4.17-2) ...\n",
            "Selecting previously unselected package glib-networking-common.\n",
            "Preparing to unpack .../14-glib-networking-common_2.72.0-1_all.deb ...\n",
            "Unpacking glib-networking-common (2.72.0-1) ...\n",
            "Selecting previously unselected package glib-networking-services.\n",
            "Preparing to unpack .../15-glib-networking-services_2.72.0-1_amd64.deb ...\n",
            "Unpacking glib-networking-services (2.72.0-1) ...\n",
            "Selecting previously unselected package session-migration.\n",
            "Preparing to unpack .../16-session-migration_0.3.6_amd64.deb ...\n",
            "Unpacking session-migration (0.3.6) ...\n",
            "Selecting previously unselected package gsettings-desktop-schemas.\n",
            "Preparing to unpack .../17-gsettings-desktop-schemas_42.0-1ubuntu1_all.deb ...\n",
            "Unpacking gsettings-desktop-schemas (42.0-1ubuntu1) ...\n",
            "Selecting previously unselected package glib-networking:amd64.\n",
            "Preparing to unpack .../18-glib-networking_2.72.0-1_amd64.deb ...\n",
            "Unpacking glib-networking:amd64 (2.72.0-1) ...\n",
            "Selecting previously unselected package libsoup2.4-common.\n",
            "Preparing to unpack .../19-libsoup2.4-common_2.74.2-3_all.deb ...\n",
            "Unpacking libsoup2.4-common (2.74.2-3) ...\n",
            "Selecting previously unselected package libsoup2.4-1:amd64.\n",
            "Preparing to unpack .../20-libsoup2.4-1_2.74.2-3_amd64.deb ...\n",
            "Unpacking libsoup2.4-1:amd64 (2.74.2-3) ...\n",
            "Selecting previously unselected package libostree-1-1:amd64.\n",
            "Preparing to unpack .../21-libostree-1-1_2022.2-3_amd64.deb ...\n",
            "Unpacking libostree-1-1:amd64 (2022.2-3) ...\n",
            "Selecting previously unselected package buildah.\n",
            "Preparing to unpack .../22-buildah_1.23.1+ds1-2_amd64.deb ...\n",
            "Unpacking buildah (1.23.1+ds1-2) ...\n",
            "Selecting previously unselected package catatonit.\n",
            "Preparing to unpack .../23-catatonit_0.1.7-1_amd64.deb ...\n",
            "Unpacking catatonit (0.1.7-1) ...\n",
            "Selecting previously unselected package conmon.\n",
            "Preparing to unpack .../24-conmon_2.0.25+ds1-1.1_amd64.deb ...\n",
            "Unpacking conmon (2.0.25+ds1-1.1) ...\n",
            "Selecting previously unselected package containernetworking-plugins.\n",
            "Preparing to unpack .../25-containernetworking-plugins_0.9.1+ds1-1_amd64.deb ...\n",
            "Unpacking containernetworking-plugins (0.9.1+ds1-1) ...\n",
            "Selecting previously unselected package libyajl2:amd64.\n",
            "Preparing to unpack .../26-libyajl2_2.1.0-3ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking libyajl2:amd64 (2.1.0-3ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package crun.\n",
            "Preparing to unpack .../27-crun_0.17+dfsg-1.1_amd64.deb ...\n",
            "Unpacking crun (0.17+dfsg-1.1) ...\n",
            "Selecting previously unselected package dns-root-data.\n",
            "Preparing to unpack .../28-dns-root-data_2023112702~ubuntu0.22.04.1_all.deb ...\n",
            "Unpacking dns-root-data (2023112702~ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package dnsmasq-base.\n",
            "Preparing to unpack .../29-dnsmasq-base_2.86-1.1ubuntu0.5_amd64.deb ...\n",
            "Unpacking dnsmasq-base (2.86-1.1ubuntu0.5) ...\n",
            "Selecting previously unselected package fuse-overlayfs.\n",
            "Preparing to unpack .../30-fuse-overlayfs_1.7.1-1_amd64.deb ...\n",
            "Unpacking fuse-overlayfs (1.7.1-1) ...\n",
            "Selecting previously unselected package golang-github-containernetworking-plugin-dnsname.\n",
            "Preparing to unpack .../31-golang-github-containernetworking-plugin-dnsname_1.3.1+ds1-2_amd64.deb ...\n",
            "Unpacking golang-github-containernetworking-plugin-dnsname (1.3.1+ds1-2) ...\n",
            "Selecting previously unselected package libslirp0:amd64.\n",
            "Preparing to unpack .../32-libslirp0_4.6.1-1build1_amd64.deb ...\n",
            "Unpacking libslirp0:amd64 (4.6.1-1build1) ...\n",
            "Selecting previously unselected package podman.\n",
            "Preparing to unpack .../33-podman_3.4.4+ds1-1ubuntu1.22.04.2_amd64.deb ...\n",
            "Unpacking podman (3.4.4+ds1-1ubuntu1.22.04.2) ...\n",
            "Selecting previously unselected package slirp4netns.\n",
            "Preparing to unpack .../34-slirp4netns_1.0.1-2_amd64.deb ...\n",
            "Unpacking slirp4netns (1.0.1-2) ...\n",
            "Setting up session-migration (0.3.6) ...\n",
            "Created symlink /etc/systemd/user/graphical-session-pre.target.wants/session-migration.service → /usr/lib/systemd/user/session-migration.service.\n",
            "Setting up libproxy1v5:amd64 (0.4.17-2) ...\n",
            "Setting up libip6tc2:amd64 (1.8.7-1ubuntu5.2) ...\n",
            "Setting up uidmap (1:4.8.1-2ubuntu2.2) ...\n",
            "Setting up libyajl2:amd64 (2.1.0-3ubuntu0.22.04.1) ...\n",
            "Setting up libnftnl11:amd64 (1.2.1-1build1) ...\n",
            "Setting up libgpgme11:amd64 (1.16.0-1.2ubuntu4.1) ...\n",
            "Setting up dns-root-data (2023112702~ubuntu0.22.04.1) ...\n",
            "Setting up golang-github-containers-image (5.16.0-3) ...\n",
            "Setting up conmon (2.0.25+ds1-1.1) ...\n",
            "Setting up catatonit (0.1.7-1) ...\n",
            "Setting up libfuse3-3:amd64 (3.10.5-1build1) ...\n",
            "Setting up libsoup2.4-common (2.74.2-3) ...\n",
            "Setting up libslirp0:amd64 (4.6.1-1build1) ...\n",
            "Setting up libnfnetlink0:amd64 (1.0.1-3build3) ...\n",
            "Setting up netbase (6.3) ...\n",
            "Setting up libavahi-glib1:amd64 (0.8-5ubuntu5.2) ...\n",
            "Setting up golang-github-containers-common (0.44.4+ds1-1) ...\n",
            "Setting up glib-networking-common (2.72.0-1) ...\n",
            "Setting up gsettings-desktop-schemas (42.0-1ubuntu1) ...\n",
            "Setting up glib-networking-services (2.72.0-1) ...\n",
            "Setting up slirp4netns (1.0.1-2) ...\n",
            "Setting up crun (0.17+dfsg-1.1) ...\n",
            "Setting up fuse3 (3.10.5-1build1) ...\n",
            "Installing new version of config file /etc/fuse.conf ...\n",
            "Setting up libnetfilter-conntrack3:amd64 (1.0.9-1) ...\n",
            "Setting up fuse-overlayfs (1.7.1-1) ...\n",
            "Setting up iptables (1.8.7-1ubuntu5.2) ...\n",
            "update-alternatives: using /usr/sbin/iptables-legacy to provide /usr/sbin/iptables (iptables) in auto mode\n",
            "update-alternatives: using /usr/sbin/ip6tables-legacy to provide /usr/sbin/ip6tables (ip6tables) in auto mode\n",
            "update-alternatives: using /usr/sbin/iptables-nft to provide /usr/sbin/iptables (iptables) in auto mode\n",
            "update-alternatives: using /usr/sbin/ip6tables-nft to provide /usr/sbin/ip6tables (ip6tables) in auto mode\n",
            "update-alternatives: using /usr/sbin/arptables-nft to provide /usr/sbin/arptables (arptables) in auto mode\n",
            "update-alternatives: using /usr/sbin/ebtables-nft to provide /usr/sbin/ebtables (ebtables) in auto mode\n",
            "Setting up dnsmasq-base (2.86-1.1ubuntu0.5) ...\n",
            "Setting up containernetworking-plugins (0.9.1+ds1-1) ...\n",
            "Setting up golang-github-containernetworking-plugin-dnsname (1.3.1+ds1-2) ...\n",
            "Setting up podman (3.4.4+ds1-1ubuntu1.22.04.2) ...\n",
            "Created symlink /etc/systemd/user/default.target.wants/podman.service → /usr/lib/systemd/user/podman.service.\n",
            "Created symlink /etc/systemd/user/sockets.target.wants/podman.socket → /usr/lib/systemd/user/podman.socket.\n",
            "Created symlink /etc/systemd/system/default.target.wants/podman.service → /lib/systemd/system/podman.service.\n",
            "Created symlink /etc/systemd/system/sockets.target.wants/podman.socket → /lib/systemd/system/podman.socket.\n",
            "Created symlink /etc/systemd/system/default.target.wants/podman-auto-update.service → /lib/systemd/system/podman-auto-update.service.\n",
            "Created symlink /etc/systemd/system/timers.target.wants/podman-auto-update.timer → /lib/systemd/system/podman-auto-update.timer.\n",
            "Created symlink /etc/systemd/system/default.target.wants/podman-restart.service → /lib/systemd/system/podman-restart.service.\n",
            "Processing triggers for libglib2.0-0:amd64 (2.72.4-0ubuntu2.2) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for dbus (1.12.20-2ubuntu4.1) ...\n",
            "Setting up glib-networking:amd64 (2.72.0-1) ...\n",
            "Setting up libsoup2.4-1:amd64 (2.74.2-3) ...\n",
            "Setting up libostree-1-1:amd64 (2022.2-3) ...\n",
            "Setting up buildah (1.23.1+ds1-2) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!sudo apt-get install podman # this is not necessary\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s9X1rz69SZfw",
        "outputId": "07346a38-7bce-4e94-a6b1-52efd028705b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/bin/podman\n"
          ]
        }
      ],
      "source": [
        "!which podman"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mTE6bJpJUo4R"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K31AsKNWNXuB"
      },
      "source": [
        "### Add API keys\n",
        "\n",
        "You have to add the api keys of the respective models to run them."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5A8OwQ_PNOcK",
        "outputId": "04858270-98ca-4c59-8b60-c8e5d3a67fd2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Writing config.json\n"
          ]
        }
      ],
      "source": [
        "%%writefile config.json\n",
        "{\n",
        "    \"container\": \"podman\",\n",
        "    \"hparams\": {\n",
        "        \"temperature\": 0.7\n",
        "    },\n",
        "    \"llms\": {\n",
        "        \"vertexai\": {\n",
        "            \"project_id\": \"TODO\"\n",
        "        },\n",
        "        \"openai\": {\n",
        "            \"api_key\": \"UPDATETHISVALUEIFNEEDED\"\n",
        "        },\n",
        "        \"mistral\": {\n",
        "            \"api_key\": \"TODO\"\n",
        "        },\n",
        "        \"cohere\": {\n",
        "            \"api_key\": \"TODO\"\n",
        "        },\n",
        "        \"anthropic\": {\n",
        "            \"api_key\": \"TODO\"\n",
        "        }\n",
        "    }\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ON5dzImSOzSz"
      },
      "source": [
        "For testing purposes, follow these instructions:\n",
        "\n",
        "\n",
        "We'll be using \"gpt-3.5-turbo\" which can be accessed via free accounts.\n",
        "\n",
        "\n",
        "In `llm.py` update the following variables.\n",
        "1. Set `llm = LLM(\"gpt-3.5-turbo\")`\n",
        "2. Set `eval_llm = LLM(\"gpt-3.5-turbo\", override_hparams={'temperature': 0.1})`\n",
        "\n",
        "\n",
        "In `evaluator.py`\n",
        "\n",
        "1. Update the variable `PYTHON_ENV = \"python3.11\"` to `PYTHON_ENV = \"python\"`\n",
        "\n",
        "\n",
        "In `docker_controller.py` (if not using `podman` (or) `docker`):\n",
        "1. Set `I_HAVE_BLIND_FAITH_IN_LLMS_AND_AM_OKAY_WITH_THEM_BRICKING_MY_MACHINE_OR_MAKING_THEM_HALT_AND_CATCH_FIRE` to `True`\n",
        "\n",
        "If you prefer running it locally:\n",
        " `Add the respective Python path in evaluator.py .`\n",
        "\n",
        "\n",
        "These changes will enable you to use \"gpt-3.5-turbo\" for testing."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W-9U4FNoUe6I"
      },
      "source": [
        "## Testing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Hurray! Everything is set up. Now, let's run a simple test case."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GUC4OCZROW81",
        "outputId": "bdd738f8-a37d-4333-94f7-a843eb74fa12"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2024-02-21 01:35:23.331165: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-02-21 01:35:23.331233: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-02-21 01:35:23.332954: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-02-21 01:35:23.341169: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-02-21 01:35:24.597293: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "gpt-3.5-turbo GETCACHE 'Program:\\n\\n```python\\nprint(\"hello world\")\\n```\\n\\nThis program uses the `print()` function in Python to output the string \"hello world\" to the console.'\n",
            "# Initial Query\n",
            "> Write a python program that prints the string \"hello world\" and tell me how it works in a sentence\n",
            "\n",
            "# LLM Generation\n",
            "\n",
            "## Query\n",
            "> Write a python program that prints the string \"hello world\" and tell me how it works in a sentence\n",
            "\n",
            "## Output\n",
            "> Program:\n",
            "> \n",
            "> ```python\n",
            "> print(\"hello world\")\n",
            "> ```\n",
            "> \n",
            "> This program uses the `print()` function in Python to output the string \"hello world\" to the console.\n",
            "\n",
            "# Extract Code\n",
            "I extracted the following code from that output:\n",
            "> ```\n",
            "> print(\"hello world\")\n",
            "> ```\n",
            "\n",
            "# Run Code Interpreter\n",
            "Running the following program:\n",
            "> ```\n",
            "> print(\"hello world\")\n",
            "> ```\n",
            "And got the output:\n",
            "```\n",
            "hello world\n",
            "```\n",
            "\n",
            "# Substring Evaluation\n",
            "Testing if the previous output contains the string `hello world`: True\n",
            "\n",
            "True\n"
          ]
        }
      ],
      "source": [
        "!PYTHONPATH='.' python tests/print_hello.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wH1T2iv2SuRh"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}