CrispAutoEval.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# @title # 🧐 LLM AutoEval\n",
        "# @markdown > 🗣️ [Large Language Model Course](https://github.com/mlabonne/llm-course)\n",
        "# @markdown ❤️ Created by [@maximelabonne](https://twitter.com/maximelabonne).\n",
        "# @markdown * This notebook allows you to **automatically evaluate your LLMs** using RunPod (please consider using my [referral link](https://runpod.io?ref=9nvk2srl)).\n",
        "# @markdown * The results are automatically uploaded to [GitHub Gist](https://gist.github.com/) and the pod is destroyed (you can safely close this tab).\n",
        "# @markdown * For further details, see the project on 💻 [GitHub](https://github.com/mlabonne/llm-autoeval).\n",
        "# @markdown ---\n",
        "# @markdown ## 🔑 Tokens\n",
        "# @markdown Enter the name of your tokens in the Secrets tab.\n",
        "\n",
        "RUNPOD_TOKEN = \"runpod\" # @param {type:\"string\"}\n",
        "GITHUB_TOKEN = \"github\" # @param {type:\"string\"}\n",
        "HF_TOKEN = \"HF_TOKEN\" # @param {type:\"string\"}\n",
        "!pip install runpod requests\n",
        "\n",
        "import runpod\n",
        "import requests\n",
        "from google.colab import userdata\n",
        "from huggingface_hub import HfApi\n",
        "\n",
        "# Fetch Tokens from Secrets\n",
        "runpod.api_key = userdata.get(RUNPOD_TOKEN)\n",
        "GITHUB_API_TOKEN = userdata.get(GITHUB_TOKEN)\n",
        "HUGGINGFACE_TOKEN = userdata.get(HF_TOKEN)\n",
        "\n",
        "# Step 1: Verify RUNPOD_TOKEN\n",
        "def verify_runpod_token(api_key):\n",
        "    runpod.api_key = api_key\n",
        "    try:\n",
        "        user_info = runpod.get_user()\n",
        "        print(\"RunPod token is valid.\")\n",
        "        #print(f\"User: {user_info}\")\n",
        "    except Exception as e:\n",
        "        print(f\"RunPod token is invalid. Error: {e}\")\n",
        "\n",
        "# Step 2: Verify GITHUB_TOKEN\n",
        "def verify_github_token(token):\n",
        "    headers = {\"Authorization\": f\"Bearer {token}\"}\n",
        "    # Test creating a gist to verify permissions\n",
        "    gist_data = {\n",
        "        \"description\": \"Test gist for verifying token\",\n",
        "        \"public\": False,\n",
        "        \"files\": {\n",
        "            \"test.txt\": {\n",
        "                \"content\": \"This is a test gist.\"\n",
        "            }\n",
        "        }\n",
        "    }\n",
        "    response = requests.post(\"https://api.github.com/gists\", headers=headers, json=gist_data)\n",
        "    if response.status_code == 201:\n",
        "        print(\"GitHub token is valid and can create gists.\")\n",
        "        # Delete the test gist\n",
        "        gist_url = response.json()['url']\n",
        "        delete_response = requests.delete(gist_url, headers=headers)\n",
        "        if delete_response.status_code == 204:\n",
        "            print(\"Test gist deleted successfully.\")\n",
        "        else:\n",
        "            print(f\"Failed to delete test gist. Status code: {delete_response.status_code}, Response: {delete_response.text}\")\n",
        "    else:\n",
        "        print(f\"GitHub token is invalid or lacks gist creation permissions. Status code: {response.status_code}, Response: {response.text}\")\n",
        "\n",
        "# Step 3: Verify HF_TOKEN\n",
        "def verify_hf_token(token):\n",
        "    api = HfApi()\n",
        "    try:\n",
        "        user_info = api.whoami(token)\n",
        "        print(\"Hugging Face token is valid.\")\n",
        "        print(f\"User: {user_info['name']}\")\n",
        "    except Exception as e:\n",
        "        print(f\"Hugging Face token is invalid. Error: {e}\")\n",
        "\n",
        "# Run the verification\n",
        "print (\"verifying tokens...\")\n",
        "verify_runpod_token(runpod.api_key)\n",
        "verify_github_token(GITHUB_API_TOKEN)\n",
        "verify_hf_token(HUGGINGFACE_TOKEN)\n",
        "\n",
        "# --- Original Script ---"
      ],
      "metadata": {
        "id": "KM_bJsoTBSkw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!pip install -qqq runpod --progress-bar off\n",
        "\n",
        "import runpod\n",
        "from google.colab import userdata\n",
        "\n",
        "# @markdown ---\n",
        "\n",
        "# @markdown ## 🔍 Evaluation\n",
        "MODEL_ID = \"cstr/Spaetzle-v85-7b\" # @param {type:\"string\"}\n",
        "BENCHMARK = \"nous\" # @param [\"nous\", \"eq-bench\", \"openllm\", \"lighteval\"]\n",
        "\n",
        "# @markdown For lighteval, select tasks as specified in the [readme](https://github.com/huggingface/lighteval?tab=readme-ov-file#usage) or in the list of [recommended tasks](https://github.com/huggingface/lighteval/blob/main/tasks_examples/recommended_set.txt).\n",
        "\n",
        "LIGHTEVAL_TASK = \"leaderboard|truthfulqa:mc|0|0,leaderboard|gsm8k|0|0\" # @param {type:\"string\"}\n",
        "\n",
        "# @markdown ---\n",
        "\n",
        "# @markdown ## ☁️ Cloud GPU\n",
        "\n",
        "GPU = \"NVIDIA RTX A6000\" # @param [\"NVIDIA A100 80GB PCIe\", \"NVIDIA A100-SXM4-80GB\", \"NVIDIA A30\", \"NVIDIA A40\", \"NVIDIA GeForce RTX 3070\", \"NVIDIA GeForce RTX 3080\", \"NVIDIA GeForce RTX 3080 Ti\", \"NVIDIA GeForce RTX 3090\", \"NVIDIA GeForce RTX 3090 Ti\", \"NVIDIA GeForce RTX 4070 Ti\", \"NVIDIA GeForce RTX 4080\", \"NVIDIA GeForce RTX 4090\", \"NVIDIA H100 80GB HBM3\", \"NVIDIA H100 PCIe\", \"NVIDIA L4\", \"NVIDIA L40\", \"NVIDIA RTX 4000 Ada Generation\", \"NVIDIA RTX 4000 SFF Ada Generation\", \"NVIDIA RTX 5000 Ada Generation\", \"NVIDIA RTX 6000 Ada Generation\", \"NVIDIA RTX A2000\", \"NVIDIA RTX A4000\", \"NVIDIA RTX A4500\", \"NVIDIA RTX A5000\", \"NVIDIA RTX A6000\", \"Tesla V100-FHHL-16GB\", \"Tesla V100-PCIE-16GB\", \"Tesla V100-SXM2-16GB\", \"Tesla V100-SXM2-32GB\"]\n",
        "NUMBER_OF_GPUS = 1 # @param {type:\"slider\", min:1, max:8, step:1}\n",
        "CONTAINER_DISK = 75 # @param {type:\"slider\", min:50, max:500, step:25}\n",
        "CLOUD_TYPE = \"COMMUNITY\" # @param [\"COMMUNITY\", \"SECURE\"]\n",
        "REPO = \"https://github.com/mlabonne/llm-autoeval.git\" # @param {type:\"string\"}\n",
        "TRUST_REMOTE_CODE = False # @param {type:\"boolean\"}\n",
        "PRIVATE_GIST = True # @param {type:\"boolean\"}\n",
        "DEBUG = False # @param {type:\"boolean\"}\n",
        "\n",
        "# @markdown ---\n",
        "\n",
        "# Create a pod\n",
        "pod = runpod.create_pod(\n",
        "    name=f\"Eval {MODEL_ID.split('/')[-1]} on {BENCHMARK.capitalize()}\",\n",
        "    image_name=\"runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel-ubuntu22.04\",\n",
        "    gpu_type_id=GPU,\n",
        "    cloud_type=CLOUD_TYPE,\n",
        "    gpu_count=NUMBER_OF_GPUS,\n",
        "    volume_in_gb=0,\n",
        "    container_disk_in_gb=CONTAINER_DISK,\n",
        "    template_id=\"au6nz6emhk\",\n",
        "    env={\n",
        "        \"BENCHMARK\": BENCHMARK,\n",
        "        \"MODEL_ID\": MODEL_ID,\n",
        "        \"REPO\": REPO,\n",
        "        \"TRUST_REMOTE_CODE\": TRUST_REMOTE_CODE,\n",
        "        \"PRIVATE_GIST\": PRIVATE_GIST,\n",
        "        \"DEBUG\": DEBUG,\n",
        "        \"GITHUB_API_TOKEN\": GITHUB_API_TOKEN,\n",
        "        \"HUGGINGFACE_TOKEN\": HUGGINGFACE_TOKEN,\n",
        "        \"LIGHT_EVAL_TASK\": LIGHTEVAL_TASK,\n",
        "        \"NUMBER_OF_GPUS\": NUMBER_OF_GPUS\n",
        "    }\n",
        ")\n",
        "\n",
        "print(\"Pod started: https://www.runpod.io/console/pods\")"
      ],
      "metadata": {
        "id": "elyxjYI_rY5W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a901ac9-833f-4ff2-fdf2-f13c33279f1d"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pod started: https://www.runpod.io/console/pods\n"
          ]
        }
      ]
    }
  ]
}