4-OLMO-LLM-Training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MonikaVen/LLM-Prompting-RAG-Aligment-Training-Tutorial/blob/main/4-OLMO-LLM-Training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download https://github.com/allenai/OLMo.git\n",
        "Download the dataset:\n",
        "\n",
        "https://olmo-data.org/preprocessed/olmo-mix/v1_5/gpt-neox-20b-pii-special/part-000-00000.npy"
      ],
      "metadata": {
        "id": "ZJG5LC9gdivd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/allenai/OLMo.git\n",
        "!pip install poetry"
      ],
      "metadata": {
        "id": "KWfaPnily7YM",
        "outputId": "58f2d84c-da7c-4c27-e108-0b5d66c86489",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'OLMo'...\n",
            "remote: Enumerating objects: 22551, done.\u001b[K\n",
            "remote: Counting objects: 100% (5026/5026), done.\u001b[K\n",
            "remote: Compressing objects: 100% (1405/1405), done.\u001b[K\n",
            "remote: Total 22551 (delta 3792), reused 4683 (delta 3599), pack-reused 17525\u001b[K\n",
            "Receiving objects: 100% (22551/22551), 93.44 MiB | 13.92 MiB/s, done.\n",
            "Resolving deltas: 100% (15707/15707), done.\n",
            "Collecting poetry\n",
            "  Downloading poetry-1.8.2-py3-none-any.whl (249 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m249.4/249.4 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: build<2.0.0,>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from poetry) (1.2.1)\n",
            "Requirement already satisfied: cachecontrol[filecache]<0.15.0,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from poetry) (0.14.0)\n",
            "Collecting cleo<3.0.0,>=2.1.0 (from poetry)\n",
            "  Downloading cleo-2.1.0-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting crashtest<0.5.0,>=0.4.1 (from poetry)\n",
            "  Downloading crashtest-0.4.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting dulwich<0.22.0,>=0.21.2 (from poetry)\n",
            "  Downloading dulwich-0.21.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (514 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m514.7/514.7 kB\u001b[0m \u001b[31m47.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fastjsonschema<3.0.0,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from poetry) (2.19.1)\n",
            "Collecting installer<0.8.0,>=0.7.0 (from poetry)\n",
            "  Downloading installer-0.7.0-py3-none-any.whl (453 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m453.8/453.8 kB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting keyring<25.0.0,>=24.0.0 (from poetry)\n",
            "  Downloading keyring-24.3.1-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: packaging>=23.1 in /usr/local/lib/python3.10/dist-packages (from poetry) (24.0)\n",
            "Requirement already satisfied: pexpect<5.0.0,>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from poetry) (4.9.0)\n",
            "Collecting pkginfo<2.0.0,>=1.9.4 (from poetry)\n",
            "  Downloading pkginfo-1.10.0-py3-none-any.whl (30 kB)\n",
            "Requirement already satisfied: platformdirs<5,>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from poetry) (4.2.0)\n",
            "Collecting poetry-core==1.9.0 (from poetry)\n",
            "  Downloading poetry_core-1.9.0-py3-none-any.whl (309 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m309.5/309.5 kB\u001b[0m \u001b[31m39.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting poetry-plugin-export<2.0.0,>=1.6.0 (from poetry)\n",
            "  Downloading poetry_plugin_export-1.7.1-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: pyproject-hooks<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from poetry) (1.0.0)\n",
            "Requirement already satisfied: requests<3.0,>=2.26 in /usr/local/lib/python3.10/dist-packages (from poetry) (2.31.0)\n",
            "Collecting requests-toolbelt<2.0.0,>=1.0.0 (from poetry)\n",
            "  Downloading requests_toolbelt-1.0.0-py2.py3-none-any.whl (54 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting shellingham<2.0,>=1.5 (from poetry)\n",
            "  Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n",
            "Requirement already satisfied: tomli<3.0.0,>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from poetry) (2.0.1)\n",
            "Collecting tomlkit<1.0.0,>=0.11.4 (from poetry)\n",
            "  Downloading tomlkit-0.12.4-py3-none-any.whl (37 kB)\n",
            "Collecting trove-classifiers>=2022.5.19 (from poetry)\n",
            "  Downloading trove_classifiers-2024.3.25-py3-none-any.whl (13 kB)\n",
            "Collecting virtualenv<21.0.0,>=20.23.0 (from poetry)\n",
            "  Downloading virtualenv-20.25.1-py3-none-any.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m92.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: msgpack<2.0.0,>=0.5.2 in /usr/local/lib/python3.10/dist-packages (from cachecontrol[filecache]<0.15.0,>=0.14.0->poetry) (1.0.8)\n",
            "Requirement already satisfied: filelock>=3.8.0 in /usr/local/lib/python3.10/dist-packages (from cachecontrol[filecache]<0.15.0,>=0.14.0->poetry) (3.13.3)\n",
            "Collecting rapidfuzz<4.0.0,>=3.0.0 (from cleo<3.0.0,>=2.1.0->poetry)\n",
            "  Downloading rapidfuzz-3.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m45.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3>=1.25 in /usr/local/lib/python3.10/dist-packages (from dulwich<0.22.0,>=0.21.2->poetry) (2.0.7)\n",
            "Collecting jaraco.classes (from keyring<25.0.0,>=24.0.0->poetry)\n",
            "  Downloading jaraco.classes-3.3.1-py3-none-any.whl (6.8 kB)\n",
            "Requirement already satisfied: importlib-metadata>=4.11.4 in /usr/local/lib/python3.10/dist-packages (from keyring<25.0.0,>=24.0.0->poetry) (7.1.0)\n",
            "Requirement already satisfied: SecretStorage>=3.2 in /usr/lib/python3/dist-packages (from keyring<25.0.0,>=24.0.0->poetry) (3.3.1)\n",
            "Requirement already satisfied: jeepney>=0.4.2 in /usr/lib/python3/dist-packages (from keyring<25.0.0,>=24.0.0->poetry) (0.7.1)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect<5.0.0,>=4.7.0->poetry) (0.7.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.26->poetry) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.26->poetry) (3.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.26->poetry) (2024.2.2)\n",
            "Collecting distlib<1,>=0.3.7 (from virtualenv<21.0.0,>=20.23.0->poetry)\n",
            "  Downloading distlib-0.3.8-py2.py3-none-any.whl (468 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.9/468.9 kB\u001b[0m \u001b[31m53.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.0.0->poetry) (3.18.1)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from jaraco.classes->keyring<25.0.0,>=24.0.0->poetry) (10.1.0)\n",
            "Installing collected packages: trove-classifiers, distlib, virtualenv, tomlkit, shellingham, rapidfuzz, poetry-core, pkginfo, jaraco.classes, installer, dulwich, crashtest, requests-toolbelt, keyring, cleo, poetry-plugin-export, poetry\n",
            "  Attempting uninstall: keyring\n",
            "    Found existing installation: keyring 23.5.0\n",
            "    Uninstalling keyring-23.5.0:\n",
            "      Successfully uninstalled keyring-23.5.0\n",
            "Successfully installed cleo-2.1.0 crashtest-0.4.1 distlib-0.3.8 dulwich-0.21.7 installer-0.7.0 jaraco.classes-3.3.1 keyring-24.3.1 pkginfo-1.10.0 poetry-1.8.2 poetry-core-1.9.0 poetry-plugin-export-1.7.1 rapidfuzz-3.7.0 requests-toolbelt-1.0.0 shellingham-1.5.4 tomlkit-0.12.4 trove-classifiers-2024.3.25 virtualenv-20.25.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://olmo-data.org/preprocessed/olmo-mix/v1_5/gpt-neox-20b-pii-special/part-000-00000.npy"
      ],
      "metadata": {
        "id": "60OKtG1-2PBe",
        "outputId": "02a4a2ba-7225-43a5-e366-3dd67510694f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-03-30 06:02:47--  https://olmo-data.org/preprocessed/olmo-mix/v1_5/gpt-neox-20b-pii-special/part-000-00000.npy\n",
            "Resolving olmo-data.org (olmo-data.org)... 172.67.175.213, 104.21.83.123, 2606:4700:3035::ac43:afd5, ...\n",
            "Connecting to olmo-data.org (olmo-data.org)|172.67.175.213|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 34359737144 (32G)\n",
            "Saving to: ‘part-000-00000.npy’\n",
            "\n",
            "part-000-00000.npy  100%[===================>]  32.00G  23.3MB/s    in 21m 2s  \n",
            "\n",
            "2024-03-30 06:23:52 (26.0 MB/s) - ‘part-000-00000.npy’ saved [34359737144/34359737144]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('OLMo')\n"
      ],
      "metadata": {
        "id": "lKY4kDAldmnQ"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -e .[all]"
      ],
      "metadata": {
        "id": "TmZDG6N81rPU",
        "outputId": "abe92030-0ba9-4df2-f225-baf2f75d843e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Obtaining file:///content/OLMo\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Checking if build backend supports build_editable ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build editable ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing editable metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (1.25.2)\n",
            "Requirement already satisfied: torch<2.3,>=2.0 in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (2.2.1+cu121)\n",
            "Collecting omegaconf (from ai2-olmo==0.3.0)\n",
            "  Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (13.7.1)\n",
            "Collecting boto3 (from ai2-olmo==0.3.0)\n",
            "  Downloading boto3-1.34.74-py3-none-any.whl (139 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: google-cloud-storage in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (2.8.0)\n",
            "Requirement already satisfied: tokenizers in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (0.15.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (24.0)\n",
            "Collecting cached-path>=1.6.2 (from ai2-olmo==0.3.0)\n",
            "  Downloading cached_path-1.6.2-py3-none-any.whl (35 kB)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (4.38.2)\n",
            "Requirement already satisfied: requests<3.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from cached-path>=1.6.2->ai2-olmo==0.3.0) (2.31.0)\n",
            "Requirement already satisfied: filelock<3.14,>=3.4 in /usr/local/lib/python3.10/dist-packages (from cached-path>=1.6.2->ai2-olmo==0.3.0) (3.13.3)\n",
            "Requirement already satisfied: huggingface-hub<0.22.0,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from cached-path>=1.6.2->ai2-olmo==0.3.0) (0.20.3)\n",
            "Collecting botocore<1.35.0,>=1.34.74 (from boto3->ai2-olmo==0.3.0)\n",
            "  Downloading botocore-1.34.74-py3-none-any.whl (12.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3->ai2-olmo==0.3.0)\n",
            "  Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
            "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3->ai2-olmo==0.3.0)\n",
            "  Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: google-auth<3.0dev,>=1.25.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->ai2-olmo==0.3.0) (2.27.0)\n",
            "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->ai2-olmo==0.3.0) (2.11.1)\n",
            "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->ai2-olmo==0.3.0) (2.3.3)\n",
            "Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->ai2-olmo==0.3.0) (2.7.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->ai2-olmo==0.3.0) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->ai2-olmo==0.3.0) (2.16.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m57.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m80.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m847.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch<2.3,>=2.0->ai2-olmo==0.3.0) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch<2.3,>=2.0->ai2-olmo==0.3.0)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m55.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ruff (from ai2-olmo==0.3.0)\n",
            "  Downloading ruff-0.3.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m61.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mypy<1.4,>=1.0 (from ai2-olmo==0.3.0)\n",
            "  Downloading mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.2/12.2 MB\u001b[0m \u001b[31m82.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting black<24.0,>=23.1 (from ai2-olmo==0.3.0)\n",
            "  Downloading black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m69.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting isort<5.13,>=5.12 (from ai2-olmo==0.3.0)\n",
            "  Downloading isort-5.12.0-py3-none-any.whl (91 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.2/91.2 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (7.4.4)\n",
            "Collecting pytest-sphinx (from ai2-olmo==0.3.0)\n",
            "  Downloading pytest_sphinx-0.6.0-py3-none-any.whl (10 kB)\n",
            "Collecting twine>=1.11.0 (from ai2-olmo==0.3.0)\n",
            "  Downloading twine-5.0.0-py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (67.7.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (0.43.0)\n",
            "Requirement already satisfied: build in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (1.2.1)\n",
            "Collecting wandb (from ai2-olmo==0.3.0)\n",
            "  Downloading wandb-0.16.5-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m76.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting beaker-gantry (from ai2-olmo==0.3.0)\n",
            "  Downloading beaker_gantry-0.22.2-py3-none-any.whl (30 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (8.1.7)\n",
            "Collecting torchmetrics (from ai2-olmo==0.3.0)\n",
            "  Downloading torchmetrics-1.3.2-py3-none-any.whl (841 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m841.5/841.5 kB\u001b[0m \u001b[31m55.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting smashed[remote]>=0.21.1 (from ai2-olmo==0.3.0)\n",
            "  Downloading smashed-0.21.5-py3-none-any.whl (95 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.4/95.4 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (0.4.2)\n",
            "Collecting datasets (from ai2-olmo==0.3.0)\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m45.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from ai2-olmo==0.3.0) (1.2.2)\n",
            "Collecting msgspec>=0.14.0 (from ai2-olmo==0.3.0)\n",
            "  Downloading msgspec-0.18.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m210.3/210.3 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting antlr4-python3-runtime==4.9.* (from omegaconf->ai2-olmo==0.3.0)\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: PyYAML>=5.1.0 in /usr/local/lib/python3.10/dist-packages (from omegaconf->ai2-olmo==0.3.0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->ai2-olmo==0.3.0) (2023.12.25)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers->ai2-olmo==0.3.0) (4.66.2)\n",
            "Collecting mypy-extensions>=0.4.3 (from black<24.0,>=23.1->ai2-olmo==0.3.0)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Collecting pathspec>=0.9.0 (from black<24.0,>=23.1->ai2-olmo==0.3.0)\n",
            "  Downloading pathspec-0.12.1-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: platformdirs>=2 in /usr/local/lib/python3.10/dist-packages (from black<24.0,>=23.1->ai2-olmo==0.3.0) (4.2.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from black<24.0,>=23.1->ai2-olmo==0.3.0) (2.0.1)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.74->boto3->ai2-olmo==0.3.0) (2.8.2)\n",
            "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.74->boto3->ai2-olmo==0.3.0) (2.0.7)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->ai2-olmo==0.3.0) (1.63.0)\n",
            "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->ai2-olmo==0.3.0) (3.20.3)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->ai2-olmo==0.3.0) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->ai2-olmo==0.3.0) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->ai2-olmo==0.3.0) (4.9)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage->ai2-olmo==0.3.0) (1.5.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->ai2-olmo==0.3.0) (0.1.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.0->cached-path>=1.6.2->ai2-olmo==0.3.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.0->cached-path>=1.6.2->ai2-olmo==0.3.0) (3.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0,>=2.0->cached-path>=1.6.2->ai2-olmo==0.3.0) (2024.2.2)\n",
            "Collecting necessary>=0.4.3 (from smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading necessary-0.4.3-py3-none-any.whl (10 kB)\n",
            "Collecting trouting>=0.3.3 (from smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading trouting-0.3.3-py3-none-any.whl (8.8 kB)\n",
            "Collecting ftfy>=6.1.1 (from smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading ftfy-6.2.0-py3-none-any.whl (54 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.4/54.4 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting glom>=21.0.0 (from smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading glom-23.5.0-py3-none-any.whl (102 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.7/102.7 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: smart-open>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from smashed[remote]>=0.21.1->ai2-olmo==0.3.0) (6.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch<2.3,>=2.0->ai2-olmo==0.3.0) (2.1.5)\n",
            "Requirement already satisfied: pkginfo>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from twine>=1.11.0->ai2-olmo==0.3.0) (1.10.0)\n",
            "Collecting readme-renderer>=35.0 (from twine>=1.11.0->ai2-olmo==0.3.0)\n",
            "  Downloading readme_renderer-43.0-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: requests-toolbelt!=0.9.0,>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from twine>=1.11.0->ai2-olmo==0.3.0) (1.0.0)\n",
            "Requirement already satisfied: importlib-metadata>=3.6 in /usr/local/lib/python3.10/dist-packages (from twine>=1.11.0->ai2-olmo==0.3.0) (7.1.0)\n",
            "Requirement already satisfied: keyring>=15.1 in /usr/local/lib/python3.10/dist-packages (from twine>=1.11.0->ai2-olmo==0.3.0) (24.3.1)\n",
            "Collecting rfc3986>=1.4.0 (from twine>=1.11.0->ai2-olmo==0.3.0)\n",
            "  Downloading rfc3986-2.0.0-py2.py3-none-any.whl (31 kB)\n",
            "Collecting beaker-py<2.0,>=1.26.2 (from beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading beaker_py-1.26.3-py3-none-any.whl (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.7/85.7 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting GitPython<4.0,>=3.0 (from beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading GitPython-3.1.42-py3-none-any.whl (195 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.4/195.4 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting click-help-colors (from beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading click_help_colors-0.9.4-py3-none-any.whl (6.4 kB)\n",
            "Collecting petname<3.0,>=2.6 (from beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading petname-2.6.tar.gz (8.4 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build->ai2-olmo==0.3.0) (1.0.0)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->ai2-olmo==0.3.0) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets->ai2-olmo==0.3.0) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->ai2-olmo==0.3.0)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->ai2-olmo==0.3.0) (1.5.3)\n",
            "Collecting xxhash (from datasets->ai2-olmo==0.3.0)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->ai2-olmo==0.3.0)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->ai2-olmo==0.3.0) (3.9.3)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.10/dist-packages (from pytest->ai2-olmo==0.3.0) (2.0.0)\n",
            "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from pytest->ai2-olmo==0.3.0) (1.4.0)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /usr/local/lib/python3.10/dist-packages (from pytest->ai2-olmo==0.3.0) (1.2.0)\n",
            "Collecting pytest (from ai2-olmo==0.3.0)\n",
            "  Downloading pytest-8.1.1-py3-none-any.whl (337 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m337.4/337.4 kB\u001b[0m \u001b[31m33.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->ai2-olmo==0.3.0) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->ai2-olmo==0.3.0) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->ai2-olmo==0.3.0) (3.4.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch<2.3,>=2.0->ai2-olmo==0.3.0) (1.3.0)\n",
            "Collecting lightning-utilities>=0.8.0 (from torchmetrics->ai2-olmo==0.3.0)\n",
            "  Downloading lightning_utilities-0.11.2-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->ai2-olmo==0.3.0) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb->ai2-olmo==0.3.0)\n",
            "  Downloading sentry_sdk-1.44.0-py2.py3-none-any.whl (264 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.9/264.9 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb->ai2-olmo==0.3.0)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting setproctitle (from wandb->ai2-olmo==0.3.0)\n",
            "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->ai2-olmo==0.3.0) (1.4.4)\n",
            "Requirement already satisfied: pydantic<3.0,>=1.8.2 in /usr/local/lib/python3.10/dist-packages (from beaker-py<2.0,>=1.26.2->beaker-gantry->ai2-olmo==0.3.0) (2.6.4)\n",
            "Collecting docker<7.0,>=5.0 (from beaker-py<2.0,>=1.26.2->beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading docker-6.1.3-py3-none-any.whl (148 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m148.1/148.1 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb->ai2-olmo==0.3.0) (1.16.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ai2-olmo==0.3.0) (4.0.3)\n",
            "Requirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /usr/local/lib/python3.10/dist-packages (from ftfy>=6.1.1->smashed[remote]>=0.21.1->ai2-olmo==0.3.0) (0.2.13)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython<4.0,>=3.0->beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting boltons>=19.3.0 (from glom>=21.0.0->smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading boltons-23.1.1-py2.py3-none-any.whl (195 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.3/195.3 kB\u001b[0m \u001b[31m25.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting face==20.1.1 (from glom>=21.0.0->smashed[remote]>=0.21.1->ai2-olmo==0.3.0)\n",
            "  Downloading face-20.1.1-py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=3.6->twine>=1.11.0->ai2-olmo==0.3.0) (3.18.1)\n",
            "Requirement already satisfied: jaraco.classes in /usr/local/lib/python3.10/dist-packages (from keyring>=15.1->twine>=1.11.0->ai2-olmo==0.3.0) (3.3.1)\n",
            "Requirement already satisfied: SecretStorage>=3.2 in /usr/lib/python3/dist-packages (from keyring>=15.1->twine>=1.11.0->ai2-olmo==0.3.0) (3.3.1)\n",
            "Requirement already satisfied: jeepney>=0.4.2 in /usr/lib/python3/dist-packages (from keyring>=15.1->twine>=1.11.0->ai2-olmo==0.3.0) (0.7.1)\n",
            "Requirement already satisfied: requirements-parser>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from necessary>=0.4.3->smashed[remote]>=0.21.1->ai2-olmo==0.3.0) (0.7.0)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-cloud-storage->ai2-olmo==0.3.0) (0.6.0)\n",
            "Collecting nh3>=0.2.14 (from readme-renderer>=35.0->twine>=1.11.0->ai2-olmo==0.3.0)\n",
            "  Downloading nh3-0.2.17-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (777 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m777.1/777.1 kB\u001b[0m \u001b[31m47.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: docutils>=0.13.1 in /usr/local/lib/python3.10/dist-packages (from readme-renderer>=35.0->twine>=1.11.0->ai2-olmo==0.3.0) (0.18.1)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->ai2-olmo==0.3.0) (2023.4)\n",
            "Requirement already satisfied: websocket-client>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from docker<7.0,>=5.0->beaker-py<2.0,>=1.26.2->beaker-gantry->ai2-olmo==0.3.0) (1.7.0)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython<4.0,>=3.0->beaker-gantry->ai2-olmo==0.3.0)\n",
            "  Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0,>=1.8.2->beaker-py<2.0,>=1.26.2->beaker-gantry->ai2-olmo==0.3.0) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0,>=1.8.2->beaker-py<2.0,>=1.26.2->beaker-gantry->ai2-olmo==0.3.0) (2.16.3)\n",
            "Requirement already satisfied: types-setuptools>=59.7.0 in /usr/local/lib/python3.10/dist-packages (from requirements-parser>=0.5.0->necessary>=0.4.3->smashed[remote]>=0.21.1->ai2-olmo==0.3.0) (69.2.0.20240317)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from jaraco.classes->keyring>=15.1->twine>=1.11.0->ai2-olmo==0.3.0) (10.1.0)\n",
            "Building wheels for collected packages: ai2-olmo, antlr4-python3-runtime, petname\n",
            "  Building editable for ai2-olmo (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ai2-olmo: filename=ai2_olmo-0.3.0-0.editable-py3-none-any.whl size=15496 sha256=cfb5fd5a24741dc8c1c6e1b85fc04b6f253a0d4d9143c8edd6f48fe049effcbc\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-u64e8usy/wheels/53/af/48/57981c6236493d577a46686beefbd325618762830eefe253f5\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=6ad12e299897f90342ff8100ab4c9b8dbe79b544864f9a010d1b60a8f031e9bb\n",
            "  Stored in directory: /root/.cache/pip/wheels/12/93/dd/1f6a127edc45659556564c5730f6d4e300888f4bca2d4c5a88\n",
            "  Building wheel for petname (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for petname: filename=petname-2.6-py3-none-any.whl size=8120 sha256=fb579ee94a768085f6f1d3c56ca4b74da744be4153967065aee125d44660f199\n",
            "  Stored in directory: /root/.cache/pip/wheels/a6/ea/67/b3166a5d29acff7207c1a19b4bc315b8bc4d58b46a6540ac7c\n",
            "Successfully built ai2-olmo antlr4-python3-runtime petname\n",
            "Installing collected packages: petname, nh3, boltons, antlr4-python3-runtime, xxhash, trouting, smmap, setproctitle, sentry-sdk, ruff, rfc3986, readme-renderer, pytest, pathspec, omegaconf, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, mypy-extensions, msgspec, lightning-utilities, jmespath, isort, ftfy, face, docker-pycreds, dill, click-help-colors, pytest-sphinx, nvidia-cusparse-cu12, nvidia-cudnn-cu12, necessary, mypy, multiprocess, glom, gitdb, docker, botocore, black, twine, smashed, s3transfer, nvidia-cusolver-cu12, GitPython, beaker-py, wandb, datasets, boto3, beaker-gantry, torchmetrics, cached-path, ai2-olmo\n",
            "  Attempting uninstall: pytest\n",
            "    Found existing installation: pytest 7.4.4\n",
            "    Uninstalling pytest-7.4.4:\n",
            "      Successfully uninstalled pytest-7.4.4\n",
            "Successfully installed GitPython-3.1.42 ai2-olmo-0.3.0 antlr4-python3-runtime-4.9.3 beaker-gantry-0.22.2 beaker-py-1.26.3 black-23.12.1 boltons-23.1.1 boto3-1.34.74 botocore-1.34.74 cached-path-1.6.2 click-help-colors-0.9.4 datasets-2.18.0 dill-0.3.8 docker-6.1.3 docker-pycreds-0.4.0 face-20.1.1 ftfy-6.2.0 gitdb-4.0.11 glom-23.5.0 isort-5.12.0 jmespath-1.0.1 lightning-utilities-0.11.2 msgspec-0.18.6 multiprocess-0.70.16 mypy-1.3.0 mypy-extensions-1.0.0 necessary-0.4.3 nh3-0.2.17 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105 omegaconf-2.3.0 pathspec-0.12.1 petname-2.6 pytest-8.1.1 pytest-sphinx-0.6.0 readme-renderer-43.0 rfc3986-2.0.0 ruff-0.3.4 s3transfer-0.10.1 sentry-sdk-1.44.0 setproctitle-1.3.3 smashed-0.21.5 smmap-5.0.1 torchmetrics-1.3.2 trouting-0.3.3 twine-5.0.0 wandb-0.16.5 xxhash-3.4.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pydevd_plugins"
                ]
              },
              "id": "881bd77223184b7d8150d73c9ce102c4"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!torchrun --nproc_per_node=8 scripts/train.py configs/official/OLMo-1B.yaml\n"
      ],
      "metadata": {
        "id": "P_Zos_9Edvtf",
        "outputId": "79de88aa-d03f-45b4-96ac-d7bcae850a99",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2024-03-30 06:00:22,623] torch.distributed.run: [WARNING] \n",
            "[2024-03-30 06:00:22,623] torch.distributed.run: [WARNING] *****************************************\n",
            "[2024-03-30 06:00:22,623] torch.distributed.run: [WARNING] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
            "[2024-03-30 06:00:22,623] torch.distributed.run: [WARNING] *****************************************\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")    \n",
            "dist.init_process_group(backend=\"nccl\")  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(Traceback (most recent call last):\n",
            "\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/OLMo/scripts/train.py\", line 251, in <module>\n",
            "    dist.init_process_group(backend=\"nccl\")\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/c10d_logger.py\", line 86, in wrapper\n",
            "    func_return = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1184, in init_process_group\n",
            "    default_pg, _ = _new_process_group_helper(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py\", line 1339, in _new_process_group_helper\n",
            "    backend_class = ProcessGroupNCCL(\n",
            "ValueError: ProcessGroupNCCL is only supported with GPUs, no GPUs found!\n",
            "[2024-03-30 06:01:32,768] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 10383) of binary: /usr/bin/python3\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/torchrun\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py\", line 347, in wrapper\n",
            "    return f(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/run.py\", line 812, in main\n",
            "    run(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/run.py\", line 803, in run\n",
            "    elastic_launch(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/launcher/api.py\", line 135, in __call__\n",
            "    return launch_agent(self._config, self._entrypoint, list(args))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/launcher/api.py\", line 268, in launch_agent\n",
            "    raise ChildFailedError(\n",
            "torch.distributed.elastic.multiprocessing.errors.ChildFailedError: \n",
            "============================================================\n",
            "scripts/train.py FAILED\n",
            "------------------------------------------------------------\n",
            "Failures:\n",
            "[1]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 1 (local_rank: 1)\n",
            "  exitcode  : 1 (pid: 10384)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[2]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 2 (local_rank: 2)\n",
            "  exitcode  : 1 (pid: 10385)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[3]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 3 (local_rank: 3)\n",
            "  exitcode  : 1 (pid: 10386)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[4]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 4 (local_rank: 4)\n",
            "  exitcode  : 1 (pid: 10387)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[5]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 5 (local_rank: 5)\n",
            "  exitcode  : 1 (pid: 10388)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[6]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 6 (local_rank: 6)\n",
            "  exitcode  : 1 (pid: 10389)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "[7]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 7 (local_rank: 7)\n",
            "  exitcode  : 1 (pid: 10390)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "------------------------------------------------------------\n",
            "Root Cause (first observed failure):\n",
            "[0]:\n",
            "  time      : 2024-03-30_06:01:32\n",
            "  host      : 735ce126e94e\n",
            "  rank      : 0 (local_rank: 0)\n",
            "  exitcode  : 1 (pid: 10383)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "============================================================\n"
          ]
        }
      ]
    }
  ]
}