refactor(models): (PR 1/4) isolate plugin registration failures

.github/hermetic-cpu-requirements.txt

@@ -0,0 +1,174 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --extra server --group dev --python-version 3.11 --python-platform x86_64-unknown-linux-gnu --no-annotate --output-file .github/hermetic-cpu-requirements.txt
+accelerate==1.14.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.3
+aiohttp-jinja2==1.6
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+antlr4-python3-runtime==4.7.2
+anyio==4.14.2
+apted==1.0.3
+attrs==26.1.0
+av==15.1.0
+black==26.5.1
+certifi==2026.7.22
+cfgv==3.5.0
+chardet==6.0.0.post1
+charset-normalizer==3.5.1
+click==8.4.2
+colorama==0.4.6
+dataproperty==1.1.1
+datasets==5.0.1
+defusedxml==0.7.1
+dill==0.4.1
+distance==0.1.3
+distlib==0.4.3
+editdistance==0.8.1
+einops==0.8.2
+et-xmlfile==2.0.0
+evaluate==0.4.6
+fastapi==0.141.1
+filelock==3.32.3
+frozenlist==1.8.0
+fsspec==2026.6.0
+ftfy==6.3.1
+gitdb==4.0.12
+gitpython==3.1.59
+h11==0.16.0
+hf-transfer==0.1.9
+hf-xet==1.6.0
+httpcore==1.0.9
+httpcore2==2.12.0
+httpx==0.28.1
+httpx2==2.12.0
+huggingface-hub==1.28.0
+identify==2.6.19
+idna==3.19
+iniconfig==2.3.0
+isort==8.0.1
+jieba==0.42.1
+jinja2==3.1.6
+jiter==0.16.0
+joblib==1.5.3
+jsonlines==4.0.0
+latex2sympy2==1.9.1
+levenshtein==0.27.4
+linkify-it-py==2.1.0
+loguru==0.7.3
+lxml==6.1.2
+markdown-it-py==4.2.0
+markupsafe==3.0.3
+math-verify==0.1.0
+mbstrdecoder==1.1.5
+mdit-py-plugins==0.6.1
+mdurl==0.1.2
+mpmath==1.3.0
+msgpack==1.2.1
+multidict==6.7.1
+multiprocess==0.70.19
+mypy-extensions==1.1.0
+narwhals==2.25.0
+networkx==3.6.1
+nltk==3.10.3
+nodeenv==1.10.0
+numexpr==2.14.2
+numpy==2.4.6
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparselt-cu12==0.6.2
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+openai==3.3.1
+opencv-python-headless==5.0.0.93
+openpyxl==3.1.5
+packaging==26.3
+pandas==3.0.5
+pathspec==1.1.1
+pathvalidate==3.3.1
+peft==0.20.0
+pillow==12.3.0
+platformdirs==4.11.3
+pluggy==1.6.0
+portalocker==4.2.0
+pre-commit==4.6.2
+propcache==0.5.2
+protobuf==6.33.6
+psutil==7.2.2
+pyarrow==25.0.1
+pybind11==3.1.0
+pycocoevalcap==1.2
+pycocotools==2.0.11
+pydantic==2.13.4
+pydantic-core==2.46.4
+pygments==2.21.0
+pytablewriter==1.2.1
+pytest==9.1.1
+python-dateutil==2.9.0.post0
+python-discovery==1.5.2
+python-dotenv==1.2.3
+pytokens==0.4.1
+pytz==2026.3.post1
+pyyaml==6.0.3
+qwen-vl-utils==0.0.14
+rapidfuzz==3.14.5
+regex==2026.7.19
+requests==2.34.2
+rich==15.0.0
+sacrebleu==2.6.0
+safetensors==0.8.0
+scikit-learn==1.9.0
+scipy==1.17.1
+sentence-transformers==6.0.0
+sentencepiece==0.2.2
+sentry-sdk==2.68.0
+setuptools==84.0.0
+shellingham==1.5.4
+six==1.17.0
+smmap==5.0.3
+sniffio==1.3.1
+starlette==1.6.0
+sympy==1.13.1
+tabledata==1.3.5
+tabulate==0.10.0
+tcolorpy==0.1.7
+tenacity==9.1.4
+textual==8.2.8
+textual-dev==1.8.0
+textual-serve==1.1.3
+threadpoolctl==3.6.0
+tiktoken==0.14.0
+timm==1.0.28
+tokenizers==0.22.2
+torch==2.6.0
+torchvision==0.21.0
+tqdm==4.70.0
+tqdm-multiprocess==0.0.11
+transformers==5.15.1
+transformers-stream-generator==0.0.5
+triton==3.2.0
+truststore==0.10.4
+typepy==1.3.5
+typer==0.27.1
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+uc-micro-py==2.0.0
+urllib3==2.7.0
+uvicorn==0.52.4
+virtualenv==21.7.4
+wandb==0.25.0
+wcwidth==0.8.2
+xxhash==4.0.1
+yarl==1.24.5
+yt-dlp==2026.8.19
+zss==1.2.0
+zstandard==0.25.0

.github/workflows/hermetic-cpu-contracts.yml

@@ -0,0 +1,47 @@
+name: Hermetic CPU Contracts
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  hermetic-cpu-contracts:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      PYTHONPATH: "."
+      CUDA_VISIBLE_DEVICES: ""
+      HF_HUB_OFFLINE: "1"
+      HF_DATASETS_OFFLINE: "1"
+      TRANSFORMERS_OFFLINE: "1"
+      WANDB_MODE: disabled
+      OPENAI_API_KEY: lmms-eval-ci-placeholder
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.7.3"
+      - name: Create Python environment
+        run: uv venv --python 3.11
+      - name: Sync pinned CPU contract dependencies
+        run: uv pip sync .github/hermetic-cpu-requirements.txt
+      - name: Collect all tests
+        run: uv run --no-sync python -m pytest --collect-only -q
+      - name: Run hermetic CPU contracts
+        run: >-
+          uv run --no-sync python -m pytest -q -m "not gpu and not api"
+          test/cli/test_cli_dispatch_parametrized.py
+          test/models/test_model_registry_v2.py
+          test/eval/test_protocol.py
+          test/eval/test_request_construction_contract.py
+          test/eval/test_task_pipeline.py
+          test/eval/test_evaluator.py
+          test/entrypoints/test_job_scheduler_subprocess.py

lmms_eval/models/registry_v2.py

@@ -45,6 +45,15 @@ def class_name(self) -> str:
         return self.class_path.rsplit(".", 1)[-1]
 
 
+@dataclass(frozen=True)
+class PluginLoadFailure:
+    """A model entry point that could not be loaded or registered."""
+
+    source: str
+    error_type: str
+    message: str
+
+
 class ModelRegistryV2:
     """Canonical model registry with aliasing and typed resolution semantics."""
 
@@ -67,17 +76,36 @@ def register_manifest(
         raise `ValueError`.
         """
 
-        merged = self._merge_manifest(self._manifests.get(manifest.model_id), manifest, overwrite=overwrite)
-        self._manifests[manifest.model_id] = merged
+        self.register_manifests((manifest,), overwrite=overwrite)
 
-        names = (merged.model_id, *merged.aliases)
-        for name in names:
-            existing = self._alias_to_model_id.get(name)
-            if existing and existing != merged.model_id and not overwrite:
-                raise ValueError(
-                    f"Alias '{name}' already points to '{existing}', cannot remap to '{merged.model_id}'",
-                )
-            self._alias_to_model_id[name] = merged.model_id
+    def register_manifests(
+        self,
+        manifests: Iterable[ModelManifest],
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        """Atomically register a batch of manifests and their aliases."""
+
+        candidate_manifests = self._manifests.copy()
+        candidate_names = self._alias_to_model_id.copy()
+        for manifest in manifests:
+            merged = self._merge_manifest(
+                candidate_manifests.get(manifest.model_id),
+                manifest,
+                overwrite=overwrite,
+            )
+            for name in (merged.model_id, *merged.aliases):
+                owner = candidate_names.get(name)
+                if owner is not None and owner != merged.model_id and not overwrite:
+                    raise ValueError(
+                        f"Model name '{name}' already points to '{owner}', " f"cannot remap to '{merged.model_id}'",
+                    )
+            candidate_manifests[merged.model_id] = merged
+            for name in (merged.model_id, *merged.aliases):
+                candidate_names[name] = merged.model_id
+
+        self._manifests = candidate_manifests
+        self._alias_to_model_id = candidate_names
 
     def resolve(self, model_name: str, force_simple: bool = False) -> ResolvedModel:
         """Resolve a model name to one concrete implementation class path."""
@@ -141,7 +169,7 @@ def load_entrypoint_manifests(
         group: str = "lmms_eval.models",
         *,
         overwrite: bool = False,
-    ) -> None:
+    ) -> tuple[PluginLoadFailure, ...]:
         """Load model manifests from Python entry points.
 
         Supported payloads per entry point:
@@ -150,12 +178,23 @@ def load_entrypoint_manifests(
         - `Callable[[], ModelManifest | Iterable[ModelManifest]]`
         """
 
-        selected = self._select_entry_points(group)
+        failures: list[PluginLoadFailure] = []
+        selected = sorted(
+            self._select_entry_points(group),
+            key=lambda ep: (ep.name, getattr(ep, "value", "")),
+        )
         for ep in selected:
-            payload = ep.load()
-            manifests = self._coerce_payload_to_manifests(payload)
-            for manifest in manifests:
-                self.register_manifest(manifest, overwrite=overwrite)
+            source = f"{ep.name} ({getattr(ep, 'value', '')})"
+            try:
+                self.register_manifests(
+                    self._coerce_payload_to_manifests(ep.load()),
+                    overwrite=overwrite,
+                )
+            except Exception as exc:
+                failures.append(
+                    PluginLoadFailure(source, type(exc).__name__, str(exc)),
+                )
+        return tuple(failures)
 
     def list_model_names(self) -> list[str]:
         """Return all known requestable model names (ids + aliases)."""
@@ -167,6 +206,11 @@ def list_canonical_model_ids(self) -> list[str]:
 
         return sorted(self._manifests)
 
+    def list_manifests(self) -> list[ModelManifest]:
+        """Return all canonical manifests without importing model classes."""
+
+        return [self._manifests[model_id] for model_id in self.list_canonical_model_ids()]
+
     def get_manifest(self, model_name: str) -> ModelManifest:
         """Return canonical manifest for a model id or alias."""
 

lmms_eval/tasks/vlmsareblind/utils.py

@@ -106,15 +106,8 @@ def vlmsareblind_aggregate_by_task(results: list[dict]) -> dict[str, float]:
         if is_correct:
             task_correct[task] += 1
 
-    task_accuracy = {
-        task: task_correct[task] / total
-        for task, total in task_total.items()
-    }
-    task_accuracy["task_mean"] = (
-        sum(task_accuracy.values()) / len(task_accuracy)
-        if task_accuracy
-        else 0.0
-    )
+    task_accuracy = {task: task_correct[task] / total for task, total in task_total.items()}
+    task_accuracy["task_mean"] = sum(task_accuracy.values()) / len(task_accuracy) if task_accuracy else 0.0
 
     return task_accuracy
 

pyproject.toml

@@ -74,6 +74,11 @@ dependencies = [
     "python-dotenv",
     "qwen-vl-utils>=0.0.14",
     "math-verify",
+    "jieba>=0.42.1",
+    "distance>=0.1.3",
+    "editdistance>=0.8.1",
+    "Levenshtein>=0.27.4",
+    "apted>=1.0.3",
 ]
 
 [project.optional-dependencies]
@@ -210,6 +215,7 @@ Repository = "https://github.com/EvolvingLMMs-Lab/lmms-eval"
 
 [dependency-groups]
 dev = [
+    "pytest>=8.0",
     "textual>=7.2.0",
     "textual-dev>=1.8.0",
 ]

test/eval/qwen2_5_vl/test_qwen2_5_vl.py

@@ -3,11 +3,19 @@
 """
 
 import unittest
+from test.eval.utils import (
+    get_gpu_count,
+    run_evaluation_test,
+    with_server,
+    with_temp_dir,
+)
 from unittest import TestCase
 
-from utils import get_gpu_count, run_evaluation_test, with_server, with_temp_dir
+import pytest
 
 
+@pytest.mark.gpu
+@pytest.mark.slow
 class TestQwen2_5_VL(TestCase):
     """Test cases for Qwen2.5-VL model evaluation."""