fix: isolate WordPress runtime state per execution test

README.md

@@ -84,7 +84,9 @@ run:
   limit: 10                  # limit tests (null = all)
   test_ids: []               # optional explicit test IDs to run
   dry_run: false             # load/filter tests without calling models
-  concurrency: 4
+  concurrency: 4             # model-call concurrency (knowledge tests)
+  execution_isolation: reset_per_test  # reset WordPress before each execution test
+  execution_concurrency: 1   # must stay 1 under reset_per_test isolation
 
 output:
   path: output/results.json

python/tests/test_execution_isolation.py

@@ -0,0 +1,212 @@
+"""Tests for execution-test state isolation (reset_per_test strategy)."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from wp_bench.config import (
+    DatasetConfig,
+    GraderConfig,
+    HarnessConfig,
+    ModelConfig,
+    OutputConfig,
+    RunConfig,
+)
+from wp_bench.core import BenchmarkRunner, MultiModelRunner
+from wp_bench.datasets import ExecutionTest
+from wp_bench.environment import ExecutionResult
+
+
+def _execution_test(test_id: str) -> ExecutionTest:
+    return ExecutionTest(
+        id=test_id,
+        suite="wp-core-v1",
+        prompt="Prompt",
+        expected_behavior="expected",
+        test_type="execution",
+        category="general",
+        difficulty="basic",
+        requirements=["Requirement"],
+        test_function=None,
+        static_checks={},
+        runtime_checks={"assertions": [{"type": "custom_assertion", "code": "return true;", "weight": 1}]},
+        reference_solution="function ref() { return true; }",
+        metadata={},
+    )
+
+
+def _passing_result() -> ExecutionResult:
+    raw = {
+        "success": True,
+        "static": {"score": 1.0, "details": {"total_weight": 1}},
+        "runtime": {"score": 1.0, "details": {"total_weight": 1}},
+    }
+    return ExecutionResult(success=True, raw=raw, stdout="", stderr="")
+
+
+def _config(tmp_path: Path, **run_overrides: Any) -> HarnessConfig:
+    return HarnessConfig(
+        dataset=DatasetConfig(source="local", name="wp-core-v1"),
+        model=ModelConfig(name="test-model"),
+        grader=GraderConfig(kind="cli"),
+        run=RunConfig(test_type="execution", **run_overrides),
+        output=OutputConfig(path=tmp_path / "results.json", jsonl_path=None),
+    )
+
+
+class SpyEnvironment:
+    """Records the interleaving of reset and execute calls."""
+
+    def __init__(self) -> None:
+        self.calls: list[str] = []
+
+    def setup(self) -> None:
+        self.calls.append("setup")
+
+    def reset(self) -> None:
+        self.calls.append("reset")
+
+    def execute_code(self, code: str, verification_spec: dict) -> ExecutionResult:
+        self.calls.append("execute")
+        return _passing_result()
+
+
+def test_execution_runner_resets_between_tests(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """Every execution test is preceded by an environment reset."""
+    tests = [_execution_test("e-one"), _execution_test("e-two"), _execution_test("e-three")]
+    monkeypatch.setattr(
+        "wp_bench.core.load_tests",
+        lambda dataset: {"execution": tests, "knowledge": []},
+    )
+    config = _config(tmp_path)
+    runner = BenchmarkRunner(config)
+    spy = SpyEnvironment()
+    runner.environment = spy  # type: ignore[assignment]
+    monkeypatch.setattr(runner.model, "generate", lambda prompt: "```php\ncode\n```")
+
+    runner.run()
+
+    assert spy.calls == [
+        "setup",
+        "reset", "execute",
+        "reset", "execute",
+        "reset", "execute",
+    ]
+
+
+def test_reference_solution_mode_resets_between_tests(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """Reference-solution mode uses the same isolation path."""
+    tests = [_execution_test("e-one"), _execution_test("e-two")]
+    monkeypatch.setattr(
+        "wp_bench.core.load_tests",
+        lambda dataset: {"execution": tests, "knowledge": []},
+    )
+    config = _config(tmp_path, check_reference_solution=True)
+    runner = BenchmarkRunner(config)
+    spy = SpyEnvironment()
+    runner.environment = spy  # type: ignore[assignment]
+
+    runner.run()
+
+    assert spy.calls == ["setup", "reset", "execute", "reset", "execute"]
+
+
+def test_multi_model_runner_resets_between_models(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """State from one model run cannot leak into the next model's tests."""
+    tests = [_execution_test("e-one")]
+    monkeypatch.setattr(
+        "wp_bench.core.load_tests",
+        lambda dataset: {"execution": tests, "knowledge": []},
+    )
+    config = HarnessConfig(
+        dataset=DatasetConfig(source="local", name="wp-core-v1"),
+        models=[ModelConfig(name="model-a"), ModelConfig(name="model-b")],
+        grader=GraderConfig(kind="cli"),
+        run=RunConfig(test_type="execution"),
+        output=OutputConfig(path=tmp_path / "results.json", jsonl_path=None),
+    )
+    runner = MultiModelRunner(config)
+    spy = SpyEnvironment()
+    runner.environment = spy  # type: ignore[assignment]
+    monkeypatch.setattr(
+        "wp_bench.core.ModelInterface",
+        lambda model_config: type(
+            "FakeModel", (), {"generate": staticmethod(lambda prompt: "```php\ncode\n```")}
+        )(),
+    )
+
+    runner.run()
+
+    # Each model's only test is preceded by its own reset: no state carries over.
+    assert spy.calls == ["setup", "reset", "execute", "reset", "execute"]
+
+
+def test_concurrency_above_one_rejected_for_reset_per_test() -> None:
+    """reset_per_test isolation cannot support concurrent execution tests."""
+    with pytest.raises(ValueError, match="execution_concurrency must be 1"):
+        RunConfig(execution_isolation="reset_per_test", execution_concurrency=4)
+
+
+def test_isolation_none_allows_concurrency() -> None:
+    """Legacy concurrent mode is available only by explicit opt-out."""
+    config = RunConfig(execution_isolation="none", execution_concurrency=4)
+    assert config.execution_concurrency == 4
+
+
+def test_execution_concurrency_must_be_positive() -> None:
+    with pytest.raises(ValueError, match="must be >= 1"):
+        RunConfig(execution_isolation="none", execution_concurrency=0)
+
+
+def test_result_metadata_records_isolation_mode(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """Isolation strategy is auditable from result metadata."""
+    monkeypatch.setattr(
+        "wp_bench.core.load_tests",
+        lambda dataset: {"execution": [_execution_test("e-one")], "knowledge": []},
+    )
+    config = _config(tmp_path)
+    runner = BenchmarkRunner(config)
+    spy = SpyEnvironment()
+    runner.environment = spy  # type: ignore[assignment]
+    monkeypatch.setattr(runner.model, "generate", lambda prompt: "```php\ncode\n```")
+
+    payload = runner.run()
+
+    assert payload["metadata"]["runtime_isolation"] == "reset_per_test"
+
+
+def test_isolation_none_still_runs_all_tests(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """Opting out of isolation preserves the legacy concurrent path."""
+    tests = [_execution_test("e-one"), _execution_test("e-two")]
+    monkeypatch.setattr(
+        "wp_bench.core.load_tests",
+        lambda dataset: {"execution": tests, "knowledge": []},
+    )
+    config = _config(tmp_path, execution_isolation="none", execution_concurrency=2)
+    runner = BenchmarkRunner(config)
+    spy = SpyEnvironment()
+    runner.environment = spy  # type: ignore[assignment]
+    monkeypatch.setattr(runner.model, "generate", lambda prompt: "```php\ncode\n```")
+
+    payload = runner.run()
+
+    assert spy.calls.count("execute") == 2
+    assert spy.calls.count("reset") == 0
+    assert payload["metadata"]["runtime_isolation"] == "none"

python/wp_bench/config.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import List, Literal, Optional
 
-from pydantic import BaseModel, Field, HttpUrl, validator
+from pydantic import BaseModel, Field, HttpUrl, model_validator, validator
 
 ArtifactKind = Literal[
     "php_snippet",
@@ -44,23 +44,50 @@ class GraderConfig(BaseModel):
     image: str = "ghcr.io/wordpress/wp-bench-grader:latest"
     container_name: str = "wp-bench-grader"
     url: Optional[HttpUrl] = None
+    base_url: str = "http://localhost:8888"
     concurrency: int = 4
     timeout_seconds: int = 90
     wp_env_dir: Optional[Path] = None
 
 
+ExecutionIsolation = Literal["reset_per_test", "none"]
+
+
 class RunConfig(BaseModel):
     suite: str = "wp-core-v1"
     test_type: Optional[Literal["knowledge", "execution"]] = None
     limit: Optional[int] = None
     test_ids: List[str] = Field(default_factory=list)
     seed: int = 1337
     concurrency: int = 5
+    execution_isolation: ExecutionIsolation = "reset_per_test"
+    execution_concurrency: int = 1
     dry_run: bool = False
     check_reference_solution: bool = False
     skip_runtime: bool = False
     skip_static: bool = False
 
+    @model_validator(mode="after")
+    def _validate_execution_concurrency(self) -> "RunConfig":
+        """Reject concurrency the isolation strategy cannot support.
+
+        ``reset_per_test`` isolation resets one shared WordPress runtime
+        before every execution test, which is only sound when execution
+        tests run serially. Fail loudly instead of silently sharing mutable
+        WordPress state across concurrent tests.
+        """
+        if self.execution_concurrency < 1:
+            raise ValueError("run.execution_concurrency must be >= 1")
+        if self.execution_isolation == "reset_per_test" and self.execution_concurrency > 1:
+            raise ValueError(
+                "run.execution_concurrency must be 1 when "
+                "run.execution_isolation is 'reset_per_test': concurrent tests "
+                "would share one mutable WordPress runtime. Set "
+                "run.execution_isolation to 'none' to opt out of isolation "
+                "(not valid for official benchmark runs)."
+            )
+        return self
+
 
 class OutputConfig(BaseModel):
     path: Path = Path("results.json")