eval tui

configs/eval/debug.toml

@@ -1,8 +1,16 @@
-model = "openai/gpt-5-mini"
+model = "openai/gpt-4.1-mini"
+save_results = true
+save_every = 10
 
 [[eval]]
 env_id = "primeintellect/wiki-search"
 
+[[eval]]
+env_id = "gsm8k"
+num_examples = 20
+rollouts_per_example = 1
+sampling_args = { max_tokens = 1024 }
+independent_scoring = true
 
 [[eval]]
-env_id = "primeintellect/math-python"
+env_id = "alphabet-sort"

configs/eval/duplicate-env.toml

@@ -0,0 +1,6 @@
+[[eval]]
+env_id = "alphabet-sort"
+
+[[eval]]
+env_id = "alphabet-sort"
+max_concurrent = 1

configs/eval/single-turn.toml

@@ -0,0 +1,20 @@
+[[eval]]
+env_id = "math500"
+num_examples = -1
+rollouts_per_example = 1
+
+[[eval]]
+env_id = "aime2024"
+num_examples = -1
+rollouts_per_example = 8
+
+[[eval]]
+env_id = "gpqa"
+num_examples = -1
+rollouts_per_example = 1
+
+[[eval]]
+env_id = "livecodebench"
+num_examples = -1
+rollouts_per_example = 1
+max_concurrent = 16 # to limit sandbox usage

docs/evaluation.md

@@ -135,6 +135,7 @@ The `--max-retries` flag enables automatic retry with exponential backoff when r
 | Flag | Short | Default | Description |
 |------|-------|---------|-------------|
 | `--verbose` | `-v` | false | Enable debug logging |
+| `--tui` | — | false | Show live-updating TUI for multi-env evals |
 | `--save-results` | `-s` | false | Save results to disk |
 | `--save-every` | `-f` | -1 | Save checkpoint every N rollouts |
 | `--state-columns` | `-C` | — | Extra state columns to save (comma-separated) |

tests/test_eval_cli.py

@@ -111,7 +111,6 @@ def _run_cli(monkeypatch, overrides, capture_all_configs: bool = False):
         "temperature": 0.9,
         "sampling_args": None,
         "verbose": False,
-        "print_results": False,
         "no_interleave_scoring": False,
         "state_columns": [],
         "save_results": False,
@@ -120,6 +119,7 @@ def _run_cli(monkeypatch, overrides, capture_all_configs: bool = False):
         "hf_hub_dataset_name": "",
         "extra_env_kwargs": {},
         "max_retries": 0,
+        "tui": False,
     }
     base_args.update(overrides)
     args_namespace = SimpleNamespace(**base_args)
@@ -134,7 +134,7 @@ def _run_cli(monkeypatch, overrides, capture_all_configs: bool = False):
     monkeypatch.setattr(vf_eval, "setup_logging", lambda *_, **__: None)
     monkeypatch.setattr(vf_eval, "load_endpoints", lambda *_: {})
 
-    async def fake_run_evaluation(config):
+    async def fake_run_evaluation(config, **kwargs):
         captured["sampling_args"] = dict(config.sampling_args)
         captured["configs"].append(config)
         metadata = _make_metadata(config)

verifiers/envs/environment.py

@@ -37,12 +37,15 @@
     DatasetBuilder,
     GenerateMetadata,
     GenerateOutputs,
+    LogCallback,
     Messages,
     MessageType,
     ModelResponse,
+    ProgressCallback,
     RolloutInput,
     RolloutTiming,
     SamplingArgs,
+    StartCallback,
     State,
 )
 from verifiers.utils.async_utils import maybe_retry, maybe_semaphore
@@ -866,6 +869,9 @@ async def generate(
         use_tqdm: bool = True,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        on_start: StartCallback | None = None,
+        on_progress: ProgressCallback | None = None,
+        on_log: LogCallback | None = None,
     ) -> GenerateOutputs:
         """
         Generate rollouts for a set of inputs.
@@ -875,6 +881,10 @@ async def generate(
         elif isinstance(inputs, list):
             inputs_list = inputs
 
+        # notify caller of actual total count (useful when num_examples=-1)
+        if on_start is not None:
+            on_start(len(inputs_list))
+
         # resolve concurrency knobs
         gen_limit = max_concurrent_generation
         score_limit = max_concurrent_scoring
@@ -936,9 +946,9 @@ async def generate(
             pbar_total = len(group_list)
             pbar_desc = f"Processing {len(group_list)} groups ({len(inputs_list)} total rollouts)"
 
-        # set up progress bar
+        # set up progress bar (only when use_tqdm=True and no external progress callback)
         pbar = None
-        if use_tqdm:
+        if use_tqdm and on_progress is None:
             from tqdm import tqdm
 
             pbar = tqdm(total=pbar_total, desc=pbar_desc, postfix=dict(reward="?"))
@@ -962,10 +972,13 @@ async def generate(
                         reward_sum += r
                         reward_count += 1
 
+                # update progress bar or call callback
                 if pbar is not None:
                     pbar.update(1)
                     if reward_count > 0:
                         pbar.set_postfix(reward=f"{reward_sum / reward_count:.3f}")
+                elif on_progress is not None:
+                    on_progress(all_states, states)
 
                 # save intermediate results
                 if (
@@ -1003,9 +1016,11 @@ async def generate(
             start_time,
         )
 
-        # Save if requested
+        # save if requested
         if save_results:
             save_rollout_results(results)
+            if on_log is not None:
+                on_log(f"Saved final results to {results['metadata']['path_to_save']}")
 
         return results
 
@@ -1070,8 +1085,12 @@ async def evaluate(
         state_columns: list[str] | None = None,
         save_results: bool = False,
         save_every: int = -1,
+        use_tqdm: bool = True,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        on_start: StartCallback | None = None,
+        on_progress: ProgressCallback | None = None,
+        on_log: LogCallback | None = None,
         **kwargs,
     ) -> GenerateOutputs:
         """
@@ -1090,8 +1109,12 @@ async def evaluate(
             state_columns=state_columns,
             save_results=save_results,
             save_every=save_every,
+            use_tqdm=use_tqdm,
             independent_scoring=independent_scoring,
             max_retries=max_retries,
+            on_start=on_start,
+            on_progress=on_progress,
+            on_log=on_log,
             **kwargs,
         )
 

verifiers/scripts/eval.py

@@ -18,6 +18,7 @@
     load_endpoints,
     load_toml_config,
     run_evaluations,
+    run_evaluations_tui,
 )
 from verifiers.utils.install_utils import check_hub_env_installed
 
@@ -253,6 +254,13 @@ def main():
         default={},
         help='Extra environment as JSON object (e.g., \'{"key": "value", "num": 42}\'). Passed to environment constructor.',
     )
+    parser.add_argument(
+        "--tui",
+        "-u",
+        default=False,
+        action="store_true",
+        help="Use TUI mode for live evaluation display",
+    )
     parser.add_argument(
         "--max-retries",
         type=int,
@@ -423,7 +431,10 @@ def build_eval_config(raw: dict) -> EvalConfig:
         logger.debug(f"Evaluation config: {config.model_dump_json(indent=2)}")
 
     eval_run_config = EvalRunConfig(evals=eval_configs)
-    asyncio.run(run_evaluations(eval_run_config))
+    if args.tui:
+        asyncio.run(run_evaluations_tui(eval_run_config))
+    else:
+        asyncio.run(run_evaluations(eval_run_config))
 
 
 if __name__ == "__main__":

verifiers/types.py

@@ -142,6 +142,11 @@ def get(self, key: str, default: Any = None) -> Any:
 # oai tools
 JsonPrimitive = Literal["string", "number", "integer", "boolean", "array", "object"]
 
+# callbacks
+StartCallback = Callable[[int], None]  # total rollouts
+ProgressCallback = Callable[[list[State], list[State]], None]  # all_states, new_states
+LogCallback = Callable[[str], None]  # log messages
+
 
 class GenerateMetadata(TypedDict):
     """Pydantic model for generation metadata."""
@@ -241,6 +246,7 @@ class EvalConfig(BaseModel):
     max_retries: int = 0
     # logging
     verbose: bool = False
+    use_tqdm: bool = True
     # saving
     state_columns: list[str] | None = None
     save_results: bool = False