Refactor CLI into command-family package

Author: Jordak

agentlab/cli.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Iterable
+
+from agentlab.cli.agent_options import (
+    _agent_factory,
+    _add_claude_options,
+    _claude_code_config_from_args,
+    _codex_config_from_args,
+)
+from agentlab.cli.doctor import handle_doctor
+from agentlab.cli.parser import build_parser
+from agentlab.cli.reports import handle_report_capability_evidence_digest
+from agentlab.cli.review import handle_review
+from agentlab.cli.run import handle_run, _print_run_summaries
+from agentlab.cli.task import (
+    handle_task_smoke_test,
+    handle_task_validate,
+    handle_task_verify_reference,
+)
+from agentlab.cli.trials import (
+    handle_runs_list,
+    handle_trials_archive_excluded,
+    handle_trials_summarize,
+)
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(list(argv) if argv is not None else None)
+
+    if not hasattr(args, "handler"):
+        parser.print_help()
+        return 2
+
+    return args.handler(args)
+
+
+__all__ = [
+    "_agent_factory",
+    "_add_claude_options",
+    "_claude_code_config_from_args",
+    "_codex_config_from_args",
+    "_print_run_summaries",
+    "build_parser",
+    "handle_doctor",
+    "handle_report_capability_evidence_digest",
+    "handle_review",
+    "handle_run",
+    "handle_runs_list",
+    "handle_task_smoke_test",
+    "handle_task_validate",
+    "handle_task_verify_reference",
+    "handle_trials_archive_excluded",
+    "handle_trials_summarize",
+    "main",
+]

agentlab/cli/__init__.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import argparse
+from typing import Callable, Sequence
+
+from agentlab.agents.claude_code import ClaudeCodeAdapter, ClaudeCodeConfig
+from agentlab.agents.codex_cli import CodexCliAdapter, CodexCliConfig
+from agentlab.agents.manual import ManualAgentAdapter
+
+
+AGENT_CHOICES = ("manual", "codex", "claude")
+PREFLIGHT_AGENT_CHOICES = ("codex", "claude")
+
+
+def add_agent_argument(
+    parser: argparse.ArgumentParser,
+    *,
+    default: str,
+    choices: Sequence[str],
+    help: str,
+) -> None:
+    parser.add_argument(
+        "--agent",
+        default=default,
+        choices=list(choices),
+        help=help,
+    )
+
+
+def add_agent_adapter_options(
+    parser: argparse.ArgumentParser,
+    *,
+    codex_timeout_default: int,
+    codex_timeout_help: str = "Maximum wall time for `codex exec`.",
+    claude_timeout_default: int,
+) -> None:
+    add_codex_options(
+        parser,
+        timeout_default=codex_timeout_default,
+        timeout_help=codex_timeout_help,
+    )
+    add_claude_options(parser, timeout_default=claude_timeout_default)
+
+
+def add_codex_options(
+    parser: argparse.ArgumentParser,
+    *,
+    timeout_default: int,
+    timeout_help: str,
+) -> None:
+    parser.add_argument(
+        "--codex-command",
+        default="codex",
+        help="Codex CLI executable to use when --agent codex.",
+    )
+    parser.add_argument(
+        "--codex-model",
+        default=None,
+        help="Optional model passed to `codex exec --model`.",
+    )
+    parser.add_argument(
+        "--codex-profile",
+        default=None,
+        help="Optional profile passed to `codex exec --profile`.",
+    )
+    parser.add_argument(
+        "--codex-sandbox",
+        default="workspace-write",
+        choices=["read-only", "workspace-write", "danger-full-access"],
+        help="Sandbox mode passed to `codex exec --sandbox`.",
+    )
+    parser.add_argument(
+        "--codex-approval",
+        default="never",
+        choices=["untrusted", "on-failure", "on-request", "never"],
+        help="Approval policy passed to `codex exec --ask-for-approval`.",
+    )
+    parser.add_argument(
+        "--codex-timeout-seconds",
+        type=int,
+        default=timeout_default,
+        help=timeout_help,
+    )
+
+
+def add_claude_options(
+    parser: argparse.ArgumentParser,
+    *,
+    timeout_default: int,
+) -> None:
+    parser.add_argument(
+        "--claude-command",
+        default="claude",
+        help="Claude Code CLI executable to use when --agent claude.",
+    )
+    parser.add_argument(
+        "--claude-model",
+        default=None,
+        help="Optional model passed to `claude --model`.",
+    )
+    parser.add_argument(
+        "--claude-permission-mode",
+        default="acceptEdits",
+        choices=[
+            "default",
+            "acceptEdits",
+            "plan",
+            "auto",
+            "dontAsk",
+            "bypassPermissions",
+        ],
+        help="Permission mode passed to Claude Code.",
+    )
+    parser.add_argument(
+        "--claude-output-format",
+        default="stream-json",
+        choices=["text", "json", "stream-json"],
+        help="Print-mode output format passed to Claude Code.",
+    )
+    parser.add_argument(
+        "--claude-max-turns",
+        type=int,
+        default=None,
+        help="Optional maximum agentic turns for Claude Code print mode.",
+    )
+    parser.add_argument(
+        "--claude-allowed-tool",
+        action="append",
+        default=[],
+        help=(
+            "Tool permission rule passed as `--allowedTools`. Can be repeated, "
+            "for example `Read` or `Bash(pytest *)`."
+        ),
+    )
+    parser.add_argument(
+        "--claude-disallowed-tool",
+        action="append",
+        default=[],
+        help=(
+            "Tool permission rule passed as `--disallowedTools`. Can be "
+            "repeated, for example `Bash(git push *)`."
+        ),
+    )
+    parser.add_argument(
+        "--claude-timeout-seconds",
+        type=int,
+        default=timeout_default,
+        help="Maximum wall time for `claude -p`.",
+    )
+    parser.add_argument(
+        "--claude-session-persistence",
+        action="store_true",
+        help=(
+            "Allow Claude Code to persist its local session. By default "
+            "agentlab passes `--no-session-persistence` for isolated trials."
+        ),
+    )
+
+
+_add_claude_options = add_claude_options
+
+
+def _build_agent(args: argparse.Namespace, show_progress: bool = True) -> object:
+    if args.agent == "manual":
+        return ManualAgentAdapter(pause=not args.no_pause)
+    if args.agent == "codex":
+        return CodexCliAdapter(_codex_config_from_args(args, show_progress))
+    if args.agent == "claude":
+        return ClaudeCodeAdapter(_claude_code_config_from_args(args, show_progress))
+    raise RuntimeError(f"unknown agent: {args.agent}")
+
+
+def _codex_config_from_args(
+    args: argparse.Namespace,
+    show_progress: bool = True,
+) -> CodexCliConfig:
+    return CodexCliConfig(
+        command=args.codex_command,
+        model=args.codex_model,
+        profile=args.codex_profile,
+        sandbox=args.codex_sandbox,
+        approval_policy=args.codex_approval,
+        timeout_seconds=args.codex_timeout_seconds,
+        show_progress=show_progress,
+    )
+
+
+def _claude_code_config_from_args(
+    args: argparse.Namespace,
+    show_progress: bool = True,
+) -> ClaudeCodeConfig:
+    return ClaudeCodeConfig(
+        command=args.claude_command,
+        model=args.claude_model,
+        permission_mode=args.claude_permission_mode,
+        output_format=args.claude_output_format,
+        max_turns=args.claude_max_turns,
+        allowed_tools=tuple(args.claude_allowed_tool),
+        disallowed_tools=tuple(args.claude_disallowed_tool),
+        timeout_seconds=args.claude_timeout_seconds,
+        show_progress=show_progress,
+        no_session_persistence=not args.claude_session_persistence,
+    )
+
+
+def _agent_factory(args: argparse.Namespace) -> Callable[[bool], object]:
+    def build(show_progress: bool) -> object:
+        return _build_agent(args, show_progress=show_progress)
+
+    return build

agentlab/cli/agent_options.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import argparse
+import shlex
+import sys
+
+from agentlab.agents.claude_code import run_claude_code_preflight
+from agentlab.agents.codex_cli import run_codex_preflight
+from agentlab.cli.agent_options import (
+    PREFLIGHT_AGENT_CHOICES,
+    add_agent_adapter_options,
+    add_agent_argument,
+    _claude_code_config_from_args,
+    _codex_config_from_args,
+)
+from agentlab.cli.output import _trim_cli_output
+from agentlab.terminal import print_error
+
+
+def add_doctor_command(subcommands: argparse._SubParsersAction) -> None:
+    doctor_parser = subcommands.add_parser(
+        "doctor",
+        help="Check local agent harness prerequisites before running trials.",
+    )
+    add_agent_argument(
+        doctor_parser,
+        default="codex",
+        choices=PREFLIGHT_AGENT_CHOICES,
+        help="Agent harness to check.",
+    )
+    add_agent_adapter_options(
+        doctor_parser,
+        codex_timeout_default=15,
+        codex_timeout_help="Maximum wall time for each Codex preflight command.",
+        claude_timeout_default=15,
+    )
+    doctor_parser.set_defaults(handler=handle_doctor)
+
+
+def handle_doctor(args: argparse.Namespace) -> int:
+    if args.agent == "codex":
+        result = run_codex_preflight(
+            _codex_config_from_args(args, show_progress=False),
+            timeout_seconds=args.codex_timeout_seconds,
+        )
+    elif args.agent == "claude":
+        result = run_claude_code_preflight(
+            _claude_code_config_from_args(args, show_progress=False),
+            timeout_seconds=args.claude_timeout_seconds,
+        )
+    else:
+        print_error(f"unknown agent: {args.agent}")
+        return 1
+
+    _print_preflight_result(result)
+    return 0 if result.passed else 1
+
+
+def _print_preflight_result(result: object) -> None:
+    print(f"Doctor: {result.agent_name}", flush=True)
+    for check in result.checks:
+        if check.passed:
+            print(f"OK {check.name}: {check.message}")
+            continue
+
+        print_error(f"{check.name}: {check.message}")
+        if check.command:
+            print(f"  Command: {shlex.join(check.command)}", file=sys.stderr)
+        output = _trim_cli_output(check.stderr or check.stdout)
+        if output:
+            print(output, file=sys.stderr)
+
+    if result.passed:
+        print("Preflight passed.")
+    else:
+        print("Preflight failed.", file=sys.stderr)

agentlab/cli/doctor.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import List
+
+
+def _print_table(headers: List[str], rows: List[List[str]]) -> None:
+    widths = [
+        max(len(str(row[column])) for row in [headers] + rows)
+        for column in range(len(headers))
+    ]
+    print("  ".join(header.ljust(widths[index]) for index, header in enumerate(headers)))
+    print("  ".join("-" * width for width in widths))
+    for row in rows:
+        print("  ".join(str(cell).ljust(widths[index]) for index, cell in enumerate(row)))
+
+
+def _trim_cli_output(output: str, max_chars: int = 1000) -> str:
+    output = output.strip()
+    if len(output) <= max_chars:
+        return output
+    return output[-max_chars:]

agentlab/cli/output.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import argparse
+
+from agentlab.cli.doctor import add_doctor_command
+from agentlab.cli.reports import add_report_commands
+from agentlab.cli.review import add_review_command
+from agentlab.cli.run import add_run_command
+from agentlab.cli.task import add_task_commands
+from agentlab.cli.trials import add_trial_commands
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agentlab",
+        description="Evaluate coding agents on reproducible software tasks.",
+    )
+    subcommands = parser.add_subparsers(dest="command")
+
+    add_task_commands(subcommands)
+    add_doctor_command(subcommands)
+    add_run_command(subcommands)
+    add_trial_commands(subcommands)
+    add_report_commands(subcommands)
+    add_review_command(subcommands)
+
+    return parser