darwinpatch/tests/test_darwinpatch_phase3.py at main · taneemishere/darwinpatch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import json
from pathlib import Path

from darwinpatch.benchmark import run_benchmark_suite
from darwinpatch.models import SearchBudget


ROOT = Path(__file__).resolve().parents[1]


def test_benchmark_suite_writes_summary_outputs(tmp_path):
    suite_path = tmp_path / "suite.json"
    suite_path.write_text(
        json.dumps(
            {
                "name": "test_suite",
                "cases": [
                    {
                        "id": "needs_repair",
                        "task_dir": str(ROOT / "tasks" / "smoke_markdown_parser"),
                        "candidate_patches": ["candidate_visible_fail.patch", "candidate.patch"],
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    payload = run_benchmark_suite(
        suite_path=suite_path,
        run_dir=tmp_path / "benchmark",
        baselines=("single_shot", "full_darwinpatch"),
        budget=SearchBudget(max_candidates=2, max_repeated_failures=2),
    )

    assert payload["summary"]["single_shot"]["solve_at_budget"] == 0.0
    assert payload["summary"]["full_darwinpatch"]["solve_at_budget"] == 1.0
    assert payload["summary"]["full_darwinpatch"]["evidence_packets_admitted"] == 1
    assert (tmp_path / "benchmark" / "benchmark_results.json").exists()
    assert (tmp_path / "benchmark" / "benchmark_summary.md").exists()


def test_evidence_aware_review_uses_bounded_evidence_and_metadata(tmp_path):
    suite_path = tmp_path / "suite.json"
    suite_path.write_text(
        json.dumps(
            {
                "name": "route_metadata_suite",
                "cases": [
                    {
                        "id": "visible_route_decoy",
                        "task_dir": str(ROOT / "tasks" / "smoke_markdown_parser"),
                        "candidate_patches": [
                            {
                                "patch": "candidate_visible_fail.patch",
                                "label": "first_pass",
                                "intent": "first_pass_attempt",
                                "compatible_routes": [],
                            },
                            {
                                "patch": "candidate_scope_fail.patch",
                                "label": "scope_decoy",
                                "intent": "wrong_route_attempt",
                                "compatible_routes": ["scope_repair"],
                            },
                            {
                                "patch": "candidate.patch",
                                "label": "behavior_repair",
                                "intent": "complete_spec_repair",
                                "compatible_routes": ["behavior_repair"],
                            },
                        ],
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    payload = run_benchmark_suite(
        suite_path=suite_path,
        run_dir=tmp_path / "benchmark",
        baselines=("linear_retry", "evidence_aware_review", "full_darwinpatch"),
        budget=SearchBudget(max_candidates=2, max_repeated_failures=2),
    )

    assert payload["summary"]["linear_retry"]["solve_at_budget"] == 0.0
    assert payload["summary"]["evidence_aware_review"]["solve_at_budget"] == 1.0
    assert payload["summary"]["evidence_aware_review"]["metadata_route_matches"] == 1
    assert payload["summary"]["evidence_aware_review"]["evidence_packets_admitted"] == 1
    assert payload["summary"]["full_darwinpatch"]["solve_at_budget"] == 1.0