Add model identity event fixtures

Author: Jordak

tests/fixtures/model_identity/claude/result-multimodel-usage-only.jsonl

@@ -0,0 +1 @@
+{"type":"result","subtype":"success","is_error":false,"modelUsage":{"claude-haiku-4-5-20251001":{},"claude-sonnet-4-6":{}},"usage":{}}

tests/fixtures/model_identity/claude/system-init-assistant-result.jsonl

@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","uuid":"00000000-0000-0000-0000-000000000000","session_id":"00000000-0000-0000-0000-000000000000","permissionMode":"acceptEdits","model":"claude-sonnet-4-6"}
+{"type":"assistant","uuid":"00000000-0000-0000-0000-000000000000","session_id":"00000000-0000-0000-0000-000000000000","message":{"id":"msg_sanitized","type":"message","role":"assistant","model":"claude-sonnet-4-6","content":[],"stop_reason":null,"stop_sequence":null,"usage":{}}}
+{"type":"result","subtype":"success","is_error":false,"modelUsage":{"claude-haiku-4-5-20251001":{},"claude-sonnet-4-6":{}},"usage":{}}

tests/fixtures/model_identity/codex/thread-turn-usage-only.jsonl

@@ -0,0 +1,4 @@
+{"type":"thread.started","thread_id":"00000000-0000-0000-0000-000000000000"}
+{"type":"turn.started"}
+{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"Sanitized agent progress update."}}
+{"type":"turn.completed","usage":{}}

tests/test_model_identity.py

@@ -1,9 +1,50 @@
 import unittest
+from pathlib import Path
 
 from agentlab.model_identity import model_identity_from_events
 
 
+FIXTURES = Path(__file__).parent / "fixtures" / "model_identity"
+
+
 class ModelIdentityTest(unittest.TestCase):
+    def test_captured_event_stream_fixtures_parse_model_identity(self):
+        cases = [
+            {
+                "name": "claude_code_system_init_assistant_message_and_result_usage",
+                "fixture": (
+                    FIXTURES / "claude" / "system-init-assistant-result.jsonl"
+                ),
+                "requested_model_name": "requested-sonnet",
+                "expected_model_name": "claude-sonnet-4-6",
+                "expected_model_source": "events",
+                "expected_requested_model_name": "requested-sonnet",
+            },
+            {
+                "name": "codex_thread_turn_completed_usage_without_runtime_model",
+                "fixture": FIXTURES / "codex" / "thread-turn-usage-only.jsonl",
+                "requested_model_name": "gpt-requested",
+                "expected_model_name": "gpt-requested",
+                "expected_model_source": "explicit",
+                "expected_requested_model_name": "gpt-requested",
+            },
+        ]
+
+        for case in cases:
+            with self.subTest(case["name"]):
+                events = case["fixture"].read_text(encoding="utf-8")
+                identity = model_identity_from_events(
+                    events,
+                    requested_model_name=case["requested_model_name"],
+                )
+
+                self.assertEqual(identity.model_name, case["expected_model_name"])
+                self.assertEqual(identity.model_source, case["expected_model_source"])
+                self.assertEqual(
+                    identity.requested_model_name,
+                    case["expected_requested_model_name"],
+                )
+
     def test_prefers_direct_event_model_over_requested_model(self):
         identity = model_identity_from_events(
             '{"type":"turn.completed","model":"actual-model"}\n',
@@ -33,6 +74,44 @@ def test_uses_single_model_usage_entry_as_last_resort(self):
         self.assertEqual(identity.model_name, "only-model")
         self.assertEqual(identity.model_source, "events")
 
+    def test_ambiguous_multimodel_usage_without_primary_model_is_unknown(self):
+        events = (
+            FIXTURES / "claude" / "result-multimodel-usage-only.jsonl"
+        ).read_text(encoding="utf-8")
+
+        identity = model_identity_from_events(events, requested_model_name=None)
+
+        self.assertIsNone(identity.model_name)
+        self.assertEqual(identity.model_source, "unknown")
+        self.assertIsNone(identity.requested_model_name)
+
+    def test_ambiguous_multimodel_usage_falls_back_to_requested_model(self):
+        events = (
+            FIXTURES / "claude" / "result-multimodel-usage-only.jsonl"
+        ).read_text(encoding="utf-8")
+
+        identity = model_identity_from_events(
+            events,
+            requested_model_name="requested-sonnet",
+        )
+
+        self.assertEqual(identity.model_name, "requested-sonnet")
+        self.assertEqual(identity.model_source, "explicit")
+        self.assertEqual(identity.requested_model_name, "requested-sonnet")
+
+    def test_malformed_and_irrelevant_lines_are_ignored_safely(self):
+        identity = model_identity_from_events(
+            "not json\n"
+            "[]\n"
+            '{"type":"turn.completed","usage":{"input_tokens":1}}\n'
+            '{"type":"turn.completed","model":"actual-model"}\n',
+            requested_model_name="requested-model",
+        )
+
+        self.assertEqual(identity.model_name, "actual-model")
+        self.assertEqual(identity.model_source, "events")
+        self.assertEqual(identity.requested_model_name, "requested-model")
+
     def test_falls_back_to_requested_model_when_events_do_not_identify_model(self):
         identity = model_identity_from_events(
             '{"type":"turn.completed","usage":{"input_tokens":1}}\n',

tests/test_results.py

@@ -9,6 +9,9 @@
 from agentlab.summary import summarize_trials
 
 
+MODEL_IDENTITY_FIXTURES = Path(__file__).parent / "fixtures" / "model_identity"
+
+
 class ResultsTest(unittest.TestCase):
     def test_file_artifacts_feed_trial_listing_summaries_reviews_and_evidence(self):
         with tempfile.TemporaryDirectory() as temp:
@@ -175,10 +178,11 @@ def test_load_results_backfills_model_identity_from_claude_events(self):
             run_dir = Path(temp) / "run-1"
             run_dir.mkdir()
             (run_dir / "claude-events.jsonl").write_text(
-                '{"type":"system","model":"claude-sonnet-4-6"}\n'
-                '{"type":"result","modelUsage":{'
-                '"claude-haiku-4-5-20251001":{"costUSD":0.1},'
-                '"claude-sonnet-4-6":{"costUSD":0.2}}}\n',
+                (
+                    MODEL_IDENTITY_FIXTURES
+                    / "claude"
+                    / "system-init-assistant-result.jsonl"
+                ).read_text(encoding="utf-8"),
                 encoding="utf-8",
             )
             result_path = run_dir / "result.json"
@@ -209,6 +213,45 @@ def test_load_results_backfills_model_identity_from_claude_events(self):
             )
             self.assertEqual(result["agent_harness_config"]["model_source"], "events")
 
+    def test_load_results_does_not_infer_model_from_codex_usage_only_fixture(self):
+        with tempfile.TemporaryDirectory() as temp:
+            run_dir = Path(temp) / "run-1"
+            run_dir.mkdir()
+            (run_dir / "codex-events.jsonl").write_text(
+                (
+                    MODEL_IDENTITY_FIXTURES
+                    / "codex"
+                    / "thread-turn-usage-only.jsonl"
+                ).read_text(encoding="utf-8"),
+                encoding="utf-8",
+            )
+            result_path = run_dir / "result.json"
+            result_path.write_text(
+                json.dumps(
+                    {
+                        "trial_kind": "agent_trial",
+                        "run_dir": str(run_dir),
+                        "agent_name": "codex",
+                        "model_name": "gpt-requested",
+                        "agent_harness_config": {
+                            "agent_harness": "codex",
+                            "agent_adapter": "codex_cli",
+                            "model_name": "gpt-requested",
+                            "model_source": "explicit",
+                        },
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            result = load_results([result_path])[0]
+
+            self.assertEqual(result["model_name"], "gpt-requested")
+            harness_config = result["agent_harness_config"]
+            self.assertEqual(harness_config["model_name"], "gpt-requested")
+            self.assertEqual(harness_config["model_source"], "explicit")
+            self.assertEqual(harness_config["requested_model_name"], "gpt-requested")
+
     def test_load_results_backfills_codex_event_model_over_requested_model(self):
         with tempfile.TemporaryDirectory() as temp:
             run_dir = Path(temp) / "run-1"