forked from omnigent-ai/omnigent
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_parser.py
More file actions
3635 lines (3120 loc) · 126 KB
/
Copy pathtest_parser.py
File metadata and controls
3635 lines (3120 loc) · 126 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Tests for omnigent.spec.parser."""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from omnigent.errors import OmnigentError
from omnigent.spec.parser import discover_host_skills, parse
from omnigent.spec.types import ApiKeyAuth, DatabricksAuth, ProviderAuth
@pytest.fixture()
def agent_dir(tmp_path: Path) -> Path:
"""Create a minimal valid agent image directory."""
config = {"spec_version": 1, "name": "test-agent"}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
return tmp_path
def test_parse_minimal(agent_dir: Path) -> None:
spec = parse(agent_dir)
assert spec.spec_version == 1
assert spec.name == "test-agent"
assert spec.description is None
assert spec.llm is None
assert spec.interaction.conversational is True
assert spec.interaction.modalities.input == ["text"]
assert spec.interaction.modalities.output == ["text"]
assert spec.tools.agents == []
assert spec.params == {}
assert spec.instructions is None
assert spec.skills == []
assert spec.mcp_servers == []
assert spec.local_tools == []
assert spec.sub_agents == []
def test_parse_missing_config_yaml(tmp_path: Path) -> None:
with pytest.raises(FileNotFoundError, match=r"config.yaml not found"):
parse(tmp_path)
def test_parse_non_mapping_config(tmp_path: Path) -> None:
(tmp_path / "config.yaml").write_text("- just a list")
with pytest.raises(OmnigentError, match=r"must be a YAML mapping"):
parse(tmp_path)
def test_parse_missing_spec_version(tmp_path: Path) -> None:
(tmp_path / "config.yaml").write_text(yaml.dump({"name": "no-version"}))
with pytest.raises(OmnigentError, match=r"missing required field: spec_version"):
parse(tmp_path)
def test_parse_full_config(tmp_path: Path) -> None:
config = {
"spec_version": 1,
"name": "full-agent",
"description": "A fully configured agent.",
"llm": {
"model": "openai/gpt-5.4",
"max_completion_tokens": 4096,
"reasoning_effort": "medium",
},
"interaction": {
"conversational": True,
"modalities": {
"input": ["text", "image", "file"],
"output": ["text"],
},
},
"tools": {"agents": ["researcher", "critic"]},
"params": {"max_results": 10, "prefer_recent": True},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.name == "full-agent"
assert spec.description == "A fully configured agent."
assert spec.llm is not None
assert spec.llm.model == "openai/gpt-5.4"
# executor.model is the canonical source — verify consolidation
assert spec.executor.model == "openai/gpt-5.4"
assert spec.llm.extra == {
"max_completion_tokens": 4096,
"reasoning_effort": "medium",
}
assert spec.interaction.conversational is True
assert spec.interaction.modalities.input == ["text", "image", "file"]
assert spec.interaction.modalities.output == ["text"]
assert spec.tools.agents == ["researcher", "critic"]
assert spec.params == {"max_results": 10, "prefer_recent": True}
def test_parse_llm_missing_model(tmp_path: Path) -> None:
config = {"spec_version": 1, "llm": {"max_completion_tokens": 100}}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
with pytest.raises(OmnigentError, match=r"missing required field: model"):
parse(tmp_path)
def test_parse_llm_arbitrary_extra_keys(tmp_path: Path) -> None:
"""All non-model keys in the llm block are collected into extra."""
config = {
"spec_version": 1,
"llm": {
"model": "anthropic/claude-sonnet-4-20250514",
"temperature": 0.7,
"top_p": 0.9,
"max_tokens": 2048,
"stop": ["\n\n"],
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.llm is not None
assert spec.llm.model == "anthropic/claude-sonnet-4-20250514"
assert spec.llm.extra == {
"temperature": 0.7,
"top_p": 0.9,
"max_tokens": 2048,
"stop": ["\n\n"],
}
def test_parse_llm_model_only(tmp_path: Path) -> None:
"""LLM block with only model has empty extra and no connection."""
config = {"spec_version": 1, "llm": {"model": "openai/gpt-4o"}}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.llm is not None
assert spec.llm.model == "openai/gpt-4o"
assert spec.llm.extra == {}
assert spec.llm.connection is None
def test_parse_llm_connection_block(tmp_path: Path) -> None:
"""The connection sub-block is parsed into LLMConfig.connection."""
config = {
"spec_version": 1,
"llm": {
"model": "databricks/databricks-gpt-5-4",
"temperature": 0.5,
"connection": {
"api_key": "dapi_test_key",
"base_url": "https://my-workspace.databricks.com/serving-endpoints",
},
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.llm is not None
assert spec.llm.model == "databricks/databricks-gpt-5-4"
assert spec.llm.extra == {"temperature": 0.5}
assert spec.llm.connection == {
"api_key": "dapi_test_key",
"base_url": "https://my-workspace.databricks.com/serving-endpoints",
}
def test_parse_llm_connection_expands_env_vars(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``${VAR}`` references in connection values are expanded."""
monkeypatch.setenv("MY_API_KEY", "sk-secret-123")
config = {
"spec_version": 1,
"llm": {
"model": "openai/gpt-5.4",
"connection": {"api_key": "${MY_API_KEY}"},
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.llm is not None
assert spec.llm.connection == {"api_key": "sk-secret-123"}
def test_parse_llm_connection_unresolved_var_raises(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Unresolved ``${VAR}`` in LLM connection raises ValueError.
:param tmp_path: Temporary directory for config files.
:param monkeypatch: Pytest monkeypatch for env vars.
"""
monkeypatch.delenv("MY_API_KEY", raising=False)
config = {
"spec_version": 1,
"llm": {
"model": "openai/gpt-4o",
"connection": {"api_key": "${MY_API_KEY}"},
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
with pytest.raises(OmnigentError, match=r"Unresolved environment variable"):
parse(tmp_path)
def test_parse_expand_env_false_keeps_var_references(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``expand_env=False`` keeps ``${VAR}`` references as literal strings.
Used during scaffolding/validation (e.g. ``omnigent create``) where
env vars may not yet be set in the current process.
"""
monkeypatch.delenv("MY_API_KEY", raising=False)
config = {
"spec_version": 1,
"llm": {
"model": "openai/gpt-4o",
"connection": {"api_key": "${MY_API_KEY}"},
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path, expand_env=False)
assert spec.llm is not None
assert spec.llm.connection == {"api_key": "${MY_API_KEY}"}
def test_parse_instructions_multiline_inline(tmp_path: Path) -> None:
"""Multiline inline instructions are not treated as file paths."""
config = {
"spec_version": 1,
"instructions": "Line one.\nLine two.\nLine three.",
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.instructions == "Line one.\nLine two.\nLine three."
def test_parse_agents_md_fallback(agent_dir: Path) -> None:
"""No instructions key in config -> falls back to AGENTS.md."""
(agent_dir / "AGENTS.md").write_text("You are a helpful research assistant.")
spec = parse(agent_dir)
assert spec.instructions == "You are a helpful research assistant."
def test_parse_instructions_inline(tmp_path: Path) -> None:
"""instructions key with inline text (not a file path)."""
config = {"spec_version": 1, "instructions": "Be concise and helpful."}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.instructions == "Be concise and helpful."
def test_parse_instructions_file_reference(agent_dir: Path) -> None:
"""instructions key pointing to an existing file."""
(agent_dir / "SYSTEM.md").write_text("Custom system prompt from file.")
config = {"spec_version": 1, "name": "test-agent", "instructions": "SYSTEM.md"}
(agent_dir / "config.yaml").write_text(yaml.dump(config))
spec = parse(agent_dir)
assert spec.instructions == "Custom system prompt from file."
def test_parse_instructions_rejects_path_traversal(tmp_path: Path) -> None:
"""An ``instructions`` value escaping the bundle is treated as literal text.
A crafted/uploaded bundle could set ``instructions: ../secret.txt`` to make
the runner read a file outside the bundle root and fold it into the agent's
system prompt (W7 spec-injection). The parser must NOT read an out-of-root
target — it falls back to treating the value as inline text, so the file's
contents never enter the spec. If this regresses, ``spec.instructions``
would contain the secret file's body.
"""
secret = tmp_path / "secret.txt"
secret.write_text("TOP SECRET RUNNER FILE")
bundle = tmp_path / "bundle"
bundle.mkdir()
config = {"spec_version": 1, "name": "evil", "instructions": "../secret.txt"}
(bundle / "config.yaml").write_text(yaml.dump(config))
spec = parse(bundle)
# The out-of-root target is never read — its contents must not leak.
assert "TOP SECRET" not in (spec.instructions or "")
# Falls back to the literal value (the existing "missing file → inline" path).
assert spec.instructions == "../secret.txt"
def test_parse_instructions_overrides_agents_md(agent_dir: Path) -> None:
"""Explicit instructions key takes precedence over AGENTS.md."""
(agent_dir / "AGENTS.md").write_text("Fallback instructions.")
config = {"spec_version": 1, "name": "test-agent", "instructions": "Inline wins."}
(agent_dir / "config.yaml").write_text(yaml.dump(config))
spec = parse(agent_dir)
assert spec.instructions == "Inline wins."
def test_parse_instructions_file_overrides_agents_md(agent_dir: Path) -> None:
"""instructions pointing to a file takes precedence over AGENTS.md."""
(agent_dir / "AGENTS.md").write_text("Fallback instructions.")
(agent_dir / "CUSTOM.md").write_text("Custom file wins.")
config = {"spec_version": 1, "name": "test-agent", "instructions": "CUSTOM.md"}
(agent_dir / "config.yaml").write_text(yaml.dump(config))
spec = parse(agent_dir)
assert spec.instructions == "Custom file wins."
def test_parse_prompt_alias_inline(tmp_path: Path) -> None:
"""``prompt:`` is an alias for ``instructions:`` (inline text)."""
config = {"spec_version": 1, "prompt": "Be concise and helpful."}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
# Without the alias, ``prompt:`` is ignored and instructions falls
# back to None (no AGENTS.md here) — the silent generic-prompt bug.
assert spec.instructions == "Be concise and helpful."
def test_parse_prompt_alias_multiline(tmp_path: Path) -> None:
"""A multiline ``prompt:`` block (the nessie config shape) loads."""
config = {
"spec_version": 1,
"name": "nessie-like",
"prompt": "You are an orchestrator.\nNever merge.\nDecompose first.",
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
assert spec.instructions == ("You are an orchestrator.\nNever merge.\nDecompose first.")
def test_parse_prompt_alias_file_reference(agent_dir: Path) -> None:
"""``prompt:`` honors the same file-path resolution as instructions."""
(agent_dir / "SYSTEM.md").write_text("Prompt body from file.")
config = {"spec_version": 1, "name": "test-agent", "prompt": "SYSTEM.md"}
(agent_dir / "config.yaml").write_text(yaml.dump(config))
spec = parse(agent_dir)
assert spec.instructions == "Prompt body from file."
def test_parse_prompt_alias_overrides_agents_md(agent_dir: Path) -> None:
"""``prompt:`` is consulted before the AGENTS.md auto-detect scan."""
(agent_dir / "AGENTS.md").write_text("Fallback instructions.")
config = {"spec_version": 1, "name": "test-agent", "prompt": "Prompt wins."}
(agent_dir / "config.yaml").write_text(yaml.dump(config))
spec = parse(agent_dir)
assert spec.instructions == "Prompt wins."
def test_parse_instructions_wins_over_prompt(tmp_path: Path) -> None:
"""When both keys are set, ``instructions:`` takes precedence."""
config = {
"spec_version": 1,
"instructions": "Canonical instructions.",
"prompt": "Legacy prompt alias.",
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
# Precedence lock: ``instructions:`` is the canonical key and carries
# file-path resolution; ``prompt:`` only fills in when it's absent.
assert spec.instructions == "Canonical instructions."
def test_auto_detect_agents_md_first_priority(agent_dir: Path) -> None:
"""AGENTS.md is chosen over CLAUDE.md and .cursorrules."""
(agent_dir / "AGENTS.md").write_text("FROM AGENTS")
(agent_dir / "CLAUDE.md").write_text("FROM CLAUDE")
(agent_dir / ".cursorrules").write_text("FROM CURSORRULES")
spec = parse(agent_dir)
assert spec.instructions == "FROM AGENTS"
def test_auto_detect_claude_md_when_no_agents_md(agent_dir: Path) -> None:
"""CLAUDE.md is chosen when AGENTS.md is absent."""
(agent_dir / "CLAUDE.md").write_text("FROM CLAUDE")
(agent_dir / ".cursorrules").write_text("FROM CURSORRULES")
spec = parse(agent_dir)
assert spec.instructions == "FROM CLAUDE"
def test_auto_detect_cursorrules_when_others_absent(agent_dir: Path) -> None:
""".cursorrules is chosen when AGENTS.md and CLAUDE.md are absent."""
(agent_dir / ".cursorrules").write_text("FROM CURSORRULES")
spec = parse(agent_dir)
assert spec.instructions == "FROM CURSORRULES"
def test_auto_detect_none_when_no_context_files(agent_dir: Path) -> None:
"""No context files present → instructions is None."""
spec = parse(agent_dir)
assert spec.instructions is None
def test_parse_skill(agent_dir: Path) -> None:
skill_dir = agent_dir / "skills" / "deep-search"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\n"
"name: deep-search\n"
"description: Search the web for sources.\n"
"---\n"
"When asked to research, use search.web."
)
spec = parse(agent_dir)
assert len(spec.skills) == 1
skill = spec.skills[0]
assert skill.name == "deep-search"
assert skill.description == "Search the web for sources."
assert skill.content == "When asked to research, use search.web."
assert skill.skill_dir == skill_dir
def test_parse_skill_missing_frontmatter(agent_dir: Path) -> None:
skill_dir = agent_dir / "skills" / "bad"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("No frontmatter here.")
with pytest.raises(OmnigentError, match=r"missing YAML frontmatter"):
parse(agent_dir)
def test_parse_skill_missing_name(agent_dir: Path) -> None:
skill_dir = agent_dir / "skills" / "no-name"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("---\ndescription: Missing name.\n---\nContent.")
with pytest.raises(OmnigentError, match=r"missing required field 'name'"):
parse(agent_dir)
def test_parse_skill_missing_description(agent_dir: Path) -> None:
skill_dir = agent_dir / "skills" / "no-desc"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("---\nname: no-desc\n---\nContent.")
with pytest.raises(OmnigentError, match=r"missing required field 'description'"):
parse(agent_dir)
# Reproduces the exact ``argument-hint:`` line from the upstream
# Claude Code skill at
# https://github.com/databricks-field-eng/vibe/blob/main/plugins/fe-databricks-tools/skills/databricks-data-generation/SKILL.md
# which broke ``omnigent --harness codex`` REPL launch before the
# host-skill scanner was made tolerant. YAML reads ``[industry]``
# as a flow sequence and then chokes on the trailing ``[--rows N]``.
_UPSTREAM_BAD_ARGUMENT_HINT = (
"argument-hint: [industry] [--rows N] [--catalog NAME] [--schema NAME]"
)
def test_parse_skill_invalid_yaml_frontmatter_in_bundle_raises(
agent_dir: Path,
) -> None:
"""
Agent-bundle skills are shipped with the spec and stay strict —
a YAML parse error in the bundle's own ``skills/`` directory
must fail loud, not silently drop the skill. ``parse()`` calls
``_discover_skills`` without the ``strict=False`` opt-in, so
this test also pins the default behavior.
"""
skill_dir = agent_dir / "skills" / "bad-yaml"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
f"---\nname: bad-yaml\ndescription: x\n{_UPSTREAM_BAD_ARGUMENT_HINT}\n---\nContent."
)
with pytest.raises(OmnigentError, match=r"invalid YAML frontmatter"):
parse(agent_dir)
def test_discover_host_skills_skips_invalid_yaml_frontmatter(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Host skill directories are user-managed (``~/.claude/skills/``,
``.claude/skills/``) and may contain third-party skills whose
frontmatter doesn't strictly parse as YAML. This test uses the
literal upstream ``argument-hint:`` line from the
``databricks-data-generation`` Claude Code skill — the exact
string that aborted ``omnigent --harness codex`` REPL launch
in production.
One bad skill must not break REPL launch: it must be logged
(with the file path so the user knows what to fix and the YAML
error so the cause is clear) and skipped, while the remaining
skills continue to load.
``discover_host_skills`` scans two locations: walking up from
``agent_root`` and ``Path.home() / ".claude" / "skills"``. We
pin ``$HOME`` at a fresh tmp dir to keep the developer's real
``~/.claude/skills/`` (which contains the actual offending
skill) out of this test.
"""
fake_home = tmp_path / "home"
fake_home.mkdir()
monkeypatch.setenv("HOME", str(fake_home))
agent_root = tmp_path / "agent"
agent_root.mkdir()
host_skills = agent_root / ".claude" / "skills"
host_skills.mkdir(parents=True)
bad_dir = host_skills / "bad-skill"
bad_dir.mkdir()
bad_md = bad_dir / "SKILL.md"
bad_md.write_text(
f"---\nname: bad-skill\ndescription: x\n{_UPSTREAM_BAD_ARGUMENT_HINT}\n---\nContent."
)
good_dir = host_skills / "good-skill"
good_dir.mkdir()
(good_dir / "SKILL.md").write_text("---\nname: good-skill\ndescription: y\n---\nContent.")
with caplog.at_level("WARNING", logger="omnigent.spec.parser"):
result = discover_host_skills(agent_root, "all")
names = [s.name for s in result]
assert names == ["good-skill"], (
"tolerant host-skill scan must drop the bad skill but keep "
"every other skill in the same directory"
)
skip_records = [rec for rec in caplog.records if "Skipping skill" in rec.message]
assert len(skip_records) == 1, "exactly one skip warning expected — one per bad skill"
msg = skip_records[0].message
# Warning must name the offending file so the user can fix it,
# and must surface the YAML parser error so the cause is clear.
assert str(bad_md) in msg, msg
assert "invalid YAML frontmatter" in msg, msg
def test_discover_host_skills_skips_unreadable_skill_file(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
File IO errors (broken symlink, permission denied) on a host
``SKILL.md`` must funnel through the same tolerant path as
malformed-frontmatter errors. A user with a stray broken
symlink under ``~/.claude/skills/`` must not see the whole
REPL launch abort.
"""
fake_home = tmp_path / "home"
fake_home.mkdir()
monkeypatch.setenv("HOME", str(fake_home))
agent_root = tmp_path / "agent"
agent_root.mkdir()
host_skills = agent_root / ".claude" / "skills"
host_skills.mkdir(parents=True)
# Broken symlink: ``SKILL.md`` exists (in the sense that
# ``Path.exists()`` follows symlinks and returns False, but the
# discoverer's ``skill_md.exists()`` check returns False too).
# Use a directory we make read-then-unreadable instead so the
# path exists but read_text() raises OSError.
bad_dir = host_skills / "unreadable"
bad_dir.mkdir()
bad_md = bad_dir / "SKILL.md"
bad_md.write_text("---\nname: unreadable\ndescription: x\n---\nbody")
bad_md.chmod(0o000)
good_dir = host_skills / "good"
good_dir.mkdir()
(good_dir / "SKILL.md").write_text("---\nname: good\ndescription: y\n---\nContent.")
try:
with caplog.at_level("WARNING", logger="omnigent.spec.parser"):
result = discover_host_skills(agent_root, "all")
finally:
# Restore so pytest can clean tmp_path on teardown.
bad_md.chmod(0o600)
assert [s.name for s in result] == ["good"]
skip_records = [rec for rec in caplog.records if "Skipping skill" in rec.message]
assert len(skip_records) == 1
msg = skip_records[0].message
assert str(bad_md) in msg
assert "could not be read" in msg
# ── top-level ``skills:`` field (host-skill filter) ──────────────
def test_parse_skills_filter_omitted_defaults_to_all(agent_dir: Path) -> None:
"""
The top-level ``skills:`` field is optional. When omitted, the
spec defaults to ``"all"`` — every host-discovered skill is
exposed by default.
Claim: a config.yaml without ``skills:`` produces
``spec.skills_filter == "all"``. A regression that flipped
the default to ``"none"`` would silently turn every existing
agent hermetic without warning.
"""
(agent_dir / "config.yaml").write_text(yaml.dump({"spec_version": 1, "name": "x"}))
spec = parse(agent_dir)
assert spec.skills_filter == "all"
def test_parse_skills_filter_explicit_all(agent_dir: Path) -> None:
"""``skills: all`` round-trips as the string ``"all"``."""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": "all"})
)
assert parse(agent_dir).skills_filter == "all"
def test_parse_skills_filter_none(agent_dir: Path) -> None:
"""``skills: none`` round-trips as the string ``"none"``."""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": "none"})
)
assert parse(agent_dir).skills_filter == "none"
def test_parse_skills_filter_empty_list_normalizes_to_none(agent_dir: Path) -> None:
"""
``skills: []`` is an explicit "no skills" declaration —
normalizes to ``"none"`` so the executor handles both the same
way.
Claim: empty list and ``"none"`` produce identical
``skills_filter`` values. A regression that distinguished the
two would create a foot-gun (silent disagreement between two
YAML shapes that look the same to the user).
"""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": []})
)
assert parse(agent_dir).skills_filter == "none"
def test_parse_skills_filter_named_subset(agent_dir: Path) -> None:
"""A list of names round-trips as a list of names."""
(agent_dir / "config.yaml").write_text(
yaml.dump(
{
"spec_version": 1,
"name": "x",
"skills": ["foo", "bar:baz"],
}
)
)
assert parse(agent_dir).skills_filter == ["foo", "bar:baz"]
def test_parse_skills_filter_invalid_string_rejects(agent_dir: Path) -> None:
"""
Strings other than ``"all"`` / ``"none"`` are rejected at
parse time — no silent coercion of typos like ``"al"`` or
``"All"`` to a permissive default.
"""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": "al"})
)
with pytest.raises(OmnigentError, match=r"\"all\".*\"none\""):
parse(agent_dir)
def test_parse_skills_filter_non_string_list_item_rejects(agent_dir: Path) -> None:
"""
Lists with non-string entries (numbers, dicts, nested lists)
fail loud rather than coercing.
"""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": ["foo", 42]})
)
with pytest.raises(OmnigentError, match=r"list items must be strings"):
parse(agent_dir)
def test_parse_skills_filter_dict_rejects(agent_dir: Path) -> None:
"""
Mappings (and other unsupported shapes — booleans, integers)
are rejected. The field is a string or list, never a dict.
"""
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": {"all": True}})
)
with pytest.raises(OmnigentError, match=r"\"all\".*\"none\""):
parse(agent_dir)
def test_parse_skills_filter_is_independent_of_bundled_skills_dir(
agent_dir: Path,
) -> None:
"""
``spec.skills`` (bundled SkillSpec list) and ``spec.skills_filter``
(host filter) are orthogonal: the bundle-side ``skills/`` dir
and the YAML ``skills:`` field don't shadow each other.
Claim: a bundle with a ``skills/researcher/SKILL.md`` AND a
YAML ``skills: none`` field parses both: ``spec.skills`` has
one entry (the bundled researcher), and ``spec.skills_filter``
is ``"none"``. A regression that conflated them would lose
bundled skills when the user opted out of host skills, or
vice versa.
"""
skill_dir = agent_dir / "skills" / "researcher"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: researcher\ndescription: Research things.\n---\nDo research.\n"
)
(agent_dir / "config.yaml").write_text(
yaml.dump({"spec_version": 1, "name": "x", "skills": "none"})
)
spec = parse(agent_dir)
# Bundled skill is preserved.
assert len(spec.skills) == 1
assert spec.skills[0].name == "researcher"
# And the host filter says "none" — bundled and host are
# separate channels.
assert spec.skills_filter == "none"
# ── lenient host-skill discovery ────────────────────
def test_discover_host_skills_skips_missing_frontmatter(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
"""
Host skills with missing YAML frontmatter are skipped with a
warning instead of crashing the CLI.
:param tmp_path: Temporary directory for test fixtures.
:param monkeypatch: Pytest monkeypatch for isolating ``Path.home()``.
:param capsys: Pytest capture fixture for stderr assertions.
"""
from omnigent.spec.parser import discover_host_skills
# Use a separate home dir so the walk-up from agent_root
# doesn't double-scan the same .claude/skills/ as Path.home().
fake_home = tmp_path / "home"
fake_home.mkdir()
monkeypatch.setattr(Path, "home", staticmethod(lambda: fake_home))
skills_dir = fake_home / ".claude" / "skills"
# Good skill.
good = skills_dir / "good-skill"
good.mkdir(parents=True)
(good / "SKILL.md").write_text(
"---\nname: good-skill\ndescription: Works fine.\n---\nContent."
)
# Bad skill — no frontmatter.
bad = skills_dir / "bad-skill"
bad.mkdir(parents=True)
(bad / "SKILL.md").write_text("# No frontmatter here")
agent_root = tmp_path / "project"
agent_root.mkdir()
result = discover_host_skills(agent_root, skills_filter="all")
assert len(result) == 1
assert result[0].name == "good-skill"
captured = capsys.readouterr()
assert "skipped 1 skill(s)" in captured.err
assert "bad-skill" in captured.err
def test_discover_host_skills_skips_yaml_syntax_error(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
"""
Host skills whose frontmatter contains invalid YAML (e.g.
unquoted colons) are skipped gracefully.
:param tmp_path: Temporary directory for test fixtures.
:param monkeypatch: Pytest monkeypatch for isolating ``Path.home()``.
:param capsys: Pytest capture fixture for stderr assertions.
"""
from omnigent.spec.parser import discover_host_skills
fake_home = tmp_path / "home"
fake_home.mkdir()
monkeypatch.setattr(Path, "home", staticmethod(lambda: fake_home))
skills_dir = fake_home / ".claude" / "skills"
broken = skills_dir / "broken-yaml"
broken.mkdir(parents=True)
# Unquoted colon in description triggers yaml.scanner.ScannerError.
(broken / "SKILL.md").write_text(
"---\nname: broken-yaml\ndescription: TRIGGER when: code imports foo\n---\nContent."
)
agent_root = tmp_path / "project"
agent_root.mkdir()
result = discover_host_skills(agent_root, skills_filter="all")
assert result == []
captured = capsys.readouterr()
assert "skipped 1 skill(s)" in captured.err
assert "broken-yaml" in captured.err
def test_discover_host_skills_skips_multiple_bad_skills(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
"""
All broken skills are reported in one pass — no whack-a-mole.
:param tmp_path: Temporary directory for test fixtures.
:param monkeypatch: Pytest monkeypatch for isolating ``Path.home()``.
:param capsys: Pytest capture fixture for stderr assertions.
"""
from omnigent.spec.parser import discover_host_skills
fake_home = tmp_path / "home"
fake_home.mkdir()
monkeypatch.setattr(Path, "home", staticmethod(lambda: fake_home))
skills_dir = fake_home / ".claude" / "skills"
for name in ("bad-a", "bad-b"):
d = skills_dir / name
d.mkdir(parents=True)
(d / "SKILL.md").write_text("No frontmatter.")
agent_root = tmp_path / "project"
agent_root.mkdir()
result = discover_host_skills(agent_root, skills_filter="all")
assert result == []
captured = capsys.readouterr()
assert "skipped 2 skill(s)" in captured.err
assert "bad-a" in captured.err
assert "bad-b" in captured.err
def test_bundled_skills_still_fail_loud_on_bad_frontmatter(
agent_dir: Path,
) -> None:
"""
Bundled skills (inside the agent directory, parsed by
:func:`parse`) must still fail loud — lenient mode is only
for host-discovered skills.
:param agent_dir: Temporary agent directory fixture.
"""
skill_dir = agent_dir / "skills" / "broken"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("No frontmatter here.")
with pytest.raises(OmnigentError, match=r"missing YAML frontmatter"):
parse(agent_dir)
def test_parse_mcp_http(
agent_dir: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Parse an HTTP MCP server config with env var expansion.
:param agent_dir: Temporary agent directory fixture.
:param monkeypatch: Pytest monkeypatch for env vars.
"""
monkeypatch.setenv("API_KEY", "sk-test-key")
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
mcp_config = {
"name": "my-service",
"transport": "http",
"url": "http://localhost:9000/mcp",
"headers": {"Authorization": "Bearer ${API_KEY}"},
}
(mcp_dir / "service.yaml").write_text(yaml.dump(mcp_config))
spec = parse(agent_dir)
mcp = spec.mcp_servers[0]
assert mcp.url == "http://localhost:9000/mcp"
# ${API_KEY} expanded to the value set via monkeypatch.
assert mcp.headers == {"Authorization": "Bearer sk-test-key"}
def test_parse_mcp_env_unresolved_var_raises(
agent_dir: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Unresolved ``${VAR}`` in MCP env raises ``OmnigentError``
at parse time instead of silently passing the literal to the
server.
:param agent_dir: Temporary agent directory fixture.
:param monkeypatch: Pytest monkeypatch for env vars.
"""
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
mcp_config = {
"name": "github",
"transport": "http",
"url": "http://localhost:9000/mcp",
"headers": {"Authorization": "Bearer ${GITHUB_TOKEN}"},
}
(mcp_dir / "github.yaml").write_text(yaml.dump(mcp_config))
with pytest.raises(OmnigentError, match=r"Unresolved environment variable"):
parse(agent_dir)
def test_parse_mcp_headers_unresolved_var_raises(
agent_dir: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Unresolved ``${VAR}`` in MCP headers raises ValueError at
parse time.
:param agent_dir: Temporary agent directory fixture.
:param monkeypatch: Pytest monkeypatch for env vars.
"""
monkeypatch.delenv("API_KEY", raising=False)
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
mcp_config = {
"name": "my-service",
"transport": "http",
"url": "http://localhost:9000/mcp",
"headers": {"Authorization": "Bearer ${API_KEY}"},
}
(mcp_dir / "service.yaml").write_text(yaml.dump(mcp_config))
with pytest.raises(OmnigentError, match=r"Unresolved environment variable"):
parse(agent_dir)
def test_parse_mcp_env_dollar_without_braces_raises(
agent_dir: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Unresolved ``$VAR`` (without braces) also raises ValueError.
:param agent_dir: Temporary agent directory fixture.
:param monkeypatch: Pytest monkeypatch for env vars.
"""
monkeypatch.delenv("MY_SECRET", raising=False)
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
mcp_config = {
"name": "test",
"transport": "http",
"url": "http://localhost:9000/mcp",
"headers": {"Secret": "$MY_SECRET"},
}
(mcp_dir / "test.yaml").write_text(yaml.dump(mcp_config))
with pytest.raises(OmnigentError, match=r"Unresolved environment variable"):
parse(agent_dir)
def test_parse_mcp_missing_name(agent_dir: Path) -> None:
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
(mcp_dir / "bad.yaml").write_text(yaml.dump({"transport": "http", "url": "http://x"}))
with pytest.raises(OmnigentError, match=r"missing required field 'name'"):
parse(agent_dir)
def test_parse_mcp_missing_transport(agent_dir: Path) -> None:
mcp_dir = agent_dir / "tools" / "mcp"
mcp_dir.mkdir(parents=True)
(mcp_dir / "bad.yaml").write_text(yaml.dump({"name": "bad"}))
with pytest.raises(OmnigentError, match=r"missing required field 'transport'"):
parse(agent_dir)
def test_parse_inline_mcp_stdio_server(tmp_path: Path) -> None:
"""
A ``tools:`` block entry with ``type: mcp`` and ``command`` parses
as a stdio MCPServerConfig.
Exercises the ``_parse_inline_mcp_servers`` code path (the tools-block
style, distinct from bundle-file discovery via ``tools/mcp/*.yaml``).
If the inline path were broken, ``spec.mcp_servers`` would be empty
even though the config declares the server.
"""
config = {
"spec_version": 1,
"name": "inline-stdio",
"tools": {
"my_mcp": {
"type": "mcp",
"command": "uvx",
"args": ["mcp-server-github"],
}
},
}
(tmp_path / "config.yaml").write_text(yaml.dump(config))
spec = parse(tmp_path)
# Exactly one server parsed from the inline tools block.
# If _parse_inline_mcp_servers skips it, len() == 0.
assert len(spec.mcp_servers) == 1
srv = spec.mcp_servers[0]
assert srv.name == "my_mcp"