-
Notifications
You must be signed in to change notification settings - Fork 45
fix(models): require Claim.evidence to be non-empty at the model layer #82
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
e142849
5c881de
423c1dd
7843182
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -299,6 +299,51 @@ def test_import_treats_missing_manifest_sha256_as_mismatch( | |
| assert not (store.kb_dir / "claims" / "c1.yaml").exists() | ||
|
|
||
|
|
||
| def test_import_rejects_uncited_claim(store: KBStore, tmp_path: Path) -> None: | ||
| """Regression for #81: a bundle whose claim YAML has evidence: [] | ||
| must be rejected by import_check / import_apply because the Claim | ||
| model now enforces the 'must cite at least one' invariant. Before | ||
| this fix, _validate_content deferred to pydantic, which accepted | ||
| evidence=[] and silently landed an uncited claim.""" | ||
| uncited_yaml = ( | ||
| b"id: bundle-uncited\n" | ||
| b'text: "shipped via bundle, no citations"\n' | ||
| b"type: fact\n" | ||
| b"status: stable\n" | ||
| b"confidence: 1.0\n" | ||
| b"evidence: []\n" | ||
| ) | ||
| bundle_path = tmp_path / "uncited.tar.gz" | ||
| manifest = { | ||
| "spec": bundle.SPEC_VERSION, | ||
| "bundle_id": "deadbeef", | ||
| "files": [{ | ||
| "path": "claims/bundle-uncited.yaml", | ||
| "size": len(uncited_yaml), | ||
| "sha256": hashlib.sha256(uncited_yaml).hexdigest(), | ||
| }], | ||
| "counts": {}, | ||
| "safety": {"has_proposed": False, "has_state_db": False, | ||
| "has_audit_log": False}, | ||
| } | ||
| with tarfile.open(bundle_path, "w:gz") as tar: | ||
| info = tarfile.TarInfo("claims/bundle-uncited.yaml") | ||
| info.size = len(uncited_yaml) | ||
| tar.addfile(info, io.BytesIO(uncited_yaml)) | ||
| mf_bytes = json.dumps(manifest).encode() | ||
| mf_info = tarfile.TarInfo(bundle.MANIFEST_NAME) | ||
| mf_info.size = len(mf_bytes) | ||
| tar.addfile(mf_info, io.BytesIO(mf_bytes)) | ||
|
|
||
| diff = bundle.import_check(store.kb_dir, bundle_path) | ||
| assert not diff.ok | ||
| assert any("schema validation failed" in i for i in diff.issues), diff.issues | ||
|
|
||
| with pytest.raises(RuntimeError, match="schema validation failed"): | ||
| bundle.import_apply(store.kb_dir, bundle_path) | ||
|
Comment on lines
+338
to
+343
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Assert the citation-specific error, not only a generic schema failure. The current checks can pass on unrelated schema errors. To keep this regression targeted to empty Suggested assertion tightening diff = bundle.import_check(store.kb_dir, bundle_path)
assert not diff.ok
- assert any("schema validation failed" in i for i in diff.issues), diff.issues
+ assert any(
+ "schema validation failed" in i and "cite at least one" in i
+ for i in diff.issues
+ ), diff.issues
- with pytest.raises(RuntimeError, match="schema validation failed"):
+ with pytest.raises(RuntimeError, match=r"schema validation failed.*cite at least one"):
bundle.import_apply(store.kb_dir, bundle_path)🤖 Prompt for AI Agents |
||
| assert not (store.kb_dir / "claims" / "bundle-uncited.yaml").exists() | ||
|
|
||
|
|
||
| def test_import_check_passes_when_member_matches_manifest( | ||
| store: KBStore, tmp_path: Path | ||
| ) -> None: | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -5,6 +5,7 @@ | |||||||||||||||||||||||||||||||||||||||
| from pathlib import Path | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| import pytest | ||||||||||||||||||||||||||||||||||||||||
| from pydantic import ValidationError | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| from vouch import audit, lifecycle | ||||||||||||||||||||||||||||||||||||||||
| from vouch.models import ( | ||||||||||||||||||||||||||||||||||||||||
|
|
@@ -108,6 +109,44 @@ def test_claim_can_be_updated(store: KBStore) -> None: | |||||||||||||||||||||||||||||||||||||||
| assert store.get_claim("c1").status == ClaimStatus.STABLE | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| def test_claim_model_rejects_empty_evidence() -> None: | ||||||||||||||||||||||||||||||||||||||||
| """Regression for #81: the 'claims must cite sources' guarantee | ||||||||||||||||||||||||||||||||||||||||
| (README §'Why this exists' point 3; CONTRIBUTING §'Things we | ||||||||||||||||||||||||||||||||||||||||
| won't merge') is now enforced on the Claim model itself, so | ||||||||||||||||||||||||||||||||||||||||
| every write path inherits the check instead of relying on | ||||||||||||||||||||||||||||||||||||||||
| proposals.propose_claim alone.""" | ||||||||||||||||||||||||||||||||||||||||
| with pytest.raises(ValidationError, match="cite at least one"): | ||||||||||||||||||||||||||||||||||||||||
| Claim(id="c1", text="uncited", evidence=[]) | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| def test_put_claim_rejects_empty_evidence(store: KBStore) -> None: | ||||||||||||||||||||||||||||||||||||||||
| """Regression for #81: store.put_claim is a direct write path | ||||||||||||||||||||||||||||||||||||||||
| that used to silently accept Claim(evidence=[]) because the | ||||||||||||||||||||||||||||||||||||||||
| only existence-check loop iterated zero times. The model-level | ||||||||||||||||||||||||||||||||||||||||
| validator now fires before put_claim is even called.""" | ||||||||||||||||||||||||||||||||||||||||
| with pytest.raises(ValidationError, match="cite at least one"): | ||||||||||||||||||||||||||||||||||||||||
| store.put_claim(Claim(id="c1", text="uncited", evidence=[])) | ||||||||||||||||||||||||||||||||||||||||
| assert not (store.kb_dir / "claims" / "c1.yaml").exists() | ||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+122
to
+129
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test can pass without ever exercising On Line 128, Proposed test adjustment def test_put_claim_rejects_empty_evidence(store: KBStore) -> None:
@@
- with pytest.raises(ValidationError, match="cite at least one"):
- store.put_claim(Claim(id="c1", text="uncited", evidence=[]))
+ src = store.put_source(b"e")
+ claim = Claim(id="c1", text="uncited", evidence=[src.id])
+ claim.evidence = []
+ with pytest.raises(ValidationError, match="cite at least one"):
+ store.put_claim(claim)
assert not (store.kb_dir / "claims" / "c1.yaml").exists()📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents |
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| def test_update_claim_rejects_empty_evidence(store: KBStore) -> None: | ||||||||||||||||||||||||||||||||||||||||
| """Regression for #81: a previously-cited claim cannot be mutated | ||||||||||||||||||||||||||||||||||||||||
| down to evidence=[] and silently re-persisted. The model's field | ||||||||||||||||||||||||||||||||||||||||
| validator only fires at construction time, so update_claim | ||||||||||||||||||||||||||||||||||||||||
| re-validates via Claim.model_validate(claim.model_dump()) before | ||||||||||||||||||||||||||||||||||||||||
| writing — otherwise in-place mutation would bypass the gate.""" | ||||||||||||||||||||||||||||||||||||||||
| src = store.put_source(b"e") | ||||||||||||||||||||||||||||||||||||||||
| store.put_claim(Claim(id="c1", text="cited", evidence=[src.id])) | ||||||||||||||||||||||||||||||||||||||||
| persisted_before = (store.kb_dir / "claims" / "c1.yaml").read_text() | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| c = store.get_claim("c1") | ||||||||||||||||||||||||||||||||||||||||
| c.evidence = [] # in-place mutation alone doesn't trigger validation | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| with pytest.raises(ValidationError, match="cite at least one"): | ||||||||||||||||||||||||||||||||||||||||
| store.update_claim(c) | ||||||||||||||||||||||||||||||||||||||||
| assert (store.kb_dir / "claims" / "c1.yaml").read_text() == persisted_before | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
| # --- pages ---------------------------------------------------------------- | ||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||
Uh oh!
There was an error while loading. Please reload this page.