refactor(redaction): name the redaction boundary and pin the concept census (#16688)

CLAUDE.md

@@ -18,6 +18,7 @@ repeated here; where the two disagree, **this file wins**.
 | Need a service, port, or architecture fact | [`AUTOBOT_REFERENCE.md`](docs/developer/AUTOBOT_REFERENCE.md) |
 | Adding an event type, WebSocket route, bus, or session state | [`EVENT_STATE_DOCTRINE.md`](docs/developer/EVENT_STATE_DOCTRINE.md) |
 | Reviewing or changing path validation, session ownership, plugin loading, or secrets | [`THREAT_MODEL.md`](docs/developer/THREAT_MODEL.md) |
+| Adding a redactor, adding a detector to one, or picking which redactor to call | [`REDACTION_BOUNDARY.md`](docs/developer/REDACTION_BOUNDARY.md) |
 | Adding or bumping a HuggingFace `from_pretrained` call site | [`MODEL_REVISION_PINNING.md`](docs/developer/MODEL_REVISION_PINNING.md) |
 | Claiming a work scope, adding a lock/lease, or any "which agent owns this" state | [`AGENT_COORDINATION.md`](docs/developer/AGENT_COORDINATION.md) |
 | Deviating from a standard pattern on purpose | [`ARCHITECTURE_EXCEPTIONS.md`](docs/developer/ARCHITECTURE_EXCEPTIONS.md) |

autobot-backend/a2a/pii_pipeline.py

@@ -26,6 +26,11 @@
     except PIIBlocked as exc:
         # log and return error to caller
         ...
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 import hashlib

autobot-backend/chat_workflow/cot_events.py

@@ -20,6 +20,11 @@
 
 Sensitive argument keys are redacted before emission so credentials and
 tokens never cross the WebSocket boundary.
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 import asyncio

autobot-backend/llm_shared/credential_redaction.py

@@ -15,12 +15,17 @@
 caller of this module's ``redact_dict``/``redact_string`` gets the wider
 coverage for free, including ``llc/services/replay_service.py``'s read-time
 redaction, which previously had no content scanner at all in that path.
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 import re
 from typing import Any, Dict
 
-from autobot_shared.secret_redaction import redact_content
+from autobot_shared.secret_redaction import CREDENTIAL_SUFFIXES, redact_content
 
 # Patterns for common API key formats
 API_KEY_PATTERNS = [
@@ -33,16 +38,24 @@
 # Keys in dicts that should be redacted. Entries MUST be in the normalized
 # form redact_dict compares against (lowercase, "_" and "-" stripped) —
 # un-normalized entries like "api_key" can never match a normalized key (#11762).
-SENSITIVE_KEYS = {
-    "apikey",
-    "token",
-    "bearer",
-    "password",
-    "secret",
-    "credential",
-    "auth",
-    "authorization",
-}
+#
+# #16688: the credential nouns are no longer hand-copied. They come from
+# ``secret_redaction.CREDENTIAL_SUFFIXES``, the shared-tier vocabulary this
+# module already delegates free-text scanning to, so a noun added there (the
+# census names it the owner of "is this name a credential") reaches this
+# module's dict/log redaction without anyone remembering to type it twice.
+#
+# The extras below are a UNION, not a replacement, and that is load-bearing.
+# CREDENTIAL_SUFFIXES is a *field-name suffix* vocabulary and carries no
+# authorization terms; dropping to it alone would stop redacting
+# ``Authorization``, ``auth_header`` and ``x_auth`` in logs — a silent
+# under-redaction, which is exactly the failure mode a redaction merge has to
+# refuse. Every entry here must therefore only ever widen the set.
+# Already covered by CREDENTIAL_SUFFIXES and deliberately not repeated:
+# apikey (via "key"), token, password, secret, credential.
+_AUTHORIZATION_KEYS = frozenset({"bearer", "auth", "authorization"})
+
+SENSITIVE_KEYS = frozenset(CREDENTIAL_SUFFIXES) | _AUTHORIZATION_KEYS
 
 
 def redact_api_key(value: str) -> str:

autobot-backend/llm_shared/tests/test_credential_redaction.py

@@ -4,7 +4,10 @@
 # Author: mrveiss
 """Tests for credential redaction (GH#9037)."""
 
+import pytest
+
 from llm_shared.credential_redaction import (
+    SENSITIVE_KEYS,
     redact_api_key,
     redact_dict,
     redact_string,
@@ -135,3 +138,66 @@ def test_redact_string_leaves_an_ordinary_sentence_untouched():
 def test_redact_string_does_not_quarantine_a_forwarded_code_snippet():
     text = "import os\napi_key = os.environ.get('SOME_KEY')\ndef getKey():\n    return apiKey"
     assert redact_string(text) == text
+
+
+# ---------------------------------------------------------------------------
+# #16688: SENSITIVE_KEYS is derived from the shared vocabulary, not hand-copied.
+# The derivation must only ever WIDEN. A redaction consolidation that narrows
+# is a silent under-redaction, and nothing else in this suite would catch it.
+# ---------------------------------------------------------------------------
+
+# The literal set this module carried before the vocabulary was shared. Kept
+# here deliberately as a frozen historical record -- it is the thing the union
+# must never lose, so it must not be imported from the module under test.
+_PRE_16688_SENSITIVE_KEYS = frozenset(
+    {"apikey", "token", "bearer", "password", "secret", "credential", "auth", "authorization"}
+)
+
+
+def _normalized(key: str) -> str:
+    return key.lower().replace("_", "").replace("-", "")
+
+
+def _is_sensitive(key: str) -> bool:
+    return any(fragment in _normalized(key) for fragment in SENSITIVE_KEYS)
+
+
+def test_derived_vocabulary_still_covers_every_key_the_literal_set_covered() -> None:
+    """Every pre-#16688 fragment must still match, or something stopped being redacted."""
+    lost = sorted(f for f in _PRE_16688_SENSITIVE_KEYS if not _is_sensitive(f))
+    assert not lost, f"these stopped being treated as sensitive: {lost}"
+
+
+@pytest.mark.parametrize(
+    "key",
+    [
+        # The authorization terms the shared suffix vocabulary does NOT carry --
+        # a straight replacement would have dropped all three.
+        "Authorization",
+        "auth_header",
+        "x_auth",
+        "bearer_token",
+        # Carried by the shared vocabulary.
+        "api_key",
+        "client_secret",
+        "db_password",
+        "session_token",
+        # Gained from the shared vocabulary; previously missed here.
+        "tls_cert",
+        "db_dsn",
+        "hmac_signature",
+        "private_pem",
+        "random_seed",
+        "salt_value",
+        "passphrase",
+    ],
+)
+def test_a_sensitive_key_is_redacted(key: str) -> None:
+    out = redact_dict({key: "sup3rSecretValue1234567890"})
+    assert "sup3rSecretValue1234567890" not in str(out), f"{key} leaked its value"
+
+
+@pytest.mark.parametrize("key", ["user_name", "model", "temperature", "message_count", "status"])
+def test_an_ordinary_key_keeps_its_value(key: str) -> None:
+    """The union widens the sensitive set, so guard the false-positive side too."""
+    assert redact_dict({key: "plain-value"})[key] == "plain-value"

autobot-backend/security/chat_message_safety.py

@@ -42,6 +42,11 @@
 rejects the turn here; every other HIGH/CRITICAL hit is logged as a
 security event for audit/trend visibility without disrupting the
 conversation.
+
+
+Boundary: this module deliberately declares **no** detector of its own — it
+composes the canonical ones. That is why it is not in the redaction census; see
+``docs/developer/REDACTION_BOUNDARY.md`` (#16688).
 """
 
 from __future__ import annotations

autobot-backend/services/config_revision_service.py

@@ -8,6 +8,11 @@
 Manages configuration audit trail: recording changes with before/after
 snapshots, computing diffs, redacting secrets, and rolling back to any
 prior revision.
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 import uuid

autobot_shared/secret_redaction.py

@@ -42,6 +42,11 @@
 text (a message body, a document, a log line) with no field name to go on,
 for exactly the case a config-model redactor cannot reach: a credential
 sitting in prose ("your temporary password is X"), not behind a named field.
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 from __future__ import annotations
@@ -238,9 +243,21 @@ class ContentMatch:
     rf"[\s\S]{{0,{_PEM_BODY_MAX}}}?-----END [A-Z0-9 ]{{0,{_PEM_HEADER_MAX}}}PRIVATE KEY-----"
 )
 
-# A JWT is three base64url segments joined by dots; the first two decode to
-# JSON objects, so both start with the base64url encoding of ``{"`` (``eyJ``).
-_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b")
+# A JWT is three base64url segments joined by dots. The HEADER always decodes
+# to a JSON object opening ``{"`` (``eyJ``) -- ``alg`` is mandatory, so it is
+# never the empty object. The PAYLOAD is only anchored on the dot structure.
+#
+# #16688: requiring ``eyJ`` on the payload too -- which this pattern did until
+# the redaction census -- makes this scanner strictly narrower than the one in
+# ``a2a/pii_pipeline.py`` (``jwt_re``), which anchors the header alone. A
+# payload only starts ``eyJ`` when its JSON begins exactly ``{"``. A serializer
+# emitting a space after the brace (``{ "sub": ...``) encodes to ``eyAi``, and
+# one padding before it gives ``IHsi``. Both are well-formed tokens that were
+# redacted on the A2A path and passed through untouched here -- and this is the
+# module ``llm_shared.credential_redaction`` delegates to, so the miss landed in
+# logs. Widened to match; deliberately a superset, never a narrowing
+# (see docs/developer/REDACTION_BOUNDARY.md).
+_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b")
 
 # Provider-specific prefixes with a fixed, well-documented shape — the same
 # patterns every mainstream secret scanner (gitleaks, trufflehog, detect-secrets)

autobot_shared/secret_redaction_test.py

@@ -38,6 +38,21 @@
         "dGhpc2lzYXNpZ25hdHVyZQ",
     ]
 )
+
+# #16688: a JWT whose PAYLOAD does not begin ``eyJ``. A payload only starts
+# ``eyJ`` when its JSON begins exactly ``{"``; a serializer that emits a space
+# after the brace (``{ "sub": ...``) base64url-encodes to ``eyAi`` instead.
+# The token below is well-formed and was matched by ``a2a/pii_pipeline.py``
+# while this scanner missed it -- and this is the module
+# ``llm_shared.credential_redaction`` delegates its log redaction to.
+# Payload decodes to: { "sub": "1234567890", "name": "A" }
+_JWT_NON_EYJ_PAYLOAD = ".".join(
+    [
+        "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
+        "eyAic3ViIjogIjEyMzQ1Njc4OTAiLCAibmFtZSI6ICJBIiB9",  # pragma: allowlist secret
+        "dGhpc2lzYXNpZ25hdHVyZQ",
+    ]
+)
 _PEM_SHAPED = "\n".join(
     [
         "-----BEGIN " + "RSA PRIVATE KEY-----",
@@ -54,6 +69,7 @@
     ("phrase_password", "Welcome! Your temporary password is Xy9#mK2!Zq. Please log in and change it."),
     ("phrase_api_key", f"Here is your API key: {_OPENAI_SHAPED}"),
     ("jwt", f"Authorization header: {_JWT_SHAPED}"),
+    ("jwt_non_eyj_payload", f"Authorization header: {_JWT_NON_EYJ_PAYLOAD}"),
     ("pem_private_key", _PEM_SHAPED),
     ("basic_auth_url", "Connect via https://admin:sup3rSecret1@db.example.com:5432/mydb"),
     ("aws_key", f"{_AWS_SHAPED} is our old access key, please rotate it"),
@@ -154,3 +170,21 @@ def test_multiple_distinct_credentials_are_all_found() -> None:
     matches = scan_content_for_credentials(text)
     assert {m.pattern for m in matches} == {"known_key_prefix"}
     assert len(matches) == 2
+
+
+def test_a_jwt_is_matched_on_its_header_not_its_payload_shape() -> None:
+    """#16688: the JWT rule itself must fire, not some other detector.
+
+    Asserting only "something matched" would pass if the high-entropy rule
+    happened to claim the span, which would leave the narrowing unfixed and
+    the test green.
+    """
+    matches = scan_content_for_credentials(_JWT_NON_EYJ_PAYLOAD)
+    kinds = {m.pattern for m in matches}
+    assert "jwt" in kinds, f"expected the jwt rule to fire, got {kinds or 'no matches'}"
+
+
+def test_the_jwt_rule_is_not_narrower_than_the_a2a_scanner() -> None:
+    """Both JWT shapes must redact, so this module cannot miss what A2A blocks."""
+    for token in (_JWT_SHAPED, _JWT_NON_EYJ_PAYLOAD):
+        assert token not in redact_content(token), f"{token!r} survived redaction"

autobot_shared/security/redaction.py

@@ -17,6 +17,11 @@
   log-line text.
 * :func:`redact_mapping` — mapping/kv level. Masks dict values whose key matches
   a known secret key fragment.
+
+
+Boundary: this module is one of seven that own a secret detector — see
+``docs/developer/REDACTION_BOUNDARY.md`` for which redactor owns which shape of
+the problem, and add a new detector there rather than starting an eighth (#16688).
 """
 
 import re