fix(engine): entity-name hygiene at resolution intake — whitespace normalization and tag-shaped category-label skip

hindsight-api-slim/hindsight_api/engine/retain/link_utils.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+import re
 import time
 from datetime import UTC
 
@@ -17,13 +18,55 @@
 from ..db.base import DatabaseConnection
 from ..db.ops import DataAccessOps
 from ..memory_engine import fq_table
+from .entity_labels import build_labels_lookup, is_label_entity, parse_entity_labels
 from .types import CausalRelation, EntityResolutionResult
 
 logger = logging.getLogger(__name__)
 
 # Sentinel UUID used in the unique index to represent NULL entity_id
 _NIL_ENTITY_UUID = "00000000-0000-0000-0000-000000000000"
 
+# Collapses any run of whitespace (including \n, \r, \t) to a single space.
+_WHITESPACE_RUN_RE = re.compile(r"\s+")
+
+
+def _normalize_entity_name(name: str) -> str:
+    """Collapse internal whitespace runs (including newlines/tabs) to a single
+    space and strip leading/trailing whitespace; case is left untouched.
+
+    Motivation: production banks were measured (2026-08-08) with entity
+    canonical_name values containing embedded newlines -- extraction
+    artifacts -- which shears any line-oriented consumer (psql -A output,
+    logs, exports). Case handling is unchanged because the entity registry
+    matches on LOWER(canonical_name) separately.
+    """
+    return _WHITESPACE_RUN_RE.sub(" ", name).strip()
+
+
+# Conservative tag-shape: lowercase-word ":" lowercase-word, e.g. "domain:lens".
+# Deliberately does NOT match URLs (contain "//"), Windows paths (contain "\"
+# or a single-letter drive prefix like "C:"), times ("12:30" -- digits before
+# the colon), names with spaces or dots ("re: subject"), or names with no
+# colon at all ("backup", "x-grok-conv-id").
+_TAG_SHAPE_RE = re.compile(r"^[a-z][a-z0-9_-]{1,15}:[a-z][a-z0-9_-]{1,24}$")
+
+
+def _is_tag_shaped_name(name: str) -> bool:
+    """Return True if `name` (already whitespace-normalized) looks like a
+    category/tag label rather than an entity, e.g. "domain:lens".
+
+    Motivation: extraction was measured (2026-08-08) minting entities named
+    domain:lens, domain:host, domain:memory -- category labels, not entities.
+    They behave like broad tags and measurably poisoned entity-based scoping
+    (61.5% vs 94.0% precision on one subject). Matching is done against a
+    lowercased copy of the name; the stored/compared name's case is untouched
+    by this check. Callers must exempt names that are configured entity
+    labels (e.g. "use:use-001") before treating a match here as skippable --
+    label values are deliberately "key:value" shaped (GH-1558).
+    """
+    return bool(_TAG_SHAPE_RE.match(name.lower()))
+
+
 # Maximum number of temporal links to keep per unit (from_unit_id).
 # Retrieval only reads top 10-20 per unit via LATERAL join, so keeping
 # more is wasted storage and write amplification.
@@ -147,27 +190,60 @@ def _prepare_entities_for_resolution(
     fact_dates: list,
     llm_entities: list[list[dict]],
     log_buffer: list[str] = None,
+    entity_labels: list | None = None,
 ) -> tuple[list[dict], list[list[dict]], list[tuple]]:
     """
     Convert LLM entities into the flat format expected by entity resolver.
 
+    Also drops candidate names that are tag-shaped (e.g. "domain:lens") rather
+    than real entities -- see `_is_tag_shaped_name`. A tag-shaped name that is
+    also a configured entity label (e.g. "use:use-001" from a tag-type label
+    group) is exempt: label values are deliberately "key:value" shaped and
+    must still reach entity resolution (GH-1558 exact-match path).
+
+    Args:
+        entity_labels: Optional configured label taxonomy, used only to
+            exempt configured label values from the tag-shape skip.
+
     Returns:
         Tuple of (all_entities_flat, all_entities, entity_to_unit) where:
         - all_entities_flat: flat list of entity dicts ready for resolve_entities_batch
         - all_entities: per-unit formatted entity lists
         - entity_to_unit: maps flat index to (unit_id, local_index, fact_date)
     """
+    labels_cfg = parse_entity_labels(entity_labels)
+    labels_lookup = build_labels_lookup(labels_cfg) if labels_cfg else set()
+
     substep_start = time.time()
     all_entities = []
+    skipped_tag_shaped = 0
     for entity_list in llm_entities:
         formatted_entities = []
         for ent in entity_list:
             if hasattr(ent, "text"):
-                formatted_entities.append({"text": ent.text, "type": "CONCEPT"})
+                raw_text, entity_type = ent.text, "CONCEPT"
             elif isinstance(ent, dict):
-                formatted_entities.append({"text": ent.get("text", ""), "type": ent.get("type", "CONCEPT")})
+                raw_text, entity_type = ent.get("text", ""), ent.get("type", "CONCEPT")
+            else:
+                continue
+
+            normalized_text = _normalize_entity_name(raw_text)
+            is_configured_label = bool(labels_cfg) and is_label_entity(normalized_text, labels_cfg, labels_lookup)
+            if not is_configured_label and _is_tag_shaped_name(normalized_text):
+                skipped_tag_shaped += 1
+                logger.debug("Skipping tag-shaped candidate entity name: %r", normalized_text)
+                continue
+
+            formatted_entities.append({"text": normalized_text, "type": entity_type})
         all_entities.append(formatted_entities)
 
+    if skipped_tag_shaped:
+        _log(
+            log_buffer,
+            f"  [6.1] Skipped {skipped_tag_shaped} tag-shaped candidate entity name(s)",
+            level="debug",
+        )
+
     total_entities = sum(len(ents) for ents in all_entities)
     _log(
         log_buffer,
@@ -242,7 +318,7 @@ async def resolve_entities_only(
         Phase 2.
     """
     all_entities_flat, _all_entities, entity_to_unit = _prepare_entities_for_resolution(
-        unit_ids, sentences, fact_dates, llm_entities, log_buffer
+        unit_ids, sentences, fact_dates, llm_entities, log_buffer, entity_labels
     )
 
     if not all_entities_flat:

hindsight-api-slim/tests/test_entity_name_hygiene.py

@@ -0,0 +1,234 @@
+"""Unit tests for entity-name intake hygiene.
+
+Covers two data-quality fixes at candidate-entity intake, in
+`hindsight_api.engine.retain.link_utils`:
+
+1. `_normalize_entity_name` -- collapses internal whitespace runs (including
+   newlines/tabs from extraction artifacts) to a single space and strips ends,
+   so stored canonical_name values never contain embedded newlines.
+2. `_is_tag_shaped_name` -- conservatively recognizes category/tag-shaped
+   names (e.g. "domain:lens") so they are silently skipped instead of being
+   minted as entities, which measurably poisoned entity-based scoping.
+
+Both are exercised directly (pure functions, no DB/LLM) and through the
+intake function `_prepare_entities_for_resolution`, which applies
+normalization then the tag-shape check in that order.
+"""
+
+import pytest
+
+from hindsight_api.engine.retain.link_utils import (
+    _is_tag_shaped_name,
+    _normalize_entity_name,
+    _prepare_entities_for_resolution,
+)
+
+
+class _FakeEntity:
+    """Minimal object-style entity: has a `.text` attribute, no `.get`."""
+
+    def __init__(self, text: str):
+        self.text = text
+
+
+def _entity_texts(all_entities_flat: list[dict]) -> list[str]:
+    return [e["text"] for e in all_entities_flat]
+
+
+# --- Attacking tests first: real entity names that must survive the tag-shape filter ---
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "https://api.x.ai",  # URL: contains "//" right after the colon
+        "C:\\HQ_Backups",  # Windows path: drive prefix + backslash
+        "12:30",  # time: digits before the colon
+        "re: subject",  # space immediately after the colon
+        "backup",  # no colon at all
+        "x-grok-conv-id",  # hyphenated, no colon
+    ],
+)
+def test_is_tag_shaped_name_does_not_flag_real_entities(name):
+    assert _is_tag_shaped_name(name) is False
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "domain:lens",
+        "topic:memory",
+        "scope:global",
+    ],
+)
+def test_is_tag_shaped_name_flags_measured_category_labels(name):
+    assert _is_tag_shaped_name(name) is True
+
+
+def test_is_tag_shaped_name_matches_against_lowercased_copy():
+    # Matching happens against a lowercased copy; the check still recognizes
+    # the shape even when the original name carries mixed case.
+    assert _is_tag_shaped_name("Domain:Lens") is True
+
+
+def test_is_tag_shaped_name_empty_string_not_flagged():
+    assert _is_tag_shaped_name("") is False
+
+
+def test_is_tag_shaped_name_rejects_short_single_char_segments():
+    # Regex requires at least 2 chars on each side of the colon; single
+    # letters (e.g. drive-letter-shaped "c:x") are not treated as tags.
+    assert _is_tag_shaped_name("c:x") is False
+
+
+# --- _normalize_entity_name ---
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("foo\nbar", "foo bar"),
+        ("a\r\n b\tc", "a b c"),
+        ("Normal Name", "Normal Name"),
+        ("  leading and trailing  ", "leading and trailing"),
+        ("multiple   spaces    inside", "multiple spaces inside"),
+    ],
+)
+def test_normalize_entity_name_collapses_whitespace(raw, expected):
+    assert _normalize_entity_name(raw) == expected
+
+
+def test_normalize_entity_name_preserves_case():
+    assert _normalize_entity_name("MiXeD\nCaSe") == "MiXeD CaSe"
+
+
+def test_normalize_entity_name_all_whitespace_becomes_empty():
+    # Matches the existing code's (lack of) special-casing for empty names:
+    # normalization just yields "" like any other already-empty candidate
+    # name would -- no new "skip empty" behavior is invented here.
+    assert _normalize_entity_name("   \n\t  ") == ""
+
+
+# --- Intake integration: _prepare_entities_for_resolution applies both fixes, in order ---
+
+
+def test_intake_normalizes_whitespace_in_stored_text():
+    all_entities_flat, _all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[[{"text": "foo\nbar", "type": "CONCEPT"}]],
+    )
+    assert _entity_texts(all_entities_flat) == ["foo bar"]
+
+
+def test_intake_normalizes_object_style_entities_with_text_attribute():
+    all_entities_flat, _all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[[_FakeEntity("a\r\n b\tc")]],
+    )
+    assert _entity_texts(all_entities_flat) == ["a b c"]
+
+
+def test_intake_keeps_real_entities_surviving_both_fixes():
+    all_entities_flat, _all_entities, entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[
+            [
+                {"text": "domain:lens", "type": "CONCEPT"},
+                {"text": "https://api.x.ai", "type": "CONCEPT"},
+            ]
+        ],
+    )
+    assert _entity_texts(all_entities_flat) == ["https://api.x.ai"]
+    assert len(entity_to_unit) == 1
+
+
+def test_intake_applies_normalization_before_tag_shape_check():
+    # A tag-shaped name padded with stray whitespace/newlines must still be
+    # recognized as tag-shaped -- exercising the documented ordering
+    # (normalize first, then check the tag shape).
+    all_entities_flat, _all_entities, entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[[{"text": "  domain:lens\n", "type": "CONCEPT"}]],
+    )
+    assert _entity_texts(all_entities_flat) == []
+    assert entity_to_unit == []
+
+
+def test_intake_keeps_real_entity_with_space_after_colon():
+    # "re: subject" survives both fixes: normalization leaves the single
+    # interior space untouched, and the tag-shape check rejects it because of
+    # the space right after the colon.
+    all_entities_flat, _all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[[{"text": "re: subject", "type": "CONCEPT"}]],
+    )
+    assert _entity_texts(all_entities_flat) == ["re: subject"]
+
+
+def test_intake_skipped_tag_shaped_entity_excluded_from_nearby_entities():
+    # A skipped tag-shaped entity must not leak into another entity's
+    # nearby_entities co-occurrence list either -- it never enters
+    # `all_entities`, which is the source `_resolve_from_candidates` reads
+    # nearby_entities from.
+    all_entities_flat, all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[
+            [
+                {"text": "domain:lens", "type": "CONCEPT"},
+                {"text": "Alice", "type": "CONCEPT"},
+            ]
+        ],
+    )
+    assert _entity_texts(all_entities_flat) == ["Alice"]
+    assert [e["text"] for e in all_entities[0]] == ["Alice"]
+
+
+def test_intake_keeps_tag_shaped_name_that_is_a_configured_label():
+    # Attacking case for the tag-shape filter itself: "key:value" is exactly
+    # how tag-type entity labels are named (GH-1558), so a candidate matching
+    # a configured label value must survive even though it is tag-shaped.
+    entity_labels = [
+        {
+            "key": "use",
+            "type": "multi-values",
+            "tag": True,
+            "values": [{"value": "use-001"}, {"value": "use-002"}],
+        }
+    ]
+    all_entities_flat, _all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[
+            [
+                {"text": "use:use-001", "type": "CONCEPT"},
+                {"text": "domain:lens", "type": "CONCEPT"},
+            ]
+        ],
+        entity_labels=entity_labels,
+    )
+    assert _entity_texts(all_entities_flat) == ["use:use-001"]
+
+
+def test_intake_tag_shape_filter_applies_without_entity_labels():
+    # Same tag-shaped name as above, but with no configured label taxonomy --
+    # the default (entity_labels=None) must still skip it.
+    all_entities_flat, _all_entities, _entity_to_unit = _prepare_entities_for_resolution(
+        unit_ids=["u1"],
+        sentences=["fact text"],
+        fact_dates=[None],
+        llm_entities=[[{"text": "use:use-001", "type": "CONCEPT"}]],
+    )
+    assert _entity_texts(all_entities_flat) == []