feat: cBioPortal→OMOP showcase — slice-0 concept mapping + slice-1 FK-closed OMOP shape with cross-study identity

.last-branch

@@ -1 +1 @@
-ralph/fix/data-unification
+ralph/feat/mapping-slice0

archive/2026-06-30-fix/data-unification/progress.txt

@@ -0,0 +1,3 @@
+# Ralph Progress Log
+Started: Tue Jun 30 14:14:58 EDT 2026
+---

scripts/check_engine_coupling.py

@@ -0,0 +1,140 @@
+"""R29 guard: no OMOP/OncoTree literal may leak into the engine core or the
+generic mapping spine.
+
+The 'any ontology' promise (R29) is enforced mechanically here, not in review.
+A single policy constant (:data:`R29_POLICY`) owns the denylist, the scanned
+core paths, and the allowlist of locations where domain literals legitimately
+live (per-vocabulary policy modules, manifest adapters, authored manifests,
+eval, tests). The same scan runs as a unit test (``tests/unit/
+test_engine_coupling.py``) so ``uv run pytest`` fails on a leak, and as a CLI:
+
+    uv run python scripts/check_engine_coupling.py
+
+Configured core paths that do not exist yet (e.g. ``src/sema/resolve/engine.py``,
+``src/sema/compile/``) are skipped so the guard is green today and tightens
+automatically as later Slice-0 stories create them.
+"""
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, NamedTuple, Sequence
+
+
+@dataclass(frozen=True)
+class EngineCouplingPolicy:
+    """Frozen R29 policy: what is forbidden, where it is checked, what is exempt.
+
+    ``core_paths`` and ``allowlist`` are repo-relative POSIX paths. A path is a
+    file (scanned directly) or a directory (scanned recursively for ``*.py``).
+    A scanned file is exempt when its repo-relative path is, or lives under, an
+    allowlist entry.
+    """
+
+    denylist: tuple[str, ...]
+    core_paths: tuple[str, ...]
+    allowlist: tuple[str, ...]
+
+
+# Case-explicit: a casing variant must not bypass the US-000 migration.
+R29_POLICY = EngineCouplingPolicy(
+    denylist=(
+        "OncoTree",
+        "oncotree",
+        "ONCOTREE",
+        "cBioPortal",
+        "cbioportal",
+        "cBio",
+        "cbio",
+        "Maps to",
+        "standard_concept",
+        "condition_occurrence",
+        "concept_id",
+    ),
+    core_paths=(
+        "src/sema/engine",
+        "src/sema/resolve/engine.py",
+        "src/sema/resolve/assembler.py",
+        "src/sema/resolve/identity_registry.py",
+        "src/sema/resolve/identity_registry_utils.py",
+        "src/sema/resolve/identity_resolver.py",
+        "src/sema/resolve/identity_collapse.py",
+        "src/sema/resolve/identity_collapse_utils.py",
+        "src/sema/compile",
+        "src/sema/targets",
+    ),
+    allowlist=(
+        "src/sema/resolve/policy.py",
+        "src/sema/resolve/policies",
+        "src/sema/targets/adapters",
+        "src/sema/eval",
+        "tests",
+    ),
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+class Violation(NamedTuple):
+    path: str
+    lineno: int
+    literal: str
+
+
+def _is_allowlisted(rel_path: str, allowlist: Sequence[str]) -> bool:
+    return any(
+        rel_path == entry or rel_path.startswith(f"{entry}/") for entry in allowlist
+    )
+
+
+def _iter_scanned_files(
+    root: Path, policy: EngineCouplingPolicy
+) -> Iterator[Path]:
+    """Yield ``*.py`` files under the core paths, skipping allowlisted ones."""
+    for core in policy.core_paths:
+        target = root / core
+        if not target.exists():
+            continue
+        candidates = [target] if target.is_file() else sorted(target.rglob("*.py"))
+        for candidate in candidates:
+            if candidate.suffix != ".py":
+                continue
+            rel = candidate.relative_to(root).as_posix()
+            if not _is_allowlisted(rel, policy.allowlist):
+                yield candidate
+
+
+def scan(root: Path, policy: EngineCouplingPolicy) -> list[Violation]:
+    """Return every denylist hit across the policy's core paths under ``root``."""
+    violations: list[Violation] = []
+    for py_file in _iter_scanned_files(root, policy):
+        rel = py_file.relative_to(root).as_posix()
+        lines = py_file.read_text(encoding="utf-8").splitlines()
+        for lineno, line in enumerate(lines, start=1):
+            for literal in policy.denylist:
+                if literal in line:
+                    violations.append(Violation(rel, lineno, literal))
+    return violations
+
+
+def find_violations(
+    root: Path = REPO_ROOT, policy: EngineCouplingPolicy = R29_POLICY
+) -> list[Violation]:
+    """Scan the live repository tree with the default R29 policy."""
+    return scan(root, policy)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    violations = find_violations()
+    if not violations:
+        print("R29 engine-coupling guard: clean")
+        return 0
+    print("R29 engine-coupling guard: VIOLATIONS")
+    for v in violations:
+        print(f"  {v.path}:{v.lineno}: {v.literal!r}")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

showcase/cbioportal_to_omop/cbioportal_fetch_utils.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import json
+import os
 from pathlib import Path
 from urllib.request import Request, urlopen
 
@@ -15,6 +16,12 @@
 )
 from sema.log import logger
 
+# Git-LFS pointer files begin with this line. The datahub stores large tables in
+# LFS; when GitHub's media CDN lacks an object it 404s and the raw host serves the
+# pointer instead of the content. Accepting that pointer silently loads a 2-row
+# garbage table, so we detect it and fail loudly (bug: silent LFS-pointer ingest).
+_LFS_POINTER_MAGIC = b"version https://git-lfs.github.com/spec/v1"
+
 
 def fetch_study_files(study_id: str, cache_dir: Path) -> Path:
     study_cache = cache_dir / study_id
@@ -43,11 +50,58 @@ def fetch_lfs_or_raw(study_id: str, filename: str, target: Path) -> None:
     media_url = MEDIA_URL_TEMPLATE.format(study_id=study_id, filename=filename)
     try:
         download_url_to(media_url, target)
-        return
+        if not _is_lfs_pointer(target):
+            return
     except Exception as media_err:
         logger.debug("media URL failed for {}: {}; falling back to raw", filename, media_err)
-    raw_url = RAW_URL_TEMPLATE.format(study_id=study_id, filename=filename)
-    download_url_to(raw_url, target)
+        raw_url = RAW_URL_TEMPLATE.format(study_id=study_id, filename=filename)
+        download_url_to(raw_url, target)
+    if not _is_lfs_pointer(target):
+        return
+    # The content we have is a git-lfs pointer, not the real file. Try an
+    # authenticated GitHub API fetch (LFS is resolved server-side when authorized),
+    # then fail loudly rather than ingest the pointer as data.
+    if _try_authenticated_lfs(study_id, filename, target) and not _is_lfs_pointer(target):
+        return
+    raise RuntimeError(
+        f"cBioPortal file '{filename}' resolved to a git-lfs POINTER, not content "
+        f"(GitHub LFS unavailable — media CDN 404, no/failed auth). Set GITHUB_TOKEN "
+        f"to authenticate, or retry when the datahub LFS object is available. "
+        f"Ingesting the pointer would create a 2-row garbage table."
+    )
+
+
+def _is_lfs_pointer(target: Path) -> bool:
+    try:
+        with target.open("rb") as fh:
+            return fh.read(len(_LFS_POINTER_MAGIC)) == _LFS_POINTER_MAGIC
+    except OSError:
+        return False
+
+
+def _try_authenticated_lfs(study_id: str, filename: str, target: Path) -> bool:
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    if not token:
+        return False
+    api_url = (
+        f"https://api.github.com/repos/cBioPortal/datahub/contents/"
+        f"public/{study_id}/{filename}?ref=master"
+    )
+    req = Request(
+        api_url,
+        headers={
+            "Accept": "application/vnd.github.raw",
+            "Authorization": f"Bearer {token}",
+        },
+    )
+    try:
+        with urlopen(req) as resp:
+            data = resp.read()
+    except Exception as err:  # noqa: BLE001 - report and fall through to fail-fast
+        logger.debug("authenticated LFS fetch failed for {}: {}", filename, err)
+        return False
+    target.write_bytes(data)
+    return True
 
 
 def list_study_entries(study_id: str) -> list[dict[str, str]]: