Skip to content

Commit cc06db6

Browse files
committed
fix(sink): promote the physical column type when a contract widens
Symptom: batch 5 of billing_transactions died with _duckdb.ConversionException: Conversion Error: Type INT64 with value 2216450393 can't be cast because the value is out of range for the destination type INT32 when casting from source column account_id The classifier had already returned widening_safe for account_id, the BACKWARD policy said EVOLVE, and contract v3 on disk said BIGINT. The load still failed. Root cause: contract evolution and physical table evolution were two separate things. LandingZone.evolve_table only issued ADD COLUMN for fields missing from the table, so ddl.build_alter_type had no callers at all. The registry said BIGINT while DuckDB still said INTEGER, which made the framework's central promise ("widening is safe, we handle it") false at the storage layer. Fix: evolve_table now reads the physical types out of information_schema, maps them back into the logical lattice with duckdb_type_to_spec, and issues ALTER COLUMN ... SET DATA TYPE for any field whose contract type is a strict widening of what the table has. The promotion is gated on is_widening so this path can never narrow a column and truncate landed history.
1 parent 2310ccb commit cc06db6

1 file changed

Lines changed: 90 additions & 14 deletions

File tree

src/schema_guard/sink.py

Lines changed: 90 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@
99
1010
* Landing is idempotent. A batch that lands twice produces the same table, because rows
1111
are merged on the contract primary key rather than appended. Replay depends on this.
12-
* Evolution only adds. ``evolve_table`` issues ADD COLUMN, never DROP COLUMN. A field
13-
removed from a contract keeps its physical column, so a report reading history under an
14-
older contract version still resolves.
12+
* Evolution never destroys. ``evolve_table`` issues ADD COLUMN for new fields and
13+
ALTER COLUMN for a widened type, and never DROP COLUMN. A field removed from a contract
14+
keeps its physical column, so a report reading history under an older contract version
15+
still resolves.
1516
"""
1617

1718
from __future__ import annotations
@@ -22,12 +23,57 @@
2223
import pandas as pd
2324

2425
from .contracts import Contract
25-
from .ddl import build_add_column, build_create_table, build_quarantine_table
26+
from .ddl import (
27+
build_add_column,
28+
build_alter_type,
29+
build_create_table,
30+
build_quarantine_table,
31+
)
2632
from .logging_setup import get_logger
2733
from .transforms import AUDIT_COLUMNS
34+
from .types import (
35+
BOOLEAN,
36+
DATE,
37+
DECIMAL,
38+
FLOAT64,
39+
INT32,
40+
INT64,
41+
STRING,
42+
TIMESTAMP,
43+
TypeSpec,
44+
is_widening,
45+
)
2846

2947
log = get_logger("sink")
3048

49+
# DuckDB reports column types through information_schema as plain strings. Mapping them
50+
# back into the logical lattice is what lets evolve_table decide whether a contract type
51+
# is a widening of what is physically there.
52+
_DUCKDB_TO_LOGICAL = {
53+
"TINYINT": INT32,
54+
"SMALLINT": INT32,
55+
"INTEGER": INT32,
56+
"BIGINT": INT64,
57+
"HUGEINT": INT64,
58+
"FLOAT": FLOAT64,
59+
"DOUBLE": FLOAT64,
60+
"VARCHAR": STRING,
61+
"BOOLEAN": BOOLEAN,
62+
"DATE": DATE,
63+
"TIMESTAMP": TIMESTAMP,
64+
}
65+
66+
67+
def duckdb_type_to_spec(data_type: str) -> TypeSpec | None:
68+
"""Parse a DuckDB information_schema data_type into a TypeSpec, or None if unknown."""
69+
upper = data_type.strip().upper()
70+
if upper.startswith("DECIMAL"):
71+
inner = upper[upper.find("(") + 1 : upper.find(")")]
72+
precision, _, scale = inner.partition(",")
73+
return TypeSpec(DECIMAL, precision=int(precision), scale=int(scale or 0))
74+
logical = _DUCKDB_TO_LOGICAL.get(upper)
75+
return None if logical is None else TypeSpec(logical)
76+
3177

3278
class LandingZone:
3379
"""Owns the DuckDB connection and every write into the landing schema."""
@@ -67,8 +113,16 @@ def columns(self, name: str) -> list[str]:
67113
).fetchall()
68114
return [r[0] for r in rows]
69115

116+
def column_types(self, name: str) -> dict[str, str]:
117+
rows = self.connection.execute(
118+
"SELECT column_name, data_type FROM information_schema.columns "
119+
"WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position",
120+
[self.schema, name],
121+
).fetchall()
122+
return {r[0]: r[1] for r in rows}
123+
70124
def ensure_table(self, contract: Contract) -> None:
71-
"""Create the landing table if absent, otherwise add any new contract columns."""
125+
"""Create the landing table if absent, otherwise bring it up to the contract."""
72126
if not self.table_exists(contract.dataset):
73127
self.connection.execute(build_create_table(contract, self.schema))
74128
log.info(
@@ -78,21 +132,43 @@ def ensure_table(self, contract: Contract) -> None:
78132
return
79133
self.evolve_table(contract)
80134

81-
def evolve_table(self, contract: Contract) -> list[str]:
82-
"""Add columns present in the contract but not yet in the table. Returns names."""
83-
existing = set(self.columns(contract.dataset))
135+
def evolve_table(self, contract: Contract) -> dict[str, list[str]]:
136+
"""Bring the physical table up to the contract: add columns, promote types.
137+
138+
Both halves are needed. Adding the new columns alone was the first version of
139+
this method and it was wrong: when the classifier promoted account_id from
140+
INTEGER to BIGINT, the contract on disk said BIGINT while DuckDB still said
141+
INTEGER, and the next insert failed with an out of range conversion. A "widening
142+
is safe" guarantee that stops at the registry is not a guarantee.
143+
144+
Promotion is gated on ``is_widening``, so this can never quietly narrow a column
145+
and truncate landed history. A narrowing must go through a contract amendment
146+
that a person approved.
147+
"""
148+
physical = self.column_types(contract.dataset)
84149
added: list[str] = []
150+
promoted: list[str] = []
85151
for f in contract.fields:
86-
if f.name in existing:
152+
if f.name not in physical:
153+
self.connection.execute(build_add_column(contract, self.schema, f.name))
154+
added.append(f.name)
87155
continue
88-
self.connection.execute(build_add_column(contract, self.schema, f.name))
89-
added.append(f.name)
90-
if added:
156+
current = duckdb_type_to_spec(physical[f.name])
157+
if current is not None and is_widening(current, f.type):
158+
self.connection.execute(build_alter_type(contract, self.schema, f.name))
159+
promoted.append(f"{f.name}:{current.sql_type()}->{f.type.sql_type()}")
160+
if added or promoted:
91161
log.info(
92162
"evolved landing table",
93-
extra={"context": {"table": contract.dataset, "added_columns": added}},
163+
extra={
164+
"context": {
165+
"table": contract.dataset,
166+
"added_columns": added,
167+
"promoted_columns": promoted,
168+
}
169+
},
94170
)
95-
return added
171+
return {"added": added, "promoted": promoted}
96172

97173
# ------------------------------------------------------------------ writes
98174

0 commit comments

Comments
 (0)