Skip to content

Commit a5f38e1

Browse files
igerberclaude
andcommitted
Surface row-count for axis-E silent coercion / drop sites
Three long-standing axis-E (silent data sanitization) sites now emit a UserWarning with the affected row count before the coercion, so users can no longer be silently shifted between the treated and control sides of an estimator or have rows vanish from a diagnostic pass without a signal. - WooldridgeDiD: NaN cohort values were filled to 0 (never-treated) both in `_filter_sample` and in `fit()`. Both now warn with the NaN row count before the fillna (finding #24). - ContinuousDiD: `first_treat=inf` was replaced with 0 silently. `fit()` now counts inf rows and warns before the replace, before any downstream drop-zero-dose / negative-dose validation (finding #26). - `_compute_outcome_changes` (the `check_parallel_trends` diff helper) dropped NaN first-differences without reporting the count. It now distinguishes the expected first-period-per-unit drops from excess drops caused by gaps / NaN outcomes and warns with the breakdown when excess drops are detected (finding #27). Finding #25 (TROP D-matrix coercion) was verified during scoping to already be resolved in `trop_local.py:60-66` via `n_missing_structural` + returned `missing_mask`; no code change required. REGISTRY updated under both WooldridgeDiD and ContinuousDiD to document the new warning contract. Covered by audit axis E (data sanitization). Findings #24, #26, #27 from docs/audits/silent-failures-findings.md. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 328dc33 commit a5f38e1

7 files changed

Lines changed: 186 additions & 8 deletions

File tree

‎diff_diff/continuous_did.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,20 @@ def fit(
227227
f"Dose must be time-invariant. Units with varying dose: {bad_units[:5]}"
228228
)
229229

230-
# Normalize first_treat: inf → 0
230+
# Normalize first_treat: inf → 0 (R-style never-treated encoding). Count
231+
# rows recategorized so users can see how many units just crossed from
232+
# "treated at some point" to "never treated" — silent recategorization
233+
# here would shift the control composition (axis-E silent coercion).
234+
inf_mask = np.isinf(df[first_treat].values)
235+
n_inf_first_treat = int(inf_mask.sum())
236+
if n_inf_first_treat > 0:
237+
warnings.warn(
238+
f"{n_inf_first_treat} row(s) have inf in '{first_treat}'; "
239+
f"treating the corresponding units as never-treated. Pass an "
240+
f"explicit never-treated marker (0) if this is not intended.",
241+
UserWarning,
242+
stacklevel=2,
243+
)
231244
df[first_treat] = df[first_treat].replace([np.inf, float("inf")], 0)
232245

233246
# Drop units with positive first_treat but zero dose (R convention)

‎diff_diff/utils.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -925,7 +925,23 @@ def _compute_outcome_changes(
925925
data_sorted = data.sort_values([unit, time])
926926
data_sorted["_outcome_change"] = data_sorted.groupby(unit)[outcome].diff()
927927

928-
# Remove NaN from first period of each unit
928+
# Remove NaN from first period of each unit. The first period per unit
929+
# has no prior observation to diff against, so n_units drops are
930+
# expected. Anything beyond that is a silent side-effect of gaps or
931+
# NaN outcomes — surface the excess via warning (axis-E drop counter).
932+
n_units_observed = int(data_sorted[unit].nunique())
933+
n_dropped = int(data_sorted["_outcome_change"].isna().sum())
934+
n_unexpected_drops = max(0, n_dropped - n_units_observed)
935+
if n_unexpected_drops > 0:
936+
warnings.warn(
937+
f"check_parallel_trends dropped {n_dropped} row(s) with NaN "
938+
f"first-differences; {n_units_observed} are the expected "
939+
f"first-period-per-unit drops, and {n_unexpected_drops} came "
940+
f"from gaps or NaN outcomes. Parallel-trend statistics are "
941+
f"computed on the remaining rows.",
942+
UserWarning,
943+
stacklevel=3,
944+
)
929945
changes_data = data_sorted.dropna(subset=["_outcome_change"])
930946

931947
treated_changes = changes_data[changes_data[treatment_group] == 1]["_outcome_change"].values

‎diff_diff/wooldridge.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
from __future__ import annotations
1515

16+
import warnings
1617
from typing import Any, Dict, List, Optional, Tuple
1718

1819
import numpy as np
@@ -128,7 +129,19 @@ def _filter_sample(
128129
(see _build_interaction_matrix).
129130
"""
130131
df = data.copy()
131-
# Normalise never-treated: fill NaN cohort with 0
132+
# Normalise never-treated: fill NaN cohort with 0. Report the row count so
133+
# callers can see how many rows were recategorized as never-treated — a
134+
# silent recategorization here would quietly move units between the
135+
# treated and control sides of the estimator (axis-E silent coercion).
136+
n_nan_cohort = int(df[cohort].isna().sum())
137+
if n_nan_cohort > 0:
138+
warnings.warn(
139+
f"{n_nan_cohort} row(s) have NaN cohort values; filling with 0 "
140+
f"and treating the corresponding units as never-treated. Pass "
141+
f"an explicit never-treated marker (0) if this is not intended.",
142+
UserWarning,
143+
stacklevel=3,
144+
)
132145
df[cohort] = df[cohort].fillna(0)
133146

134147
treated_mask = df[cohort] > 0
@@ -396,6 +409,18 @@ def fit(
396409
``NotImplementedError``.
397410
"""
398411
df = data.copy()
412+
# See `_filter_sample` for the analogous warning; fit() does its own
413+
# fillna earlier in the pipeline so we warn here too to cover the
414+
# direct-fit path.
415+
n_nan_cohort = int(df[cohort].isna().sum())
416+
if n_nan_cohort > 0:
417+
warnings.warn(
418+
f"{n_nan_cohort} row(s) have NaN cohort values; filling with 0 "
419+
f"and treating the corresponding units as never-treated. Pass "
420+
f"an explicit never-treated marker (0) if this is not intended.",
421+
UserWarning,
422+
stacklevel=2,
423+
)
399424
df[cohort] = df[cohort].fillna(0)
400425

401426
# 0a. Validate cohort is time-invariant within unit

‎docs/methodology/REGISTRY.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,7 @@ See `docs/methodology/continuous-did.md` Section 4 for full details.
720720
- [ ] Lowest-dose-as-control (Remark 3.1)
721721
- [x] Survey design support (Phase 3): weighted B-spline OLS, TSL on influence functions; bootstrap+survey supported (Phase 6)
722722
- **Note:** ContinuousDiD bootstrap with survey weights supported (Phase 6) via PSU-level multiplier weights
723+
- **Note:** The R-style convention of coding never-treated units as `first_treat=inf` is still accepted and normalized to `first_treat=0` internally, but the estimator now emits a `UserWarning` reporting the row count so the silent recategorization is surfaced (axis-E silent coercion under the Phase 2 audit). Pass `0` directly to avoid the warning.
723724

724725
---
725726

@@ -1303,6 +1304,7 @@ The saturated ETWFE regression includes:
13031304
The interaction coefficient `δ_{g,t}` identifies `ATT(g, t)` under parallel trends.
13041305
- **Note:** OLS path uses iterative alternating-projection within-transformation (uniform weights) for exact FE absorption on both balanced and unbalanced panels. One-pass demeaning (`y - ȳ_i - ȳ_t + ȳ`) is only exact for balanced panels.
13051306
- **Note:** The weighted within-transformation (`utils.within_transform` with `weights`) is invoked on every WooldridgeDiD fit (survey weights when provided, `np.ones` otherwise) and emits a `UserWarning` on non-convergence per the shared convention documented under *Absorbed Fixed Effects with Survey Weights*.
1307+
- **Note:** NaN values in the `cohort` column are filled with 0 (treated as never-treated), both in `_filter_sample` and in `fit()`. This recategorization now emits a `UserWarning` reporting the affected row count so it is no longer silent (axis-E silent coercion under the Phase 2 audit). Pass `0` directly for never-treated units to avoid the warning.
13061308

13071309
*Nonlinear extensions (Wooldridge 2023):*
13081310

‎tests/test_continuous_did.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -641,16 +641,40 @@ def test_few_treated_units(self):
641641
assert isinstance(results, ContinuousDiDResults)
642642

643643
def test_inf_first_treat_normalization(self):
644-
"""first_treat=inf should be treated as never-treated."""
644+
"""first_treat=inf should be treated as never-treated, and the caller
645+
must receive a UserWarning reporting the affected row count so the
646+
recategorization is not silent (axis-E counter)."""
645647
data = generate_continuous_did_data(n_units=50, n_periods=3, seed=42)
646648
data["first_treat"] = data["first_treat"].astype(float)
647-
data.loc[data["first_treat"] == 0, "first_treat"] = np.inf
649+
inf_mask = data["first_treat"] == 0
650+
n_inf_rows = int(inf_mask.sum())
651+
data.loc[inf_mask, "first_treat"] = np.inf
648652
est = ContinuousDiD()
649-
results = est.fit(
650-
data, "outcome", "unit", "period", "first_treat", "dose"
651-
)
653+
654+
with pytest.warns(
655+
UserWarning,
656+
match=rf"{n_inf_rows} row\(s\) have inf in 'first_treat'",
657+
):
658+
results = est.fit(
659+
data, "outcome", "unit", "period", "first_treat", "dose"
660+
)
652661
assert results.n_control_units > 0
653662

663+
def test_no_inf_first_treat_no_warning(self):
664+
"""No inf rows in first_treat — no recategorization warning."""
665+
import warnings
666+
667+
data = generate_continuous_did_data(n_units=50, n_periods=3, seed=42)
668+
data["first_treat"] = data["first_treat"].astype(float)
669+
est = ContinuousDiD()
670+
671+
with warnings.catch_warnings(record=True) as w:
672+
warnings.simplefilter("always")
673+
est.fit(data, "outcome", "unit", "period", "first_treat", "dose")
674+
675+
inf_warnings = [x for x in w if "inf in 'first_treat'" in str(x.message)]
676+
assert inf_warnings == []
677+
654678
def test_custom_dvals(self):
655679
data = generate_continuous_did_data(n_units=100, n_periods=3, seed=42)
656680
custom_grid = np.array([1.0, 2.0, 3.0])

‎tests/test_utils.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -827,6 +827,57 @@ def test_changes_reflect_trend(self):
827827
np.testing.assert_array_almost_equal(treated_changes, 2.0, decimal=5)
828828
np.testing.assert_array_almost_equal(control_changes, 2.0, decimal=5)
829829

830+
def test_silent_on_balanced_panel(self):
831+
"""Balanced panel: only first-period-per-unit drops, no warning."""
832+
import warnings
833+
834+
rng = np.random.default_rng(0)
835+
rows = []
836+
for unit in range(10):
837+
treated = int(unit >= 5)
838+
for t in range(1, 5):
839+
rows.append({
840+
"unit": unit, "period": t,
841+
"treated": treated, "outcome": rng.normal(),
842+
})
843+
df = pd.DataFrame(rows)
844+
845+
with warnings.catch_warnings(record=True) as w:
846+
warnings.simplefilter("always")
847+
_compute_outcome_changes(
848+
df, outcome="outcome", time="period",
849+
treatment_group="treated", unit="unit",
850+
)
851+
852+
drop_warnings = [
853+
x for x in w if "check_parallel_trends dropped" in str(x.message)
854+
]
855+
assert drop_warnings == []
856+
857+
def test_warns_on_nan_outcomes_with_excess_drop_count(self):
858+
"""Extra NaN-outcome rows beyond first-period drops must surface via
859+
a UserWarning reporting the excess count (axis-E drop counter)."""
860+
rng = np.random.default_rng(0)
861+
rows = []
862+
for unit in range(10):
863+
treated = int(unit >= 5)
864+
for t in range(1, 5):
865+
rows.append({
866+
"unit": unit, "period": t,
867+
"treated": treated, "outcome": rng.normal(),
868+
})
869+
df = pd.DataFrame(rows)
870+
df.loc[[5, 12, 22], "outcome"] = np.nan
871+
872+
with pytest.warns(
873+
UserWarning,
874+
match=r"check_parallel_trends dropped \d+ row\(s\).*first-period-per-unit",
875+
):
876+
_compute_outcome_changes(
877+
df, outcome="outcome", time="period",
878+
treatment_group="treated", unit="unit",
879+
)
880+
830881

831882
# =============================================================================
832883
# Tests for check_parallel_trends_robust

‎tests/test_wooldridge.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1593,3 +1593,50 @@ def test_survey_aggregate_and_summary(self, survey_panel):
15931593
s = r.summary()
15941594
assert "Survey Design" in s
15951595
assert "pweight" in s
1596+
1597+
1598+
class TestCohortNaNWarning:
1599+
"""Axis-E: silent recategorization of NaN cohort rows as never-treated."""
1600+
1601+
@staticmethod
1602+
def _make_panel_with_nan_cohort():
1603+
rows = []
1604+
for unit in range(10):
1605+
cohort_val = np.nan if unit < 2 else 0.0
1606+
for t in range(1, 5):
1607+
rows.append({
1608+
"unit": unit, "time": t, "cohort": cohort_val,
1609+
"y": unit + t + np.random.default_rng(unit).normal(0, 0.1),
1610+
})
1611+
return pd.DataFrame(rows)
1612+
1613+
def test_fit_warns_on_nan_cohort_with_count(self):
1614+
df = self._make_panel_with_nan_cohort()
1615+
est = WooldridgeDiD(method="ols")
1616+
with pytest.warns(UserWarning, match=r"8 row\(s\) have NaN cohort values"):
1617+
try:
1618+
est.fit(df, outcome="y", unit="unit", time="time", cohort="cohort")
1619+
except Exception:
1620+
pass
1621+
1622+
def test_fit_silent_on_clean_cohort(self):
1623+
import warnings
1624+
df = self._make_panel_with_nan_cohort()
1625+
df["cohort"] = df["cohort"].fillna(0)
1626+
est = WooldridgeDiD(method="ols")
1627+
with warnings.catch_warnings(record=True) as w:
1628+
warnings.simplefilter("always")
1629+
try:
1630+
est.fit(df, outcome="y", unit="unit", time="time", cohort="cohort")
1631+
except Exception:
1632+
pass
1633+
nan_warnings = [x for x in w if "NaN cohort values" in str(x.message)]
1634+
assert nan_warnings == []
1635+
1636+
def test_select_sample_helper_warns(self):
1637+
df = self._make_panel_with_nan_cohort()
1638+
with pytest.warns(UserWarning, match=r"8 row\(s\) have NaN cohort values"):
1639+
_filter_sample(
1640+
df, unit="unit", time="time", cohort="cohort",
1641+
control_group="never_treated", anticipation=0,
1642+
)

0 commit comments

Comments
 (0)