Skip to content

Commit f25fff0

Browse files
committed
fix(gate,health): exact seeded-interval query check and honest probe budget
Review fixes (PR #218): - The query phase asks for the exact deterministic seeded interval [FirstWindow, LastWindow+5m) — exactly 2016 windows, inside the engine read-range cap, excluding the protocol's live windows — instead of a trailing window with boundary-trimmed expectations. HOT_RETENTION_DAYS=8 in the gate config keeps every seeded window alive through the run. Completeness now asserts extra == 0 as well as missing == 0; the extra count was previously computed and discarded. - READY_AGGREGATE_DISK_BUDGET_MB returns to the real tier allocation (2304 MB) and READY_MAX_AGGREGATE_DISK_RATIO defaults to 1.0: readiness fails exactly at the tier boundary instead of using a synthetic 2560 denominator to fake the same trip point. Boundary pinned by tests at one byte below, exactly at, and one byte above the tier. - Comment wording: 4 MiB/window projects to 4.5 GiB, which exceeds the 4.0 GiB main tier rather than merely consuming it.
1 parent 0c29eca commit f25fff0

10 files changed

Lines changed: 65 additions & 36 deletions

File tree

CLAUDE.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -701,8 +701,8 @@ Thresholds (`0` disables that probe):
701701
| `READY_MAX_FINALIZE_FAILURE_STREAK` | 3 | Same shape, on the finalizer. |
702702
| `READY_MAX_ADMISSION_RATIO` | 0.9 | Below the 0.95 the DLQ/pipeline probes use: the writer's admission bound is what turns an Export into `RESOURCE_EXHAUSTED`, so readiness says "stop sending" before clients are refused, not while they are. |
703703
| `READY_MAX_DELTA_LOG_AGE_S` | 1800 | 2× (`WindowSize` 5m + `AllowedLateness` 10m). A window is finalizable 900s after it opens, so a healthy oldest entry tops out just past 900s plus one finalize tick. |
704-
| `READY_AGGREGATE_DISK_BUDGET_MB` | 2560 | `aggregate.db`'s share of the 8 GiB data budget (#201 Q1). The disk watchdog enforces the **volume**; this enforces the **tier**, so a runaway aggregate file is visible before it eats another tier's allocation. |
705-
| `READY_MAX_AGGREGATE_DISK_RATIO` | 0.9 | Warn inside the tier before the volume-level ladder starts shedding. |
704+
| `READY_AGGREGATE_DISK_BUDGET_MB` | 2304 | `aggregate.db`'s share of the 8 GiB data budget (#201 Q1). The disk watchdog enforces the **volume**; this enforces the **tier**, so a runaway aggregate file is visible before it eats another tier's allocation. |
705+
| `READY_MAX_AGGREGATE_DISK_RATIO` | 1.0 | Readiness fails exactly at the tier allocation. Full-density steady state (~0.92 of the tier) is legitimate; earlier warning is the volume-level watchdog ladder's job. |
706706

707707
## Security & Supply Chain
708708

internal/api/health_runtime_test.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func TestReadyFlipsOnEachAggregateRuntimeProbe(t *testing.T) {
6363
{"finalize failure streak", "aggregate_finalizer", func(rt *AggregateRuntime) { rt.FinalizeFailureStreak = 4 }, "4"},
6464
{"admission saturation", "aggregate_admission", func(rt *AggregateRuntime) { rt.AdmissionRatio = 0.91 }, "0.91"},
6565
{"delta log age", "aggregate_delta_log", func(rt *AggregateRuntime) { rt.DeltaLogAgeSeconds = 1801 }, "1801"},
66-
{"aggregate disk", "aggregate_disk", func(rt *AggregateRuntime) { rt.DiskUsedBytes = 1400 << 20 }, "0.91"},
66+
{"aggregate disk", "aggregate_disk", func(rt *AggregateRuntime) { rt.DiskUsedBytes = 1536 << 20 }, "1"},
6767
} {
6868
t.Run(tc.name, func(t *testing.T) {
6969
s := newTestServer(t)
@@ -213,3 +213,32 @@ func TestLiveUnaffectedByRuntimeProbeFailures(t *testing.T) {
213213
t.Fatalf("/live status = %q, want alive", body["status"])
214214
}
215215
}
216+
217+
// TestAggregateDiskProbeTierBoundary pins the probe to the honest semantics:
218+
// the budget field IS the tier allocation and the 1.0 default ratio means
219+
// readiness fails exactly at the boundary — one byte below passes, the
220+
// boundary itself and anything above it fail.
221+
func TestAggregateDiskProbeTierBoundary(t *testing.T) {
222+
const tier = int64(2304) << 20
223+
for _, tc := range []struct {
224+
name string
225+
used int64
226+
want int
227+
}{
228+
{"one byte below the tier", tier - 1, http.StatusOK},
229+
{"exactly at the tier", tier, http.StatusServiceUnavailable},
230+
{"one byte above the tier", tier + 1, http.StatusServiceUnavailable},
231+
} {
232+
t.Run(tc.name, func(t *testing.T) {
233+
s := newTestServer(t)
234+
rt := healthyRuntime()
235+
rt.DiskUsedBytes = tc.used
236+
rt.DiskBudgetBytes = tier
237+
s.SetAggregateRuntimeProbe(func() AggregateRuntime { return rt })
238+
if code, checks := readyChecks(t, s); code != tc.want {
239+
t.Fatalf("/ready = %d with used=%d budget=%d, want %d (checks: %v)",
240+
code, tc.used, tier, tc.want, checks)
241+
}
242+
})
243+
}
244+
}

internal/api/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func DefaultReadinessThresholds() ReadinessThresholds {
119119
MaxFinalizeFailureStreak: 3,
120120
MaxAdmissionRatio: 0.9,
121121
MaxDeltaLogAgeSeconds: 1800,
122-
MaxAggregateDiskRatio: 0.9,
122+
MaxAggregateDiskRatio: 1.0,
123123
}
124124
}
125125

internal/config/config.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -664,9 +664,9 @@ func Load(customPath string) (*Config, error) {
664664
ExemplarTracesPerServiceWindow: getEnvInt("EXEMPLAR_TRACES_PER_SERVICE_WINDOW", 25),
665665
ExemplarTracesGlobalWindow: getEnvInt("EXEMPLAR_TRACES_GLOBAL_WINDOW", 1500),
666666
ExemplarBytesPerServiceWindow: getEnvInt("EXEMPLAR_BYTES_PER_SERVICE_WINDOW", 512*1024),
667-
// 3 MiB, not 4 (#201 Q2). 4 MiB/window consumes the entire 4.0 GiB
668-
// main tier under the optimistic 2x amplification assumption and
669-
// leaves no operational margin; it stays configurable, it is not the
667+
// 3 MiB, not 4 (#201 Q2). 4 MiB/window projects to 4.5 GiB under the
668+
// 2x amplification assumption, exceeding the 4.0 GiB main tier
669+
// outright; it stays configurable, it is not the
670670
// default until the seven-day gate (#202) proves it fits.
671671
ExemplarBytesGlobalWindow: getEnvInt("EXEMPLAR_BYTES_GLOBAL_WINDOW", 3*1024*1024),
672672
ExemplarHealthyRate: getEnvFloat("EXEMPLAR_HEALTHY_RATE", 0.005),
@@ -700,17 +700,17 @@ func Load(customPath string) (*Config, error) {
700700
// readiness should say "stop sending" before clients are being refused,
701701
// not while they are.
702702
ReadyMaxAdmissionRatio: getEnvFloat("READY_MAX_ADMISSION_RATIO", 0.9),
703-
// The aggregate tier allocation is 2.25 GiB (#201 Q1, rebalanced
703+
// 2.25 GiB is the aggregate tier allocation (#201 Q1, rebalanced
704704
// 2026-08-22 after the gate measured 2.08 GiB at full density). The
705-
// probe budget is deliberately 2560 so the 0.90 warn ratio trips at
706-
// 0.9 x 2560 MB = 2304 MB — exactly the tier boundary. Full-density
707-
// steady state (~0.92 of the tier) is legitimate and must not flip
708-
// readiness; crossing the tier allocation must.
705+
// field IS the tier budget — no synthetic denominators. Full-density
706+
// steady state (~0.92 of the tier) is legitimate, so the ratio below
707+
// defaults to 1.0: readiness fails exactly at the tier boundary, and
708+
// earlier warning belongs to the volume-level watchdog ladder.
709709
// The disk watchdog enforces the VOLUME; this enforces the tier, so a
710710
// runaway aggregate file is visible before it eats another tier's
711711
// allocation and takes the whole volume past 95% with it.
712-
ReadyAggregateDiskBudgetMB: getEnvInt("READY_AGGREGATE_DISK_BUDGET_MB", 2560),
713-
ReadyMaxAggregateDiskRatio: getEnvFloat("READY_MAX_AGGREGATE_DISK_RATIO", 0.9),
712+
ReadyAggregateDiskBudgetMB: getEnvInt("READY_AGGREGATE_DISK_BUDGET_MB", 2304),
713+
ReadyMaxAggregateDiskRatio: getEnvFloat("READY_MAX_AGGREGATE_DISK_RATIO", 1.0),
714714
}
715715

716716
// Parse AGGREGATE_METRIC_DIMS config

internal/config/config_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1034,11 +1034,11 @@ func TestLoad_ReadinessThresholdDefaults(t *testing.T) {
10341034
if cfg.ReadyMaxAdmissionRatio != 0.9 {
10351035
t.Errorf("READY_MAX_ADMISSION_RATIO = %v, want 0.9", cfg.ReadyMaxAdmissionRatio)
10361036
}
1037-
if cfg.ReadyAggregateDiskBudgetMB != 2560 {
1038-
t.Errorf("READY_AGGREGATE_DISK_BUDGET_MB = %d, want 2560", cfg.ReadyAggregateDiskBudgetMB)
1037+
if cfg.ReadyAggregateDiskBudgetMB != 2304 {
1038+
t.Errorf("READY_AGGREGATE_DISK_BUDGET_MB = %d, want 2304", cfg.ReadyAggregateDiskBudgetMB)
10391039
}
1040-
if cfg.ReadyMaxAggregateDiskRatio != 0.9 {
1041-
t.Errorf("READY_MAX_AGGREGATE_DISK_RATIO = %v, want 0.9", cfg.ReadyMaxAggregateDiskRatio)
1040+
if cfg.ReadyMaxAggregateDiskRatio != 1.0 {
1041+
t.Errorf("READY_MAX_AGGREGATE_DISK_RATIO = %v, want 1.0", cfg.ReadyMaxAggregateDiskRatio)
10421042
}
10431043
}
10441044

test/gate/gatecore/assert.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,10 @@ func queryAssertions(r *Result, t Thresholds) []Assertion {
540540
"query surface "+c.Name+" returned every seeded window",
541541
c.URL, int64(c.WindowsReturned), int64(c.WindowsExpected),
542542
fmt.Sprintf("%d windows missing", c.MissingWindows)))
543+
out = append(out, eqInt(id+".windows_extra", catQuery,
544+
"query surface "+c.Name+" returned no windows outside the seeded interval",
545+
c.URL, int64(c.ExtraWindows), 0,
546+
fmt.Sprintf("%d extra windows", c.ExtraWindows)))
543547
}
544548
if c.CoverageExpected != "" {
545549
out = append(out, pass(id+".coverage", catQuery,

test/gate/gatecore/jsonscan_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ func TestWindowCoverage(t *testing.T) {
9797
base := time.Date(2026, 8, 22, 0, 0, 0, 0, time.UTC)
9898
pts := []WindowPoint{
9999
{Timestamp: base},
100-
{Timestamp: base.Add(10 * time.Minute)},
101-
{Timestamp: base.Add(30 * time.Minute)}, // outside the expected set
100+
{Timestamp: base.Add(10*time.Minute + 42*time.Second)}, // unaligned: same window as base+10m
101+
{Timestamp: base.Add(30 * time.Minute)}, // outside the expected set
102102
}
103103
expected := []int64{base.Unix(), base.Add(5 * time.Minute).Unix(), base.Add(10 * time.Minute).Unix()}
104104
returned, missing, extra := WindowCoverage(pts, expected, 300)

test/gate/gatecore/result.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ type QueryCheck struct {
351351
WindowsReturned int `json:"windows_returned,omitempty"`
352352
WindowsExpected int `json:"windows_expected,omitempty"`
353353
MissingWindows int `json:"missing_windows,omitempty"`
354+
ExtraWindows int `json:"extra_windows,omitempty"`
354355
Scalars map[string]float64 `json:"scalars,omitempty"`
355356
BodyBytes int `json:"body_bytes"`
356357
Error string `json:"error,omitempty"`

test/gate/main.go

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -694,22 +694,16 @@ func (g *gate) collectDisk() {
694694
}
695695

696696
func (g *gate) collectQueries(prefill prefillFacts) {
697-
// The trailing true seven-day window, not first-seeded-window..now: after
698-
// ~4h of protocol the seeded range spans 7d+4h, which the engine's read
699-
// guard refuses (cap: 7d + one window). Retention has also made seeded
700-
// windows older than 7d eligible for purge, so they cannot be part of a
701-
// completeness expectation. Expected coverage is trimmed to the seeded
702-
// windows strictly inside the trailing range.
703-
end := time.Now().UTC()
704-
start := end.Add(-7 * 24 * time.Hour)
697+
// The exact deterministic seeded interval, not first-seeded-window..now:
698+
// the latter grows past the engine's read-range cap (7d + one window) as
699+
// the protocol runs. [FirstWindow, LastWindow+5m) spans exactly the 2016
700+
// seeded windows (7d, inside the cap), excludes the protocol's own live
701+
// windows, and HOT_RETENTION_DAYS=8 in the gate config keeps every seeded
702+
// window alive through the run — so completeness is missing==0 AND
703+
// extra==0 over the full deterministic set.
704+
start := time.Unix(prefill.FirstWindow, 0).UTC()
705+
end := time.Unix(prefill.LastWindow, 0).UTC().Add(5 * time.Minute)
705706
expected := windowRange(prefill.FirstWindow, prefill.LastWindow)
706-
trimmed := expected[:0]
707-
for _, w := range expected {
708-
if w >= start.Unix()+300 {
709-
trimmed = append(trimmed, w)
710-
}
711-
}
712-
expected = trimmed
713707

714708
g.res.Queries.PrefillRangeStart = start
715709
g.res.Queries.PrefillRangeEnd = end

test/gate/queries.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,9 @@ func (g *gate) runAPICheck(spec gatecore.APICheck, sevenDayStart, sevenDayEnd ti
7777
c.Error = perr.Error()
7878
return c
7979
}
80-
returned, missing, _ := gatecore.WindowCoverage(pts, expectedWindows, gatecore.WindowSecs)
80+
returned, missing, extra := gatecore.WindowCoverage(pts, expectedWindows, gatecore.WindowSecs)
8181
c.WindowsReturned, c.MissingWindows, c.WindowsExpected = returned, missing, len(expectedWindows)
82+
c.ExtraWindows = extra
8283
}
8384
// Whatever coverage marker arrived is always recorded; only surfaces the
8485
// config marks as required are gated on it.

0 commit comments

Comments
 (0)