compare_analysis sigma-bands the v2 PostgreSQL baselined metrics — I/O read latency, replay lag, WAL volume — so a 16× latency move stops reading by the ladder alone; the write family has one definition of "WAL tracked" (partial #3691)

Darling/Darling.Tests/CompareAnalysisPgBandingLiveTests.cs

@@ -36,9 +36,18 @@ namespace Darling.Tests;
 /// window's start exactly as the detector anchors, and the row comes back <c>worse</c> by sigma with the bucket's
 /// figures beside it. W2 rides along: the payload's <c>plan_cache_churn.note</c> is the PostgreSQL sentence.
 ///
+/// <para><b>The v2 keys (#3691 exit check, the same gap once more).</b> The v2 lanes stored three more <c>pg_</c>
+/// buckets and the exit check found <c>PG_IO_READ_LATENCY_MS</c>, <c>PG_REPLICATION_LAG</c> and <c>PG_WAL_VOLUME_SHIFT</c>
+/// banded <c>absolute</c> / <c>baseline_metric: null</c> in the window their detectors called 25σ. The same store
+/// now plants the I/O series beside the TPS one — 31 days at ~1.5 ms per read, the surge window at 25 ms — and the
+/// I/O row comes back <c>worse</c> by sigma against <c>pg_io_read_latency</c>; the three keys' arithmetic and
+/// withheld shapes are pinned ungated in <c>ComparisonBandingTests</c>.</para>
+///
 /// <para>Gated on <c>DARLING_TEST_PG</c>; <c>[Collection("live-postgres")]</c> (#1776) and the #1902 cleanup shape.
-/// The planted series is the anomaly e2e's (<c>PgTargetAnomalyTests</c>) with only <c>pg_database_stats</c>, which is
-/// also the coverage witness, so both windows read fully observed and no coverage caveat rides on the verdict.</para>
+/// The planted series is the anomaly e2e's (<c>PgTargetAnomalyTests</c>) on <c>pg_database_stats</c>, which is also
+/// the coverage witness, so both windows read fully observed and no coverage caveat rides on the verdict; the
+/// <c>pg_io_stats</c> series is one identity (<c>client backend</c> / <c>normal</c> / <c>relation</c>) at the same
+/// minute cadence, so the fact's window quotient and the bucket's quarter-hour quotient are the same number.</para>
 /// </summary>
 [Collection("live-postgres")]
 public sealed class CompareAnalysisPgBandingLiveTests
@@ -95,11 +104,40 @@ INSERT INTO pg_database_stats
                 await plant.ExecuteNonQueryAsync(ct);
             }
 
-            /* The bucket the tool will read: trustworthy, centred on the steady state, tight. */
+            /* The I/O series, same clock: one identity, 110 reads a minute (1,650 a quarter-hour — well over the
+               bucket's 250-read floor), cumulative read time at 1.5 ms per read with a deterministic ±0.15 ms ripple
+               so the bucket's MAD is small but non-zero; 25 ms per read across the surge minutes. Counters are
+               cumulative with no reset, as the PG 18 collector writes them; writes are NULL (not under test). */
+            using (var plantIo = new NpgsqlCommand(@"
+WITH s AS (
+    SELECT n, 110 AS reads, CASE WHEN n >= $5 THEN 25.0 ELSE 1.5 + 0.15 * sin(n) END AS ms_per_read
+    FROM generate_series(0, $6) AS n
+)
+INSERT INTO pg_io_stats
+    (collection_id, collection_time, server_id, server_name, backend_type, object_type, context,
+     reads, read_time_ms, writes, write_time_ms, stats_reset)
+SELECT $1 + n, $2 + (n * interval '1 minute'), $3, $4, 'client backend', 'relation', 'normal',
+       SUM(reads) OVER (ORDER BY n), SUM(reads * ms_per_read) OVER (ORDER BY n), NULL::bigint, NULL::double precision, NULL::timestamp
+FROM s", connection) { CommandTimeout = 120 })
+            {
+                plantIo.Parameters.AddWithValue(CollectionIdGenerator.Next() + 3_000_000L);
+                plantIo.Parameters.AddWithValue(start);
+                plantIo.Parameters.AddWithValue(ServerId);
+                plantIo.Parameters.AddWithValue(ServerName);
+                plantIo.Parameters.AddWithValue(surgeFrom);
+                plantIo.Parameters.AddWithValue(minutes);
+                await plantIo.ExecuteNonQueryAsync(ct);
+            }
+
+            /* The buckets the tool will read: trustworthy, centred on the steady state, tight. */
             var bucket = await new PgTargetBaselineProvider(postgres).GetBaselineAsync(ServerId, MetricNames.PgTps, end.AddHours(-4), ct);
             Assert.True(bucket.IsTrustworthy, "the 31-day pg_tps bucket is not trustworthy");
             Assert.InRange(bucket.Median, 9.0, 11.0);
             Assert.True(bucket.EffectiveRobustSigma > 0);
+            var ioBucket = await new PgTargetBaselineProvider(postgres).GetBaselineAsync(ServerId, MetricNames.PgIoReadLatency, end.AddHours(-4), ct);
+            Assert.True(ioBucket.IsTrustworthy, "the 31-day pg_io_read_latency bucket is not trustworthy");
+            Assert.InRange(ioBucket.Median, 1.3, 1.7);
+            Assert.True(ioBucket.EffectiveRobustSigma > 0);
 
             var service = new DarlingAnalysisService(postgres);
             var json = await DarlingMcpTools.CompareAnalysis(service, postgres, ServerName, hours_back: 4, baseline_hours_back: 28);
@@ -121,7 +159,19 @@ INSERT INTO pg_database_stats
 
             var family = Assert.Single(root.GetProperty("families").EnumerateArray(), f => f.GetProperty("family").GetString() == PgTargetFactKeys.Tps);
             Assert.Equal(ComparisonBanding.StatusWorse, family.GetProperty("status").GetString());
-            Assert.True(root.GetProperty("summary").GetProperty("families_worse").GetInt32() >= 1);
+            Assert.True(root.GetProperty("summary").GetProperty("families_worse").GetInt32() >= 2);
+
+            /* The v2 key: the exit check's before picture was absolute / null on exactly this shape (1.55 → 25 ms). */
+            var io = Assert.Single(facts.EnumerateArray(), f => f.GetProperty("key").GetString() == PgTargetFactKeys.IoReadLatencyMs);
+            Assert.InRange(io.GetProperty("baseline_value").GetDouble(), 1.3, 1.7);
+            Assert.InRange(io.GetProperty("comparison_value").GetDouble(), 24.9, 25.1);
+            Assert.Equal(ComparisonBanding.StatusWorse, io.GetProperty("status").GetString());
+            Assert.Equal(ComparisonBanding.BandSourceBaseline, io.GetProperty("band_source").GetString());
+            Assert.Equal(MetricNames.PgIoReadLatency, io.GetProperty("baseline_metric").GetString());
+            Assert.True(io.GetProperty("beyond_anomaly_cutoff").GetBoolean());
+            Assert.Equal(AnomalyThresholds.SigmaDisplayCap, io.GetProperty("delta_sigma").GetDouble());
+            Assert.InRange(io.GetProperty("baseline_median").GetDouble(), 1.3, 1.7);
+            Assert.False(io.GetProperty("coverage_caveat").GetBoolean());
 
             Assert.Equal(PlanCacheChurn.PostgresNote, root.GetProperty("plan_cache_churn").GetProperty("note").GetString());
 
@@ -141,6 +191,7 @@ private static async Task DeleteRowsAsync(NpgsqlConnection connection, Cancellat
     {
         using var cleanup = new NpgsqlCommand(
             $"DELETE FROM pg_database_stats WHERE server_id = {ServerId}; " +
+            $"DELETE FROM pg_io_stats WHERE server_id = {ServerId}; " +
             $"DELETE FROM analysis_findings WHERE server_id = {ServerId}; " +
             $"DELETE FROM analysis_muted WHERE server_id = {ServerId}; " +
             $"DELETE FROM servers WHERE server_id = {ServerId};", connection);

Darling/Darling.Tests/PgTargetBetweenWavesV2Tests.cs

@@ -268,10 +268,14 @@ static string Description(string text)
         var darling = Description(RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpTools.cs"));
         var lite = Description(RepoFile.ReadRepoFile("Lite", "Mcp", "McpAnalysisTools.cs"));
         Assert.Equal(darling, lite);
-        Assert.Contains("(SQL Server: CPU %, read latency, connections; PostgreSQL: transactions/sec, session count, deadlocks/hour, CPU as % of capacity)", darling, StringComparison.Ordinal);
-        /* The four named are exactly the four ComparisonBanding maps to a pg_ metric (#3713's seam). */
-        foreach (var key in new[] { PgTargetFactKeys.Tps, PgTargetFactKeys.ConnectionSaturation, PgTargetFactKeys.DeadlockRate, PgTargetFactKeys.CpuPercent })
+        Assert.Contains("(SQL Server: CPU %, read latency, connections; PostgreSQL: transactions/sec, session count, deadlocks/hour, CPU as % of capacity, read latency, replay lag bytes, WAL bytes/sec)", darling, StringComparison.Ordinal);
+        /* The seven named are exactly the seven ComparisonBanding maps to a pg_ metric (#3713's seam, extended by the
+           v2 exit-check residue for the I/O, replication and WAL buckets). */
+        var mapped = new[] { PgTargetFactKeys.Tps, PgTargetFactKeys.ConnectionSaturation, PgTargetFactKeys.DeadlockRate, PgTargetFactKeys.CpuPercent, PgTargetFactKeys.IoReadLatencyMs, PgTargetFactKeys.ReplicationLag, PgTargetFactKeys.WalVolumeShift };
+        foreach (var key in mapped)
             Assert.StartsWith("pg_", ComparisonBanding.BaselinedMetricFor(key)!, StringComparison.Ordinal);
+        Assert.Equal(7, typeof(PgTargetFactKeys).GetFields().Where(f => f.IsLiteral && f.FieldType == typeof(string))
+            .Select(f => (string)f.GetRawConstantValue()!).Count(k => ComparisonBanding.BaselinedMetricFor(k) is not null));
 
         using var doc = System.Text.Json.JsonDocument.Parse(System.Text.Json.JsonSerializer.Serialize(ComparisonBanding.BandRulesPayload));
         var rule = doc.RootElement.GetProperty("baseline").GetString()!;

Darling/Darling.Tests/PgTargetWriteTests.cs

@@ -56,6 +56,8 @@ public sealed class PgTargetWriteTests
     private static readonly int AuroraServerId = ServerIdHelper.GetDeterministicHashCode(AuroraServerName);
     private const string StockServerName = "darling-pg-target-write-stock";
     private static readonly int StockServerId = ServerIdHelper.GetDeterministicHashCode(StockServerName);
+    private const string ZeroWalServerName = "darling-pg-target-write-zero-wal";
+    private static readonly int ZeroWalServerId = ServerIdHelper.GetDeterministicHashCode(ZeroWalServerName);
 
     private const double MiB = 1024.0 * 1024.0;
 
@@ -235,6 +237,42 @@ public void TheGate_HoldsTheFloor_AgainstAQuietServersManySigmas_AndFallsBackToS
         Assert.Equal(1.0, above.FallbackExceedance, precision: 9);
     }
 
+    /* ───────────────────────── one definition of "WAL tracked" (#3691 exit check) ───────────────────────── */
+
+    /// <summary>
+    /// The one predicate both write reads apply. The real shapes: all-NULL <c>wal_bytes</c> (Aurora, pre-14) is
+    /// untracked whatever the sums coalesced to; bytes moving is tracked; records moving alone is tracked (either
+    /// counter moving is the counter being populated). The exit check's fixture shape — 0-not-NULL bytes AND zero
+    /// records over a window in which checkpoints ran — is untracked, where lane 15's checkpoint read said
+    /// <c>wal_tracked 1, wal_bytes 0</c> beside a shift fact saying <c>unavailable</c>.
+    /// </summary>
+    [Theory]
+    [InlineData(false, 0.0, 0L, false)]
+    [InlineData(false, 1e9, 1_000_000L, false)]
+    [InlineData(true, 0.0, 0L, false)]
+    [InlineData(true, 1e9, 0L, true)]
+    [InlineData(true, 0.0, 12L, true)]
+    [InlineData(true, 1e9, 1_000_000L, true)]
+    public void WalIsTracked_IsOneDecision_NullnessAndBothSums(bool anyNonNull, double bytes, long records, bool expected)
+        => Assert.Equal(expected, PgTargetFactCollector.WalIsTracked(anyNonNull, bytes, records));
+
+    /// <summary>The checkpoint read now returns the record sum the predicate needs, APPENDED after <c>sample_count</c> so
+    /// the ten ordinals lane 2 and lane 15 read did not move; the differencing is the WAL read's, verbatim.</summary>
+    [Fact]
+    public void TheCheckpointRead_CarriesTheWalRecordSum_AsItsLastColumn_ForTheSharedPredicate()
+    {
+        var sql = PgTargetFactCollector.PgTargetCheckpointSql;
+        Assert.Contains("wal_records                - LAG(wal_records)                OVER series AS raw_wal_records", sql, StringComparison.Ordinal);
+        Assert.Contains("(wal_bytes IS NOT NULL) AS wal_tracked", sql, StringComparison.Ordinal);
+        var outerSelect = sql[sql.LastIndexOf("SELECT", StringComparison.Ordinal)..sql.LastIndexOf("FROM sampled", StringComparison.Ordinal)];
+        var aliases = System.Text.RegularExpressions.Regex.Matches(outerSelect, @"\s+AS\s+(\w+)\s*(,|$)", System.Text.RegularExpressions.RegexOptions.Multiline)
+            .Select(m => m.Groups[1].Value).ToList();
+        Assert.Equal(12, aliases.Count);
+        Assert.Equal("sample_count", aliases[10]);
+        Assert.Equal("wal_records", aliases[11]);
+        Assert.Contains("CAST(coalesce(SUM(GREATEST(raw_wal_records, 0)), 0) AS bigint)  AS wal_records", outerSelect, StringComparison.Ordinal);
+    }
+
     /* ───────────────────────── the SQL: one differencing, three readers ───────────────────────── */
 
     [Fact]
@@ -467,6 +505,76 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) =
         }
     }
 
+    /// <summary>
+    /// The exit check's fixture shape, live: a stock-stamped server whose <c>wal_bytes</c> is 0-not-NULL (constant,
+    /// non-NULL) and whose <c>wal_records</c> never move, with a checkpoint a minute. Lane 15's two reads disagreed here
+    /// — pressure <c>wal_tracked 1, wal_bytes 0, wal_bytes_per_sec 0</c>, shift <c>unavailable</c>. Through the one
+    /// predicate both facts now say untracked: the pressure fact's WAL figures are ABSENT (not zero), the shift is
+    /// <c>unavailable</c> / <c>reason_wal_stats_not_reported</c>. Unreachable from the real collector (it types the
+    /// columns NULL where <c>pg_stat_wal</c> is not reported) — pinned because a read that can contradict its sibling on
+    /// any input is two definitions. The stock e2e below is the tracked shape's pin (bytes moving ⇒ both tracked).
+    /// </summary>
+    [Fact]
+    public async Task AZeroNotNullWalSeries_ReadsUntracked_OnBothWriteFacts_ThroughTheOnePredicate()
+    {
+        var cs = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+        Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the write-family trackedness e2e.");
+
+        var ct = TestContext.Current.CancellationToken;
+        using var connection = new NpgsqlConnection(cs);
+        await connection.OpenAsync(ct);
+        await PgMigrations.MigrateAsync(connection, ct);
+        await DeleteRowsAsync(connection, ct);
+
+        await using var postgres = NpgsqlDataSource.Create(cs!);
+
+        var bodySucceeded = false;
+        try
+        {
+            await PgTargetFactCollectorTests.RegisterServerAsync(connection, ZeroWalServerId, ZeroWalServerName, MonitoredEngineKind.Postgres, 18, ct);
+
+            var windowEnd = TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1);
+            var windowStart = windowEnd.AddHours(-4);
+
+            /* The coverage witness over the window (the collector stamps observed time from pg_database_stats FIRST and
+               the write read returns on zero observed time), then the write series: one timed checkpoint a minute; a
+               zero per-minute WAL increment plants wal_bytes CONSTANT and non-NULL and wal_records at 0 on every row —
+               the 0-not-NULL shape. */
+            for (var minute = 0; minute <= 4 * 60 + 1; minute++)
+                await PgTargetFactCollectorTests.PlantDatabaseStatsAsync(connection, ZeroWalServerId, ZeroWalServerName, windowStart.AddMinutes(minute - 1), ct);
+            await PlantWriteSeriesAsync(connection, ZeroWalServerId, ZeroWalServerName, windowStart.AddMinutes(-1), minutes: 4 * 60 + 1,
+                requestedPerMinute: 0, timedPerMinute: 1, walBytesPerMinuteBase: 0L, walBytesPerMinuteSpike: 0L, spikeFromMinute: int.MaxValue, ct);
+
+            var facts = await new PgTargetFactCollector(postgres).CollectFactsAsync(new AnalysisContext
+            {
+                ServerId = ZeroWalServerId, ServerName = ZeroWalServerName, TimeRangeStart = windowStart, TimeRangeEnd = windowEnd, ServerUtcOffset = TimeSpan.Zero,
+                Coverage = new WindowCoverage { NominalMs = 4 * 3_600_000, ObservedMs = 4 * 3_600_000, SampleCount = 240 },
+            });
+
+            var pressure = Assert.Single(facts, f => f.Key == PgTargetFactKeys.CheckpointPressure);
+            Assert.Equal(240, pressure.Metadata["checkpoints_timed"]);
+            Assert.Equal(0, pressure.Metadata["wal_tracked"]);
+            Assert.False(pressure.Metadata.ContainsKey("wal_bytes"));
+            Assert.False(pressure.Metadata.ContainsKey("wal_bytes_per_sec"));
+            Assert.False(pressure.Metadata.ContainsKey("wal_bytes_per_checkpoint"));
+            Assert.False(pressure.Metadata.ContainsKey("not_applicable"));   // stock: graded, just without WAL figures
+
+            var shift = Assert.Single(facts, f => f.Key == PgTargetFactKeys.WalVolumeShift);
+            Assert.Equal(0, shift.Value);
+            Assert.Equal(0, shift.Metadata["wal_tracked"]);
+            Assert.Equal(1, shift.Metadata["unavailable"]);
+            Assert.Equal(1, shift.Metadata["reason_wal_stats_not_reported"]);
+            Assert.False(shift.Metadata.ContainsKey("peak_wal_bytes_per_sec"));  // and so ComparisonBanding withholds it
+
+            bodySucceeded = true;
+        }
+        finally
+        {
+            await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) =>
+                await DeleteRowsAsync(cleanup, cleanupCt));
+        }
+    }
+
     /// <summary>
     /// Stock-stamped, 31 days of one-minute <c>pg_write_stats</c> with WAL at 1 MiB/s (a deterministic ±5 % ripple)
     /// and, for the four-hour window, 12 MiB/s; requested-dominant checkpoints throughout so the pressure fact fires.
@@ -797,12 +905,12 @@ CASE WHEN wal_inc IS NULL THEN NULL ELSE (SUM(wal_inc) OVER (ORDER BY n)) / 8192
     private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct)
     {
         using var cleanup = new NpgsqlCommand(
-            $"DELETE FROM pg_database_stats WHERE server_id IN ({AuroraServerId}, {StockServerId}); " +
-            $"DELETE FROM pg_write_stats WHERE server_id IN ({AuroraServerId}, {StockServerId}); " +
-            $"DELETE FROM pg_server_config WHERE server_id IN ({AuroraServerId}, {StockServerId}); " +
-            $"DELETE FROM analysis_findings WHERE server_id IN ({AuroraServerId}, {StockServerId}); " +
-            $"DELETE FROM analysis_muted WHERE server_id IN ({AuroraServerId}, {StockServerId}); " +
-            $"DELETE FROM servers WHERE server_id IN ({AuroraServerId}, {StockServerId});", connection);
+            $"DELETE FROM pg_database_stats WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId}); " +
+            $"DELETE FROM pg_write_stats WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId}); " +
+            $"DELETE FROM pg_server_config WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId}); " +
+            $"DELETE FROM analysis_findings WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId}); " +
+            $"DELETE FROM analysis_muted WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId}); " +
+            $"DELETE FROM servers WHERE server_id IN ({AuroraServerId}, {StockServerId}, {ZeroWalServerId});", connection);
         await cleanup.ExecuteNonQueryAsync(ct);
     }
 }