Darling: the raw procedure duration trend reads per-collection totals, written with each procedure_stats batch (#5638)

Darling/Darling.Tests/AwsPerServerRoleRungTests.cs

@@ -107,15 +107,16 @@ public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 4})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal);
-        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal);
+        Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal);
+        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal);
 
         Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
 
         var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!;
         var arity = method.GetParameters().Length;
 
         /* A newer rung's sentinel (V175's, #5631) is the last argument. */
-        Assert.Equal(ProbeOrdinal + 6, arity - 1);
+        Assert.Equal(ProbeOrdinal + 7, arity - 1);
 
         var all = Enumerable.Repeat((object)true, arity).ToArray();
         Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);

Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs

@@ -318,6 +318,13 @@ the group index still present — the same fixture shape CaggGroupIndexDropUpgra
     /// A future rung that adds a new probe sentinel must add its artifact's removal here.</summary>
     private static async Task DropArtifactsOfLaterRungsAsync(NpgsqlConnection connection, int simulatedVersion, CancellationToken ct)
     {
+        if (simulatedVersion < ProcedureStatsTotals.RungVersion)
+        {
+            /* V176 (#5638) - the per-collection procedure totals table is the probe's sentinel. */
+            await using var dropTotals = new NpgsqlCommand("DROP TABLE IF EXISTS collect.procedure_stats_totals", connection);
+            await dropTotals.ExecuteNonQueryAsync(ct);
+        }
+
         if (simulatedVersion < 175)
         {
             /* V175 (#5631) - the rows-per-call column of the daily plan regression totals is the probe's sentinel. */

Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs

@@ -361,12 +361,14 @@ public void ProcedureDurationTrendSql_PrefersTheStoredInterval_NeverFabricatesZe
         Assert.Contains("CASE WHEN is_stored AND max_interval_seconds = 0 THEN NULL", sql, StringComparison.Ordinal);
         Assert.Contains($"CASE WHEN gap_seconds <= {CollectorDeltaCalculator.DefaultMaxGapSeconds} THEN gap_seconds END", sql, StringComparison.Ordinal);
         Assert.Contains("WHEN is_stored THEN NULLIF(max_interval_seconds, 0)", sql, StringComparison.Ordinal);
-        Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(sql, @"collection_time >= \$2 - INTERVAL '3600 seconds'").Count);
+        /* #5638: the stored collections come from two arms (the rows below the server's totals floor, the totals from it), each bounded
+           from $2 less the gap, and the runs are the third. */
+        Assert.Equal(3, System.Text.RegularExpressions.Regex.Matches(sql, @"collection_time >= \$2 - INTERVAL '3600 seconds'").Count);
         Assert.Contains("FROM gapped\nWHERE collection_time >= $2", string.Join('\n', Lines(sql)), StringComparison.Ordinal);
         /* A run is logged after it finishes, so one that began inside the window can be logged just past its end: the runs are read
-           an hour past $3 (once), the stored rows and the final points stop at $3 (twice). */
+           an hour past $3 (once), the stored rows, the stored totals and the final points stop at $3 (three times, #5638). */
         Assert.Single(System.Text.RegularExpressions.Regex.Matches(sql, @"collection_time <= \$3 \+ INTERVAL '3600 seconds'"));
-        Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(sql, @"collection_time <= \$3\r?\n").Count);
+        Assert.Equal(3, System.Text.RegularExpressions.Regex.Matches(sql, @"collection_time <= \$3\r?\n").Count);
         Assert.DoesNotContain("COALESCE(MAX(", sql, StringComparison.Ordinal);
         Assert.DoesNotContain("series_first", sql, StringComparison.Ordinal);
         Assert.Contains("CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds END AS elapsed_ms_per_second", sql, StringComparison.Ordinal);
@@ -379,13 +381,15 @@ public void ProcedureDurationTrendSql_PrefersTheStoredInterval_NeverFabricatesZe
            carries NO database term, so the interval is the collection's), is the same text. */
         static bool IsFilteredSumLine(string l) =>
             l.Contains("delta_elapsed_time", StringComparison.Ordinal) || l.Contains("delta_execution_count", StringComparison.Ordinal)
+            || l.Contains("total_elapsed_time", StringComparison.Ordinal) || l.Contains("total_execution_count", StringComparison.Ordinal)
             || l.Contains("matched_rows", StringComparison.Ordinal);
         foreach (var cte in new[] { "stored AS\n(", "runs AS\n(", "events AS\n(", "sequenced AS\n(", "points AS\n(", "gapped AS\n(", "raw AS\n(" })
         {
             var viewerCte = Cte(Lf(ViewerDataService.ProcedureDurationTrendSql), cte);
             if (cte == "stored AS\n(")
             {
-                Assert.Equal(3, viewerCte.Split('\n').Count(l => l.Contains("FILTER (WHERE $4::text[] IS NULL OR database_name = ANY($4))", StringComparison.Ordinal)));
+                /* #5638: three filtered sums on the rows arm and three on the totals arm. */
+                Assert.Equal(6, viewerCte.Split('\n').Count(l => l.Contains("FILTER (WHERE $4::text[] IS NULL OR database_name = ANY($4))", StringComparison.Ordinal)));
             }
             var viewer = string.Join('\n', viewerCte.Split('\n').Where(l => !IsFilteredSumLine(l)).Select(l => l.Trim()));
             var mcp = string.Join('\n', Cte(Lf(sql), cte).Split('\n').Where(l => !IsFilteredSumLine(l)).Select(l => l.Trim()));

Darling/Darling.Tests/DurationTrendDatabaseFilterLiveTests.cs

@@ -362,7 +362,9 @@ every database. */
 
         /* Raw: the per-collection CTE, the filter at $4 and the bucket width at $5. */
         Assert.Equal(3, Count(DarlingTrendReader.QueryDurationTrendFilteredSql, filterPredicate4));
-        Assert.Equal(3, Count(DarlingTrendReader.ProcedureDurationTrendFilteredSql, filterPredicate4));
+        /* #5638: the procedure statement's stored CTE has two arms, the rows below the totals floor and the per-database totals from
+           it, and each carries the three FILTERs (elapsed, executions, matched rows), so six. */
+        Assert.Equal(6, Count(DarlingTrendReader.ProcedureDurationTrendFilteredSql, filterPredicate4));
         Assert.Equal(0, Count(DarlingTrendReader.QueryDurationTrendFilteredSql, predicate4));
         Assert.Equal(0, Count(DarlingTrendReader.ProcedureDurationTrendFilteredSql, predicate4));
         /* #5414 round 2: empty is decided once for the whole window, never per bucket (a per-bucket HAVING dropped the

Darling/Darling.Tests/LegacyPinCandidateRungTests.cs

@@ -138,11 +138,12 @@ public void TheProbeCarriesTheTable_AtItsOwnOrdinal()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal);
-        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal);
+        Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal);
+        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal);
 
         var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!;
         var parameters = method.GetParameters();
-        Assert.Equal(ProbeOrdinal + 8, parameters.Length - 1);
+        Assert.Equal(ProbeOrdinal + 9, parameters.Length - 1);
         Assert.Equal("hasLegacyPinCandidates", parameters[ProbeOrdinal].Name);
         Assert.Equal("hasPagerDutyAutoResolve", parameters[ProbeOrdinal - 1].Name);
 

Darling/Darling.Tests/PlanRegressionDailyRungTests.cs

@@ -336,15 +336,16 @@ public void TheProbeCarriesTheBuiltTableAtItsOwnOrdinal_AndMapsItsRung()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal);
-        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal);
+        Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal);
+        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal);
 
         Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
 
         var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!;
         var parameters = method.GetParameters();
 
         /* A newer rung's sentinel (V175's, #5631) is the last argument. */
-        Assert.Equal(ProbeOrdinal + 7, parameters.Length - 1);
+        Assert.Equal(ProbeOrdinal + 8, parameters.Length - 1);
         Assert.Equal("hasPlanRegressionDaily", parameters[ProbeOrdinal].Name);
 
         /* Every sentinel true is a fully-migrated store, which maps to exactly this build's version. */

Darling/Darling.Tests/PlanRegressionRowcountRungTests.cs

@@ -120,21 +120,26 @@ public void TheProbeNamesTheColumn_AndMapsAFullyMigratedStoreToThisTopRung()
 
         var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs");
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
-        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
+        Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
+        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 2})", viewer, StringComparison.Ordinal);
 
         var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!;
         var parameters = method.GetParameters();
         Assert.Equal("hasPlanRegressionDailyRowcount", parameters[ProbeOrdinal].Name);
         Assert.Equal("hasServiceVersion", parameters[ProbeOrdinal - 1].Name);
-        Assert.Equal(ProbeOrdinal, parameters.Length - 1);
+        Assert.Equal(ProbeOrdinal + 1, parameters.Length - 1);
 
         /* Every sentinel present maps to exactly this build's version; this sentinel absent maps one rung back. */
         var all = Enumerable.Repeat((object)true, parameters.Length).ToArray();
         Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);
-        Assert.Equal(RungVersion, (int)method.Invoke(null, all)!);
+        /* V176 (#5638) sits above this rung: with only this sentinel and those below it the store maps to this rung. */
+        var topAbsent = (object[])all.Clone();
+        topAbsent[ProbeOrdinal + 1] = false;
+        Assert.Equal(RungVersion, (int)method.Invoke(null, topAbsent)!);
 
         var behind = (object[])all.Clone();
         behind[ProbeOrdinal] = false;
+        behind[ProbeOrdinal + 1] = false;
         Assert.Equal(RungVersion - 1, (int)method.Invoke(null, behind)!);
 
         /* In the source the arm sits ABOVE V174's and returns this rung's version. */