Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 226 additions & 0 deletions Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs

Large diffs are not rendered by default.

300 changes: 300 additions & 0 deletions Darling/Darling.Tests/DarlingPerformanceTrendsReadTests.cs

Large diffs are not rendered by default.

80 changes: 69 additions & 11 deletions Darling/Darling.Tests/QueryStoreTrendRoutingLiveTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ public sealed class QueryStoreTrendRoutingLiveTests

private const string ServerName = "qs-trend-routing-e2e";

/// <summary>The second server (#3541 A2): sampled only in the unmaterialized tail. Distinctive fake id
/// for the same reason as <see cref="TestServerId"/>.</summary>
private const int TailOnlyServerId = -927361;

private const string TailOnlyServerName = "qs-trend-routing-tail-only";

[Fact]
public async Task DurationTrend_ServesTheRollupBelowTheBoundary_AndRanksOnlyTheTail()
{
Expand Down Expand Up @@ -91,6 +97,14 @@ await SeedSnapshotsAsync(connection, intervalId: 3102, queryId: 62, intervalStar
await SeedSnapshotsAsync(connection, intervalId: 3103, queryId: 63, intervalStart: hour12,
avgDurationUs: 300, [(hour12.AddMinutes(5), 3L), (hour12.AddMinutes(20), 9L)], ct);

/* ── a SECOND server with one interval in the tail only (#3541 A2): sampled, so a window over the
rollup region that finds nothing for it is a QUIET stretch — and when that window's head sits
below the rollup's floor, the empty answer has to say which part is quiet and which part is
unserved rather than advising a wider window for both. ── */
await DarlingMcpTestData.RegisterServerAsync(connection, TailOnlyServerId, TailOnlyServerName, ct);
await SeedSnapshotsAsync(connection, intervalId: 3104, queryId: 64, intervalStart: hour12,
avgDurationUs: 500, [(hour12.AddMinutes(10), 4L)], ct, serverId: TailOnlyServerId, serverName: TailOnlyServerName);

await EnsureAggregatesWithoutRefreshPoliciesAsync(connection, ct);

/* Materialize ONLY the 10:00 and 11:00 buckets — the 12:00 hour stays raw, the exact state a live
Expand Down Expand Up @@ -149,18 +163,29 @@ interval START (10:00 — so that hour reads 21+40=61) and the 11:00 point carri

Assert.Equal(3, payload.GetProperty("trend").GetArrayLength());

var source = payload.GetProperty("source");
Assert.Equal("rollup+raw", source.GetProperty("tier").GetString());
Assert.Equal(TimescaleSupport.QueryStoreStatsCorrectedHourlyView, source.GetProperty("rollup").GetString());
Assert.StartsWith("2026-03-04T12:00:00", source.GetProperty("raw_from").GetString()!, StringComparison.Ordinal);
/* #3541 A2: the disclosure speaks the vocabulary get_query_trend and the two plan-cache siblings
share — `source` is the tier WORD, and the #2736 seam detail lives under `routing`. Same
assertions as before the move, on the moved keys. */
Assert.Equal("rollup+raw", payload.GetProperty("source").GetString());
var routing = payload.GetProperty("routing");
Assert.Equal(TimescaleSupport.QueryStoreStatsCorrectedHourlyView, routing.GetProperty("rollup").GetString());
Assert.StartsWith("2026-03-04T12:00:00", routing.GetProperty("raw_from").GetString()!, StringComparison.Ordinal);
Assert.Contains("routing.raw_from", payload.GetProperty("aggregate_note").GetString()!, StringComparison.Ordinal);
Assert.Contains("1 hour before routing.raw_from", payload.GetProperty("bucket").GetString()!, StringComparison.Ordinal);

/* The window starts at 07:00, below the rollup's 10:00 floor — a degraded answer must label
itself: points before the floor are missing, not zero, and the remedy is named. */
Assert.StartsWith("2026-03-04T10:00:00", source.GetProperty("unserved_before").GetString()!, StringComparison.Ordinal);
Assert.Contains("--backfill-rollups", source.GetProperty("unserved_note").GetString()!, StringComparison.Ordinal);
itself: points before the floor are missing, not zero, and the remedy is named. The shared
coverage words say the same thing in the shared shape: the series begins at 10:00, three hours
after what was asked for, so it is truncated. */
Assert.StartsWith("2026-03-04T10:00:00", routing.GetProperty("unserved_before").GetString()!, StringComparison.Ordinal);
Assert.Contains("--backfill-rollups", routing.GetProperty("unserved_note").GetString()!, StringComparison.Ordinal);
Assert.StartsWith("2026-03-04T10:00:00", payload.GetProperty("effective_start").GetString()!, StringComparison.Ordinal);
Assert.Equal(3.0, payload.GetProperty("effective_hours_back").GetDouble());
Assert.True(payload.GetProperty("truncated").GetBoolean());

/* ── a window ENTIRELY below the floor is a coverage gap, not a quiet server: the empty answer
names the mechanism and the remedy instead of advising a wider window. ── */
names the mechanism and the remedy instead of advising a wider window — and carries the same
disclosure block the data envelope does, with the served span honestly zero. ── */
var belowFloor = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend(
postgres, ServerName, hours_back: 1, as_of: "2026-03-04T09:30:00Z")).RootElement;

Expand All @@ -169,6 +194,38 @@ names the mechanism and the remedy instead of advising a wider window. ── */
Assert.Contains("--backfill-rollups", message, StringComparison.Ordinal);
Assert.Contains("2026-03-04T10:00:00", message, StringComparison.Ordinal);
Assert.DoesNotContain("widen", message, StringComparison.OrdinalIgnoreCase);
Assert.Equal("rollup+raw", belowFloor.GetProperty("source").GetString());
Assert.StartsWith("2026-03-04T09:30:00", belowFloor.GetProperty("effective_start").GetString()!, StringComparison.Ordinal);
Assert.Equal(0.0, belowFloor.GetProperty("effective_hours_back").GetDouble());
Assert.True(belowFloor.GetProperty("truncated").GetBoolean());

/* ── #3541 A2: the tail-only server over a window whose head is below the floor and whose covered
part holds nothing for it. Sampled, so not "unavailable"; nothing in [10:00, 11:00), so
"empty" — but the message must split the window: quiet from the floor on, UNSERVED before it,
and "widen" only for what widening can do. ── */
var partialHead = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend(
postgres, TailOnlyServerName, hours_back: 2, as_of: "2026-03-04T11:00:00Z")).RootElement;

Assert.Equal("empty", partialHead.GetProperty("status").GetString());
var partialMessage = partialHead.GetProperty("message").GetString()!;
Assert.Contains("2026-03-04T10:00:00", partialMessage, StringComparison.Ordinal);
Assert.Contains("unserved rather than quiet", partialMessage, StringComparison.Ordinal);
Assert.Contains("--backfill-rollups", partialMessage, StringComparison.Ordinal);
Assert.Contains("cannot reach the unserved head", partialMessage, StringComparison.Ordinal);
Assert.DoesNotContain("EVER", partialMessage, StringComparison.Ordinal);

/* And the same server over its tail alone: one raw-arm point at 12:00, exactly where the window
starts, so nothing is truncated and the served span is the whole hour. */
var tailOnly = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend(
postgres, TailOnlyServerName, hours_back: 1, as_of: "2026-03-04T13:00:00Z")).RootElement;

Assert.Equal(1, tailOnly.GetProperty("trend").GetArrayLength());
Assert.Equal("rollup+raw", tailOnly.GetProperty("source").GetString());
Assert.StartsWith("2026-03-04T12:00:00", tailOnly.GetProperty("effective_start").GetString()!, StringComparison.Ordinal);
Assert.False(tailOnly.GetProperty("truncated").GetBoolean());
Assert.Equal(
tailOnly.GetProperty("trend")[0].GetProperty("value").GetDouble(),
tailOnly.GetProperty("trend")[0].GetProperty("elapsed_ms_per_second").GetDouble());
}

/// <summary>
Expand All @@ -178,7 +235,8 @@ names the mechanism and the remedy instead of advising a wider window. ── */
/// </summary>
private static async Task SeedSnapshotsAsync(
NpgsqlConnection connection, long intervalId, long queryId, DateTime intervalStart,
long avgDurationUs, (DateTime When, long Count)[] snapshots, CancellationToken ct)
long avgDurationUs, (DateTime When, long Count)[] snapshots, CancellationToken ct,
int serverId = TestServerId, string serverName = ServerName)
{
const string sql = @"
INSERT INTO collect.query_store_stats
Expand All @@ -195,8 +253,8 @@ INSERT INTO collect.query_store_stats
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue(when);
command.Parameters.AddWithValue(intervalId);
command.Parameters.AddWithValue(TestServerId);
command.Parameters.AddWithValue(ServerName);
command.Parameters.AddWithValue(serverId);
command.Parameters.AddWithValue(serverName);
command.Parameters.AddWithValue(queryId);
command.Parameters.AddWithValue(intervalStart);
command.Parameters.AddWithValue(count);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,9 @@ public static string Build(DarlingPeerDirectory.Snapshot peers)
| `get_perfmon_trend` | A single performance counter's value + delta over time (summed across instances) | `counter_name` (required), `server_name`, `hours_back` (default 24), `as_of` |
| `get_file_io_trend` | Per-database file I/O read/write latency over time (top-10 busiest files). An empty result distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) | `server_name`, `hours_back` (default 24), `as_of` |
| `get_query_trend` | One query's per-collection history (deltas, avg cpu/elapsed, DOP) by query_hash | `query_hash` (required), `database_name` (required), `server_name`, `hours_back` (default 24), `as_of` |
| `get_query_duration_trend` | Overall query elapsed-ms/sec + executions/sec across all queries over time, from the PLAN CACHE. Each point carries `value` (ms/sec), `execution_count` and `executions_per_second` — the last two are the same quantity, and `execution_count` is truncated to an integer, so read `executions_per_second` on a quiet server where the rate is below 1. An empty result distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) | `server_name`, `hours_back` (default 24), `as_of` |
| `get_procedure_duration_trend` | The same series over `procedure_stats`. NOT a duplicate of the above: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs — this charges the whole call to the procedure. Read the two together to tell an ad-hoc regression from a procedure regression | `server_name`, `hours_back` (default 24), `as_of` |
| `get_query_store_duration_trend` | The same series over Query Store. The plan-cache trends lose everything an eviction or a restart takes with them; Query Store persists per interval, so this is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work RAN. Its `unavailable` names the cause the other two do not have: Query Store may simply be OFF on every database | `server_name`, `hours_back` (default 24), `as_of` |
| `get_query_duration_trend` | Overall query elapsed-ms/sec + executions/sec across all queries over time, from the PLAN CACHE. Each point carries `value` (ms/sec), `execution_count` and `executions_per_second` — the last two are the same quantity, and `execution_count` is truncated to an integer, so read `executions_per_second` on a quiet server where the rate is below 1. Each point also carries `elapsed_ms_per_second`, the same quantity as `value` with its unit in the name — read that one. Raw `query_stats` rows are dropped at 4 days on a TimescaleDB store, so a window reaching past that is served from the hourly rollup: `source` says which tier answered (`raw` or `hourly`), `bucket` its grain (`per-collection` or `1 hour`), `effective_start` / `effective_hours_back` where the served series actually begins, `truncated` whether that head sits later than asked, and `aggregate_note` what the hourly tier trades (each point is the hour's work over 3,600 s, so a partly-collected hour reads low; the rollup trails the clock by up to two hours). An empty result carries the same block and distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) — and from a window whose head the tier does not hold (`empty`, but the message says the rows were dropped or not yet materialized and that widening cannot help; `--backfill-rollups` is the remedy) | `server_name`, `hours_back` (default 24), `as_of` |
| `get_procedure_duration_trend` | The same series over `procedure_stats`. NOT a duplicate of the above: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs — this charges the whole call to the procedure. Read the two together to tell an ad-hoc regression from a procedure regression. Same tier routing and the same `source` / `effective_start` / `truncated` / `bucket` / `aggregate_note` block as `get_query_duration_trend`, over `procedure_stats` / `procedure_stats_hourly` | `server_name`, `hours_back` (default 24), `as_of` |
| `get_query_store_duration_trend` | The same series over Query Store. The plan-cache trends lose everything an eviction or a restart takes with them; Query Store persists per interval, so this is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work RAN. Its `unavailable` names the cause the other two do not have: Query Store may simply be OFF on every database. Carries the same `source` / `effective_start` / `truncated` / `bucket` / `aggregate_note` block as its siblings; where the store has the corrected Query Store rollup, `source` is `rollup+raw` and a `routing` block names the rollup, `raw_from` (the seam: 1-hour buckets before it, per-interval points from it) and, when the window reaches below what the rollup has materialized, `unserved_before` — points before that instant are missing, not zero | `server_name`, `hours_back` (default 24), `as_of` |

### System-health parse-on-read tools

Expand Down
Loading
Loading