Extreme, corroborated anomaly findings can now cross the notify floor, so the baseline engine is no longer notification-inert at shipped settings (Fixes #3526)

Lite.Tests/FactScorerTests.cs

@@ -457,6 +457,331 @@ public void Score_CxPacketWithSignificantResourceSemaphore_CapReleased()
         Assert.True(cx.Severity >= 1.5, "a significant RESOURCE_SEMAPHORE releases the cap into CRITICAL");
     }
 
+    /* ── Layer 3: #3526 anomaly extremity escape + the anomaly co-fire amplifier arm ── */
+
+    // Before #3526 every ANOMALY_* fact was capped at 1.49 by the tuning-class cap, had NO amplifier arm
+    // (so its final severity was its <= 1.0 base), and the shipped notify floor is 1.5: the baseline
+    // engine could never page. These tests pin the two halves of the fix — an anomaly whose deviation is
+    // EXTREME (>= 3x the cutoff it fired at) leaves the cap, and the new co-fire arm can carry an
+    // extreme, CORROBORATED anomaly past 1.5 — and, just as deliberately, everything the fix must NOT do.
+
+    // A deviation-scored anomaly on a TRUSTWORTHY baseline (baseline_low_quality = 0). fireThreshold 0
+    // means "omit the key" — a pre-#1743 fact, which the scorer anchors at the classical 2.0.
+    private static Fact TrustedAnomaly(string key, double deviationSigma, double fireThreshold = 0.0)
+    {
+        var metadata = new Dictionary<string, double>
+        {
+            ["deviation_sigma"] = deviationSigma,
+            ["baseline_low_quality"] = 0.0,
+            ["fallback_exceedance"] = 0.0,
+            ["confidence"] = 1.0
+        };
+        if (fireThreshold > 0) metadata["fire_threshold"] = fireThreshold;
+        return new Fact { Source = "anomaly", Key = key, Value = 1, Metadata = metadata };
+    }
+
+    // (a) The issue's own 3am shape: a 20σ session spike (robust 3.5 cutoff → the escape opens at 10.5σ)
+    // beside a 15σ batch-request anomaly and SQL CPU at 85%. Before the fix the best it could do was 1.0.
+    // Now: base 1.0 (saturated), extreme → cap released, two co-fires (+0.3 batch, +0.3 CPU >= 80) → 1.6.
+    // The batch anomaly is extreme too (15 >= 10.5) and is corroborated symmetrically (+0.3 sessions,
+    // +0.3 CPU) → 1.6 as well. Both clear the shipped 1.5 notify floor for the first time.
+    [Fact]
+    public void Score_ExtremeCorroboratedAnomaly_CrossesNotifyFloor()
+    {
+        var facts = new List<Fact>
+        {
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5),
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5),
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 },
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE");
+        Assert.Equal(1.0, sessions.BaseSeverity, precision: 4);
+        Assert.Equal(1.6, sessions.Severity, precision: 4);
+        Assert.True(sessions.Severity >= 1.5, "an extreme, twice-corroborated anomaly must reach the CRITICAL band");
+        Assert.Contains(sessions.AmplifierResults, a => a.Matched && a.Description.StartsWith("Batch-request anomaly co-fired"));
+        Assert.Contains(sessions.AmplifierResults, a => a.Matched && a.Description.StartsWith("SQL Server CPU >= 80%"));
+
+        var batch = facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS");
+        Assert.Equal(1.6, batch.Severity, precision: 4);
+    }
+
+    // (b) The SAME fact set with the session spike at exactly its fire threshold: base 0.5, the two
+    // co-fires lift it to 0.8, and it stays far inside WARNING. Corroboration alone cannot page a
+    // fact that only just fired.
+    [Fact]
+    public void Score_AnomalyAtFireThreshold_WithCoFires_StaysWarning()
+    {
+        var facts = new List<Fact>
+        {
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 3.5, 3.5),      // exactly at the cutoff → base 0.5
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5),
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 },
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE");
+        Assert.Equal(0.5, sessions.BaseSeverity, precision: 4);
+        Assert.Equal(0.8, sessions.Severity, precision: 4);   // 0.5 x (1 + 0.3 + 0.3)
+        Assert.True(sessions.Severity <= 1.49, "an anomaly at its fire threshold must stay in WARNING");
+    }
+
+    // The routine-but-saturated case the cap exists for: 7σ on the 3.5 cutoff is 2x the anchor — the
+    // ramp's saturation point (base 1.0) but NOT extreme (the escape needs 10.5σ). Three co-fires push
+    // the Layer-2 product to 1.9, and the cap holds it at 1.49: a busy evening does not page.
+    [Fact]
+    public void Score_SaturatedButRoutineAnomaly_WithThreeCoFires_CappedAtWarningCeiling()
+    {
+        var facts = new List<Fact>
+        {
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 7.0, 3.5),      // 2x anchor → base 1.0, not extreme
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 7.0, 3.5),     // +0.3
+            TrustedAnomaly("ANOMALY_CPU_SPIKE", 7.0, 3.5),          // +0.3
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE");
+        Assert.Equal(1.0, sessions.BaseSeverity, precision: 4);
+        // 1.0 x (1 + 0.3 + 0.3 + 0.3) = 1.9 pre-cap; the anomaly is not extreme, so Layer 3 caps it.
+        Assert.Equal(1.49, sessions.Severity, precision: 4);
+        Assert.True(sessions.Severity < 1.5, "saturation is not extremity — a routine 2x-anchor anomaly must stay in WARNING");
+    }
+
+    // The escape bar is 3x the cutoff the fact FIRED at, on every path the detectors take: classical 2.0
+    // (a pre-#1743 fact with no fire_threshold) → 6σ; robust 3.5 → 10.5σ; heavy-tail 5.0 → 15σ. Each
+    // row plants the root beside the same two co-fires (+0.3 +0.3 → 1.6 when released, 1.49 when
+    // capped) and probes one side of the bar. The root's base is 1.0 in every row (all are >= 2x anchor).
+    // The last two rows pin the display-cap bound (review-caught on this PR): AnomalyGate clamps the
+    // stored deviation_sigma at SigmaDisplayCap (25σ), so an operator-scaled anchor of 10.0 (3x = 30σ,
+    // unreachable) must open at 25σ instead of going silently dead — and a hair under 25 stays capped.
+    [Theory]
+    [InlineData(6.0, 0.0, 1.6)]      // classical: 3 x 2.0 → released
+    [InlineData(5.99, 0.0, 1.49)]    // classical: a hair under → capped
+    [InlineData(10.5, 3.5, 1.6)]     // robust: 3 x 3.5 → released
+    [InlineData(10.49, 3.5, 1.49)]   // robust: a hair under → capped
+    [InlineData(15.0, 5.0, 1.6)]     // heavy-tail: 3 x 5.0 → released
+    [InlineData(14.99, 5.0, 1.49)]   // heavy-tail: a hair under → capped
+    [InlineData(25.0, 10.0, 1.6)]    // hard-tuned anchor: min(30, 25) = the display cap → released
+    [InlineData(24.99, 10.0, 1.49)]  // hard-tuned anchor: a hair under the cap → capped
+    public void Score_AnomalyExtremityEscape_OpensAtThreeTimesTheFireThreshold(
+        double deviationSigma, double fireThreshold, double expected)
+    {
+        var facts = new List<Fact>
+        {
+            TrustedAnomaly("ANOMALY_QUERY_DURATION", deviationSigma, fireThreshold),
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5),           // +0.3
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var root = facts.First(f => f.Key == "ANOMALY_QUERY_DURATION");
+        Assert.Equal(1.0, root.BaseSeverity, precision: 4);
+        Assert.Equal(expected, root.Severity, precision: 4);
+    }
+
+    // (c) The never-blind fallback path: on an untrustworthy baseline the stored deviation_sigma is a
+    // meaningless z the detector refused to trust, so the escape must NOT read it — a planted 30σ (far
+    // past any sigma bar) must not release the cap. The fallback path escapes only when the ABSOLUTE
+    // exceedance is equally extreme: 3x the fallback bar (for batch requests, 15,000/s against the
+    // 5,000/s bar). 2.5x saturates the base at 1.0 but stays capped; 3.0x is released.
+    [Theory]
+    [InlineData(2.5, 1.49)]   // past saturation (2x), under the 3x escape bar → capped
+    [InlineData(2.99, 1.49)]  // a hair under → capped
+    [InlineData(3.0, 1.6)]    // 3x the absolute bar → released, corroborated → pages
+    public void Score_LowQualityAnomaly_EscapesOnlyAtThreeTimesItsAbsoluteBar_NeverOnSigma(
+        double exceedance, double expected)
+    {
+        var facts = new List<Fact>
+        {
+            new()
+            {
+                Source = "anomaly",
+                Key = "ANOMALY_BATCH_REQUESTS",
+                Value = 1,
+                Metadata = new()
+                {
+                    ["deviation_sigma"] = 30.0,          // would be extreme on the trusted path; must be ignored here
+                    ["fire_threshold"] = 3.5,
+                    ["baseline_low_quality"] = 1.0,
+                    ["fallback_exceedance"] = exceedance,
+                    ["confidence"] = 1.0
+                }
+            },
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 15.0, 3.5),           // +0.3
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var batch = facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS");
+        Assert.Equal(1.0, batch.BaseSeverity, precision: 4);   // the fallback ramp saturates at 2x the bar
+        Assert.Equal(expected, batch.Severity, precision: 4);
+    }
+
+    // (d) The cap's original target is untouched: CXPACKET with its non-impact amplifiers (1.7 pre-cap)
+    // stays at 1.49 even when two EXTREME anomalies co-fire beside it — anomalies are not impact peers,
+    // and the extremity escape is per-fact, for ANOMALY_* only. In the same fact set the anomalies
+    // themselves ARE released (each is extreme and corroborated by the other + CPU): the two escapes
+    // are independent.
+    [Fact]
+    public void Score_CxPacketWithExtremeAnomalyCoFires_StillCapped_AnomaliesReleased()
+    {
+        var facts = new List<Fact>
+        {
+            new() { Source = "waits", Key = "CXPACKET", Value = 0.80 },        // base 1.0
+            new() { Source = "config", Key = "CONFIG_CTFP", Value = 5 },       // +0.3
+            new() { Source = "config", Key = "CONFIG_MAXDOP", Value = 0 },     // +0.2
+            new() { Source = "queries", Key = "QUERY_HIGH_DOP", Value = 10 },  // +0.2
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5),
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5),
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 },
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var cx = facts.First(f => f.Key == "CXPACKET");
+        Assert.Equal(1.49, cx.Severity, precision: 4);
+        Assert.True(cx.Severity < 1.5, "extreme anomalies are not impact peers — CXPACKET must stay capped");
+
+        Assert.Equal(1.6, facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE").Severity, precision: 4);
+        Assert.Equal(1.6, facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS").Severity, precision: 4);
+    }
+
+    // (e) A LONE extreme anomaly does not page. 25σ is the detectors' display cap — the most extreme
+    // reading a fact can carry — and with nothing else moving it scores exactly its 1.0 base: the cap
+    // is released but there is nothing for the release to preserve. The CRITICAL band is earned only
+    // with corroboration (the rule every base fact follows), and a solitary extreme reading is the shape
+    // a collector hiccup or a variance-collapsed baseline pinned at the display cap produces.
+    [Fact]
+    public void Score_LoneExtremeAnomaly_StaysAtBase_DoesNotPage()
+    {
+        var facts = new List<Fact> { TrustedAnomaly("ANOMALY_SESSION_SPIKE", 25.0, 3.5) };
+
+        new FactScorer().ScoreAll(facts);
+
+        var sessions = facts[0];
+        Assert.Equal(1.0, sessions.BaseSeverity, precision: 4);
+        Assert.Equal(1.0, sessions.Severity, precision: 4);
+        Assert.True(sessions.Severity < 1.5, "a single uncorroborated anomaly must not page, however extreme");
+        Assert.All(sessions.AmplifierResults, a => Assert.False(a.Matched));
+    }
+
+    // One corroborator is not enough either: extreme + a single +0.3 co-fire = 1.3, WARNING. Two
+    // independent corroborators is the bar (see the AnomalyAmplifiers worked numbers).
+    [Fact]
+    public void Score_ExtremeAnomaly_SingleCoFire_StaysWarning()
+    {
+        var facts = new List<Fact>
+        {
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5),
+            TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5),   // the only co-fire: +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE");
+        Assert.Equal(1.3, sessions.Severity, precision: 4);
+        Assert.True(sessions.Severity < 1.5, "one corroborator lifts an extreme anomaly, but not into CRITICAL");
+    }
+
+    // A family never corroborates itself: the CPU anomaly's arm carries the other three load siblings and
+    // the measured-CPU confirmation, but no "CPU anomaly co-fired" entry.
+    [Fact]
+    public void Score_AnomalyCoFireArm_DoesNotSelfCorroborate()
+    {
+        var facts = new List<Fact> { TrustedAnomaly("ANOMALY_CPU_SPIKE", 12.0, 3.5) };
+
+        new FactScorer().ScoreAll(facts);
+
+        var cpu = facts[0];
+        Assert.DoesNotContain(cpu.AmplifierResults, a => a.Description.StartsWith("CPU anomaly co-fired"));
+        Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Session-count anomaly co-fired"));
+        Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Batch-request anomaly co-fired"));
+        Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Query-duration anomaly co-fired"));
+        Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("SQL Server CPU >= 80%"));
+    }
+
+    // The wait-profile family: graded and escaped off the SAME statistic. On the robust trigger the
+    // escape opens at 3 x the 5.0 heavy-tail cutoff = 15σ (also the ramp's saturation point); read-
+    // latency and query-duration anomalies corroborate at +0.3 each → 1.6. A hair under 15σ stays
+    // capped at 1.49 with the same co-fires.
+    [Theory]
+    [InlineData(15.0, 1.6)]
+    [InlineData(14.99, 1.49)]
+    public void Score_WaitProfileAnomaly_EscapesAtThreeTimesTheHeavyTailCutoff(double modifiedZ, double expected)
+    {
+        var facts = new List<Fact>
+        {
+            new()
+            {
+                Source = "anomaly",
+                Key = "ANOMALY_WAIT_PROFILE",
+                Value = 1,
+                Metadata = new() { ["modified_z"] = modifiedZ, ["ratio"] = 3.0, ["is_new"] = 0 }
+            },
+            TrustedAnomaly("ANOMALY_READ_LATENCY", 12.0, 3.5),      // +0.3
+            TrustedAnomaly("ANOMALY_QUERY_DURATION", 15.0, 5.0),    // +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var profile = facts.First(f => f.Key == "ANOMALY_WAIT_PROFILE");
+        Assert.Equal(expected, profile.Severity, precision: 4);
+    }
+
+    // A first-occurrence wait profile (is_new) carries the NoBaselineRatio sentinel (100 — far past any
+    // ratio bar) as a scoring device, not a measurement. It must never escape: "no baseline" cannot be
+    // "extreme against baseline". With the same two co-fires the 1.6 product is capped to 1.49.
+    [Fact]
+    public void Score_IsNewWaitProfile_NeverEscapes_DespiteSentinelRatio()
+    {
+        var facts = new List<Fact>
+        {
+            new()
+            {
+                Source = "anomaly",
+                Key = "ANOMALY_WAIT_PROFILE",
+                Value = 1,
+                Metadata = new() { ["modified_z"] = 0.0, ["ratio"] = PerformanceMonitor.Analysis.Baselines.AnomalyThresholds.NoBaselineRatio, ["is_new"] = 1 }
+            },
+            TrustedAnomaly("ANOMALY_READ_LATENCY", 12.0, 3.5),      // +0.3
+            TrustedAnomaly("ANOMALY_QUERY_DURATION", 15.0, 5.0),    // +0.3
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var profile = facts.First(f => f.Key == "ANOMALY_WAIT_PROFILE");
+        Assert.Equal(1.0, profile.BaseSeverity, precision: 4);   // the sentinel ratio saturates the ramp
+        Assert.Equal(1.49, profile.Severity, precision: 4);
+    }
+
+    // The ratio/count families are NOT released: a blocking spike at 50x its baseline rate (base 1.0,
+    // saturated) has no extremity arm and no co-fire arm — it scores its base, and the cap is moot. Its
+    // impact reaches CRITICAL through the never-capped BLOCKING_EVENTS / DEADLOCKS keys instead.
+    [Fact]
+    public void Score_BlockingSpikeAnomaly_NoEscape_NoArm()
+    {
+        var facts = new List<Fact>
+        {
+            new() { Source = "anomaly", Key = "ANOMALY_BLOCKING_SPIKE", Value = 1, Metadata = new() { ["ratio"] = 50.0, ["is_new"] = 0 } },
+            TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5),
+            new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 },
+        };
+
+        new FactScorer().ScoreAll(facts);
+
+        var spike = facts.First(f => f.Key == "ANOMALY_BLOCKING_SPIKE");
+        Assert.Equal(1.0, spike.BaseSeverity, precision: 4);
+        Assert.Equal(1.0, spike.Severity, precision: 4);
+        Assert.Empty(spike.AmplifierResults);
+    }
+
     // THREADPOOL is the impact-bearing escalation path and is NEVER capped. The raised CXPACKET
     // amplifier (+0.5) and the new runnable-queue amplifier (+0.5, off the RUNNABLE_TASKS context fact)
     // drive a genuine parallelism -> thread-exhaustion meltdown to CRITICAL through THREADPOOL.