feat: implement issue #941 — [Fleet Monitor] petry-projects/.github-private — .github/workflows/test-deletion-guard.yml

AGENTS.md

@@ -47,6 +47,10 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c
   reviewable decision. The other half is the **"require branches up to date before merging"** ruleset on `main`
   (already enabled). Together they close the gap behind #655, where a PR merged from a stale base reverted a
   shipped fix and deleted its regression test in the same green-CI diff. Do not remove either guard.
+  Because this is a **gate** workflow whose `failure` conclusion is intentional enforcement (it blocks a
+  bad PR), it is exempt from the Fleet Monitor's high-failure issue tracking via `FLEET_GATE_WORKFLOWS`
+  in `scripts/fleet_report.sh` (#941) — otherwise a guard doing its job produces false-positive trackers.
+  `holdout-guard.yml` is exempt for the same reason. Add new gate workflows of this class to that list.
 - **Exception:** `pr-review-sweep.yml` (stuck-review sweep, #573/#898) is a documented repo-specific
   workflow with no corresponding org template in `standards/workflows/`. It re-dispatches reviews for PRs
   that went green after a ci-pending/ci-failing skip, via two triggers: a scheduled cron (the guaranteed

scripts/fleet_monitor.sh

@@ -297,41 +297,11 @@ fi
 # objects for every workflow whose failure rate exceeds 10%. The downstream
 # "Track high-failure workflows" step reads this file to find/update or create
 # a tracked GitHub Issue for each item and apply the dev-lead label.
+# Gate workflows (FLEET_GATE_WORKFLOWS in fleet_report.sh) are excluded here —
+# their failures are intentional policy enforcement, not breakage (#941).
 HIGH_FAILURE_FILE="fleet_high_failure.json"
 if [ -s "$metrics_file" ]; then
-  jq -Rn '
-    [inputs | select(length > 0) | split("\t") | select(length >= 12) |
-     {
-       sort_key:  .[0],
-       repo:      .[1],
-       workflow:  .[2],
-       total:    (.[3]  | tonumber? // 0),
-       success:  (.[4]  | tonumber? // 0),
-       failed:   (.[5]  | tonumber? // 0),
-       cancelled:(.[6]  | tonumber? // 0),
-       rate:      .[7],
-       p50:      (.[8]  | tonumber? // 0),
-       p95:      (.[9]  | tonumber? // 0),
-       label:     .[10],
-       rate_int: (.[11] | tonumber? // 0)
-     }] |
-    # Apply confidence filter: CRITICAL rows with < 5 total runs become LOW-CONF
-    map(if .label == "CRITICAL" and .total < 5 then .label = "LOW-CONF" else . end) |
-    # Keep trackable items:
-    #   ERROR rows (monitor could not read runs — always noteworthy regardless of rate)
-    #   WARNING/DEGRADED/CRITICAL with exact failure rate > 10%
-    #   (uses .failed/.total directly to avoid integer-truncation false negatives)
-    map(select(
-      .label == "ERROR" or
-      (
-        .label != "LOW-CONF" and
-        (.label | IN("WARNING","DEGRADED","CRITICAL")) and
-        .total > 0 and
-        (.failed * 100.0 / .total > 10)
-      )
-    )) |
-    sort_by(.failed * 100.0 / (if .total > 0 then .total else 1 end)) | reverse
-  ' < "$metrics_file" > "$HIGH_FAILURE_FILE"
+  filter_high_failure "$metrics_file" > "$HIGH_FAILURE_FILE"
   hf_count=$(jq 'length' "$HIGH_FAILURE_FILE")
   echo "High-failure items (>10%): ${hf_count}"
   [ -n "${GITHUB_ENV:-}" ] && echo "HIGH_FAILURE_COUNT=${hf_count}" >> "$GITHUB_ENV"

scripts/fleet_report.sh

@@ -6,6 +6,15 @@
 #   1:sort_key  2:repo  3:wf_file  4:total  5:success  6:failed
 #   7:cancelled  8:rate_display  9:p50(s)  10:p95(s)  11:label  12:rate_int
 
+# Gate workflows: their `failure` conclusion is intentional policy enforcement
+# (e.g. blocking a PR that violates a rule), not flakiness or breakage. A high
+# "failure rate" for these is the guard doing its job, so they are excluded from
+# high-failure issue tracking to avoid false-positive Fleet Monitor trackers
+# (#941). Matched by workflow file basename. See AGENTS.md "Exception:
+# test-deletion-guard.yml". Override via env with the full space-separated
+# gate list for testing; add permanent gates to the default below.
+FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml}"
+
 # label_to_icon <label>
 # Returns the health icon matching the scorecard legend.
 label_to_icon() {
@@ -54,6 +63,58 @@ apply_confidence_filter() {
   }'
 }
 
+# filter_high_failure <metrics_file>
+# Reads a metrics TSV and prints fleet_high_failure JSON — the array of workflows
+# worth tracking as a per-workflow issue:
+#   * ERROR rows (the monitor could not read runs — always noteworthy)
+#   * WARNING/DEGRADED/CRITICAL with exact failure rate > 10%
+# CRITICAL rows with < 5 total runs are downgraded to LOW-CONF and dropped.
+# Gate workflows (FLEET_GATE_WORKFLOWS, matched by basename) are excluded from
+# the rate-based path — their failures are intentional policy enforcement (#941) —
+# but a gate's ERROR row still surfaces (a read failure is not the gate's doing).
+filter_high_failure() {
+  local f="${1:-}"
+  if [ -z "$f" ] || [ ! -f "$f" ]; then
+    echo "Error: metrics file '${f}' does not exist or is not specified" >&2
+    return 1
+  fi
+  jq -Rn --arg gates "$FLEET_GATE_WORKFLOWS" '
+    ($gates | split(" ") | map(select(length > 0))) as $gate |
+    [inputs | select(length > 0) | split("\t") | select(length >= 12) |
+     {
+       sort_key:  .[0],
+       repo:      .[1],
+       workflow:  .[2],
+       total:    (.[3]  | tonumber? // 0),
+       success:  (.[4]  | tonumber? // 0),
+       failed:   (.[5]  | tonumber? // 0),
+       cancelled:(.[6]  | tonumber? // 0),
+       rate:      .[7],
+       p50:      (.[8]  | tonumber? // 0),
+       p95:      (.[9]  | tonumber? // 0),
+       label:     .[10],
+       rate_int: (.[11] | tonumber? // 0)
+     }] |
+    # Apply confidence filter: CRITICAL rows with < 5 total runs become LOW-CONF
+    map(if .label == "CRITICAL" and .total < 5 then .label = "LOW-CONF" else . end) |
+    # Keep trackable items:
+    #   ERROR rows (monitor could not read runs — always noteworthy regardless of rate)
+    #   non-gate WARNING/DEGRADED/CRITICAL with exact failure rate > 10%
+    #   (uses .failed/.total directly to avoid integer-truncation false negatives)
+    map(select(
+      .label == "ERROR" or
+      (
+        ((.workflow | split("/") | last | IN($gate[])) | not) and
+        .label != "LOW-CONF" and
+        (.label | IN("WARNING","DEGRADED","CRITICAL")) and
+        .total > 0 and
+        (.failed * 100.0 / .total > 10)
+      )
+    )) |
+    sort_by(.failed * 100.0 / (if .total > 0 then .total else 1 end)) | reverse
+  ' < "$f"
+}
+
 # detect_systemic_failures <metrics_file>
 # Prints the name of each workflow file that has failures in 3 or more repos.
 detect_systemic_failures() {

tests/fleet_report.bats

@@ -281,3 +281,80 @@ setup() {
   run generate_mermaid_bar "$METRICS"
   [[ "$output" =~ "bar [" ]]
 }
+
+# ---------------------------------------------------------------------------
+# filter_high_failure — high-failure JSON export + gate-workflow exclusion (#941)
+# ---------------------------------------------------------------------------
+
+# Writes a metrics TSV to a temp file and echoes its path.
+_mk_metrics() {
+  local f
+  f="$(mktemp)" || { echo "Failed to create temp file" >&2; exit 1; }
+  printf '%s\n' "$@" > "$f"
+  echo "$f"
+}
+
+@test "filter_high_failure: gate workflow over 10% is excluded from tracking" {
+  local m
+  m="$(_mk_metrics \
+    $'2\tpetry-projects/.github-private\t.github/workflows/test-deletion-guard.yml\t18\t16\t2\t0\t11.1%\t11\t82\tWARNING\t11')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 0 ]
+  rm -f "$m"
+}
+
+@test "filter_high_failure: gate matched by bare basename is excluded" {
+  local m
+  m="$(_mk_metrics \
+    $'2\tpetry-projects/.github-private\ttest-deletion-guard.yml\t18\t16\t2\t0\t11.1%\t11\t82\tWARNING\t11')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 0 ]
+  rm -f "$m"
+}
+
+@test "filter_high_failure: non-gate workflow over 10% is still tracked" {
+  local m
+  m="$(_mk_metrics \
+    $'0\tpetry-projects/.github-private\tci.yml\t23\t3\t20\t0\t87.0%\t67\t206\tCRITICAL\t87')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 1 ]
+  [ "$(echo "$output" | jq -r '.[0].workflow')" = "ci.yml" ]
+  rm -f "$m"
+}
+
+@test "filter_high_failure: ERROR row for a gate workflow is still surfaced" {
+  local m
+  m="$(_mk_metrics \
+    $'6\tpetry-projects/.github-private\t.github/workflows/test-deletion-guard.yml\t?\t\t\t\terror\t0\t0\tERROR\t0')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 1 ]
+  [ "$(echo "$output" | jq -r '.[0].label')" = "ERROR" ]
+  rm -f "$m"
+}
+
+@test "filter_high_failure: CRITICAL with under 5 runs is dropped as LOW-CONF" {
+  local m
+  m="$(_mk_metrics \
+    $'0\tpetry-projects/.github-private\tflaky.yml\t3\t0\t3\t0\t100%\t5\t5\tCRITICAL\t100')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 0 ]
+  rm -f "$m"
+}
+
+@test "filter_high_failure: mixed rows keep only the tracked non-gate workflow" {
+  local m
+  m="$(_mk_metrics \
+    $'0\tpetry-projects/.github-private\tci.yml\t23\t3\t20\t0\t87.0%\t67\t206\tCRITICAL\t87' \
+    $'2\tpetry-projects/.github-private\t.github/workflows/test-deletion-guard.yml\t18\t16\t2\t0\t11.1%\t11\t82\tWARNING\t11' \
+    $'3\tpetry-projects/.github-private\thealthy.yml\t50\t50\t0\t0\t0%\t10\t20\tHEALTHY\t0')"
+  run filter_high_failure "$m"
+  [ "$status" -eq 0 ]
+  [ "$(echo "$output" | jq 'length')" -eq 1 ]
+  [ "$(echo "$output" | jq -r '.[0].workflow')" = "ci.yml" ]
+  rm -f "$m"
+}