feat(auto-rebase-health): measure post-restriction fan-out reduction (#739) #890
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,6 +30,18 @@ setup() { | |
| {"conclusion":"success","created_at":"2026-06-11T00:00:00Z"}, | ||
| {"conclusion":"success","created_at":"2026-06-11T06:00:00Z"} | ||
| ]' | ||
|
|
||
| # PR eligibility metadata: 4 open non-Dependabot PRs, 2 eligible. | ||
| # approved non-draft → eligible | ||
| # ready-labelled non-draft → eligible | ||
| # plain non-draft → not eligible | ||
| # approved but draft → not eligible (draft) | ||
| PRS_META_JSON='[ | ||
| {"draft":false,"approved":true,"labels":[]}, | ||
| {"draft":false,"approved":false,"labels":[{"name":"auto-rebase:ready"}]}, | ||
| {"draft":false,"approved":false,"labels":[{"name":"bug"}]}, | ||
| {"draft":true,"approved":true,"labels":[]} | ||
| ]' | ||
| } | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
|
|
@@ -138,3 +150,109 @@ setup() { | |
| [ "$status" -eq 0 ] | ||
| [[ "$output" == *"n/a"* ]] | ||
| } | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # fmt_reduction — behind→eligible multiplier reduction | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| @test "fmt_reduction: renders an integer percentage decrease" { | ||
| run fmt_reduction 8 2 | ||
| [ "$output" = "75%" ] | ||
| } | ||
|
|
||
| @test "fmt_reduction: no reduction renders 0%" { | ||
| run fmt_reduction 5 5 | ||
| [ "$output" = "0%" ] | ||
| } | ||
|
|
||
| @test "fmt_reduction: zero base renders n/a (no divide-by-zero)" { | ||
| run fmt_reduction 0 0 | ||
| [ "$output" = "n/a" ] | ||
| } | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # pr_has_current_approval — current review state wins | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| @test "pr_has_current_approval: a lone APPROVED review counts as approved" { | ||
| run pr_has_current_approval '[{"user":{"login":"a"},"state":"APPROVED"}]' | ||
| [ "$status" -eq 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: a later CHANGES_REQUESTED cancels an earlier APPROVED (same user)" { | ||
| run pr_has_current_approval '[{"user":{"login":"a"},"state":"APPROVED"},{"user":{"login":"a"},"state":"CHANGES_REQUESTED"}]' | ||
| [ "$status" -ne 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: one user's APPROVED survives another's CHANGES_REQUESTED" { | ||
| run pr_has_current_approval '[{"user":{"login":"a"},"state":"APPROVED"},{"user":{"login":"b"},"state":"CHANGES_REQUESTED"}]' | ||
| [ "$status" -eq 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: COMMENTED-only reviews are not an approval" { | ||
| run pr_has_current_approval '[{"user":{"login":"a"},"state":"COMMENTED"}]' | ||
| [ "$status" -ne 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: empty/absent reviews are not an approval" { | ||
| run pr_has_current_approval "" | ||
| [ "$status" -ne 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: null-user review is skipped without error" { | ||
| run pr_has_current_approval '[{"user":null,"state":"APPROVED"}]' | ||
| [ "$status" -ne 0 ] | ||
| } | ||
|
|
||
| @test "pr_has_current_approval: null-user review does not block a valid approval" { | ||
| run pr_has_current_approval '[{"user":null,"state":"CHANGES_REQUESTED"},{"user":{"login":"a"},"state":"APPROVED"}]' | ||
| [ "$status" -eq 0 ] | ||
| } | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # count_eligible — non-draft AND (approved OR ready label) | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| @test "count_eligible: counts non-draft approved-or-labelled PRs" { | ||
| run count_eligible "$PRS_META_JSON" "auto-rebase:ready" | ||
| [ "$status" -eq 0 ] | ||
| [ "$output" -eq 2 ] | ||
| } | ||
|
|
||
| @test "count_eligible: a different ready label drops the label-only PR" { | ||
| run count_eligible "$PRS_META_JSON" "some-other-label" | ||
| [ "$output" -eq 1 ] | ||
| } | ||
|
|
||
| @test "count_eligible: empty/absent JSON returns 0 (does not error)" { | ||
| run count_eligible "" "auto-rebase:ready" | ||
| [ "$status" -eq 0 ] | ||
| [ "$output" -eq 0 ] | ||
| } | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # render_report — post-restriction section (eligibility supplied) | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| @test "render_report: post-restriction section appears only when eligible is supplied" { | ||
| run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 8 2026-06-15 | ||
| [[ "$output" != *"Post-restriction"* ]] | ||
|
|
||
| run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 8 2026-06-15 2 auto-rebase:ready | ||
| [[ "$output" == *"Post-restriction fan-out"* ]] | ||
| } | ||
|
|
||
| @test "render_report: post-restriction reports reduction and meets the ≥50% metric" { | ||
| # 8 behind → 2 eligible = 75% reduction (≥50% → met); 5 runs × 2 = 10 restricted re-runs | ||
| run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 8 2026-06-15 2 auto-rebase:ready | ||
| [ "$status" -eq 0 ] | ||
| [[ "$output" == *"75%"* ]] | ||
| [[ "$output" == *"met"* ]] | ||
| [[ "$output" == *"~10"* ]] | ||
| } | ||
|
|
||
| @test "render_report: sub-50% reduction is reported as not met" { | ||
| # 4 behind → 3 eligible = 25% reduction (< 50% → not met) | ||
| run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 4 2026-06-15 3 auto-rebase:ready | ||
| [[ "$output" == *"not met"* ]] | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If a review is submitted by a deleted user, the
.userobject in the GitHub API response can benull. Attempting to access.user.logindirectly will causejqto throw aCannot index null with string "login"error. We should safely navigate.userand ensure.user?.loginis not null before indexing.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done in scripts/auto_rebase_health.sh
pr_has_current_approval: added.user?.login != nullto theselectpredicate so reviews from deleted users (where.useris null) are filtered out before thereduceaccesses.user.login, preventing theCannot index null with string "login"jq error. Added two new bats tests covering the null-user case in tests/auto_rebase_health.bats.