Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions scripts/canary-rollout.sh
Original file line number Diff line number Diff line change
Expand Up @@ -187,16 +187,33 @@ _reusable_differs() {
# unless state is BLOCKED (then REGRESSION | PRE_EXISTING).
_frontier_state() {
local agent="$1"
local cand chans frontier=""
local cand chans frontier="" prev_on=()
local soak_start use_soak_as_candidate=false next_health_signal
soak_start="$(_agent_field "$agent" soak_start_ring)"
[ "$soak_start" = "null" ] && soak_start=""
next_health_signal="$(_agent_field "$agent" next_tier_health_signal)"
[ "$next_health_signal" = "null" ] && next_health_signal=""

cand="$(channel_commit "$agent" next)"
# soak_start_ring agents have no @next channel caller — the innermost soaked ring is
# the candidate source. Fall back only when next genuinely has no tag.
if [ -z "$cand" ] && [ -n "$soak_start" ]; then
cand="$(channel_commit "$agent" "$soak_start")"
use_soak_as_candidate=true
fi
chans="$(ordered_channels "$agent")"

# Collect the rings already on the candidate (starting at next / soak_start) and find the frontier.
local chan_array=()
IFS=, read -r -a chan_array <<< "$chans"
local ch
for ch in "${chan_array[@]}"; do
local c; c="$(channel_commit "$agent" "$ch")"
if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi
if [ "$ch" = "next" ] || { [ "$use_soak_as_candidate" = "true" ] && [ "$ch" = "$soak_start" ]; } || [ "$c" = "$cand" ]; then
prev_on+=("$ch")
else
frontier="$ch"; break
fi
done
if [ -z "$frontier" ]; then
echo "$cand - - COMPLETE 0 0 0 0 0 0 -"; return 0
Expand All @@ -207,18 +224,36 @@ _frontier_state() {
source="${transition%%->*}"
cut_z="$(candidate_cut_date "$agent" "$cand")"
if [ -z "$cut_z" ]; then
# Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries.
echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 -"; return 0
if [ -z "$soak_start" ]; then
# Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries.
echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 -"; return 0
fi
# soak_start_ring agents may lack a vX.Y.Z release tag; proceed with an unbounded window.
fi
now_epoch="$(date -u +%s)"

# Source-tier repos (the tier currently running the candidate).
local src_repos=() r
while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source")
# When the frontier IS the soak_start_ring (transition is empty — no source ring below it),
# use the host repo with next_tier_health_signal as the pre-soak health source.
local src_repos=() r use_health_signal=false
if [ -z "$transition" ] && [ -n "$next_health_signal" ]; then
use_health_signal=true
src_repos=("$(_agent_field "$agent" host)")
else
while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source")
fi

# Sample on the source tier over the per-candidate window.
local sample earliest
read -r sample earliest < <(_tier_sample "$agent" "$cut_z" "${src_repos[@]}")
if [ "$use_health_signal" = "true" ]; then
local hs_json
hs_json="$(_run_json "${src_repos[0]}" "$next_health_signal" "$cut_z")"
sample=$(jq '[.[]?|select(.conclusion=="success" or .conclusion=="failure")]|length' 2>/dev/null <<< "$hs_json" || echo 0)
earliest="$(jq -r '[.[]?|select(.conclusion=="success" or .conclusion=="failure")|.createdAt?]|min // empty' 2>/dev/null <<< "$hs_json")"
[ -z "$earliest" ] && earliest="-"
else
read -r sample earliest < <(_tier_sample "$agent" "$cut_z" "${src_repos[@]}")
fi

# Dwell is always measured from the candidate's own cut (tagger date), per #548 spec.
local dwell_h=0
Expand Down
30 changes: 29 additions & 1 deletion standards/canary-rings.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
{
"version": 1,
"description": "Ring membership for agent canary rollout (initiative #495, issues #500/#501). The promotion automation (scripts/canary-rollout.sh) reads this as the single source of truth. `next` is host-relative: it resolves to the repo that HOSTS the reusable; `ring0` covers the other org-infra repo (the host already sits in `next`).",
"description": "Ring membership for agent canary rollout (initiative #495, issues #500/#501; pipeline onboarding #1008). The promotion automation (scripts/canary-rollout.sh) reads this as the single source of truth. `next` is host-relative: it resolves to the repo that HOSTS the reusable; `ring0` covers the other org-infra repo (the host already sits in `next`). An agent may set `soak_start_ring` to name the innermost ring that actually has a channel caller — used when the host dogfoods the reusable via a local `./` ref (so there is no `@next`/`ring0` channel caller) and no org-infra repo consumes it as a channel member; soak then begins at that ring. `unmanaged` records reusable-less / non-channel-promotable workflows (e.g. direct `gh workflow run` dispatch) so the automation does not try to gate them.",
"org_infra_repos": ["petry-projects/.github", "petry-projects/.github-private"],
"member_tokens": {
"$host": "the agent's host repo (the repo that owns the reusable)",
"$org_infra": "org_infra_repos minus the host (the host is already in `next`)",
"*": "every other consumer not named in an earlier ring"
},
"unmanaged": {
"initiative-driver": {
"reason": "Direct `gh workflow run` dispatch to petry-projects/.github-private; no reusable workflow, so it is not channel-promotable. Recorded here (out of model, #1008) so the canary automation does not attempt to gate it.",
"health_source": ".github/workflows/initiative-driver-canary.yml"
}
},
"agents": {
"dev-lead": {
"host": "petry-projects/.github-private",
Expand Down Expand Up @@ -40,6 +46,28 @@
}
}
}
},
"initiative-planner": {
"host": "petry-projects/.github",
"reusable": ".github/workflows/initiative-planner-reusable.yml",
"run_workflow": "Initiative Planner — Approval Trigger",
"soak_start_ring": "ring1",
"next_tier_health_signal": ".github/workflows/initiative-planner-canary.yml",
Comment thread
don-petry marked this conversation as resolved.
Comment thread
Copilot marked this conversation as resolved.
"rings": [
{ "channel": "ring1", "order": 2, "members": ["petry-projects/TalkTerm", "petry-projects/bmad-bgreat-suite"] },
{ "channel": "stable", "order": 3, "members": ["*"] }
]
Comment thread
don-petry marked this conversation as resolved.
},
"idea-triage": {
"host": "petry-projects/.github",
"reusable": ".github/workflows/idea-triage-reusable.yml",
"run_workflow": "Idea Triage — Weekly Shortlist",
"soak_start_ring": "ring1",
Comment thread
don-petry marked this conversation as resolved.
"next_tier_health_signal": ".github/workflows/idea-triage-canary.yml",
"rings": [
Comment thread
Copilot marked this conversation as resolved.
{ "channel": "ring1", "order": 2, "members": ["petry-projects/TalkTerm", "petry-projects/bmad-bgreat-suite"] },
{ "channel": "stable", "order": 3, "members": ["*"] }
]
}
}
}
131 changes: 131 additions & 0 deletions tests/canary_rollout.bats
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,69 @@ setup() {
[ "$status" -eq 0 ]
}

# ── canary-rings.json: idea→initiative pipeline onboarding (#1008) ─────────────
# initiative-planner + idea-triage are onboarded as managed agents whose reusable
# lives cross-repo in petry-projects/.github. Their host dogfoods via a local `./`
# ref (no @next caller) and .github-private runs a full central copy (not a channel
# consumer), so there is no next/ring0 channel caller — soak begins at ring1.
_pipeline_agent_shape() {
# $1 = agent key. Asserts the shared onboarding shape for planner/triage.
local agent="$1"
# hosted in the public repo (reusable lives there, not in .github-private)
run jq -e --arg a "$agent" '.agents[$a].host == "petry-projects/.github"' "$RINGS"
[ "$status" -eq 0 ]
# soak starts at ring1 (owner-confirmed: no channel caller below ring1)
run jq -e --arg a "$agent" '.agents[$a].soak_start_ring == "ring1"' "$RINGS"
[ "$status" -eq 0 ]
# only ring1 + stable channels are modelled, ordered ring1 -> stable
run jq -r --arg a "$agent" '.agents[$a].rings | sort_by(.order) | map(.channel) | join(",")' "$RINGS"
[ "$output" = "ring1,stable" ]
# ring1 = the two low-traffic consumers
run jq -e --arg a "$agent" '.agents[$a].rings[] | select(.channel=="ring1") | (.members | index("petry-projects/TalkTerm")) and (.members | index("petry-projects/bmad-bgreat-suite"))' "$RINGS"
[ "$status" -eq 0 ]
# stable = ["*"] (un-enrolled today; populated by the promotion automation)
run jq -e --arg a "$agent" '.agents[$a].rings[] | select(.channel=="stable") | .members == ["*"]' "$RINGS"
[ "$status" -eq 0 ]
# .github-private runs a full central copy — never a channel member of these agents
run jq -e --arg a "$agent" '[.agents[$a].rings[].members[]] | index("petry-projects/.github-private") | not' "$RINGS"
[ "$status" -eq 0 ]
}

@test "canary-rings.json: initiative-planner onboarded (soak_start_ring=ring1, cross-repo host)" {
_pipeline_agent_shape "initiative-planner"
# its pre-ring1 health signal is the in-repo canary workflow
run jq -e '.agents["initiative-planner"].next_tier_health_signal == ".github/workflows/initiative-planner-canary.yml"' "$RINGS"
[ "$status" -eq 0 ]
}

@test "canary-rings.json: idea-triage onboarded (soak_start_ring=ring1, cross-repo host)" {
_pipeline_agent_shape "idea-triage"
# same pattern as initiative-planner: needs next_tier_health_signal for pre-ring1 gate
run jq -e '.agents["idea-triage"].next_tier_health_signal == ".github/workflows/idea-triage-canary.yml"' "$RINGS"
[ "$status" -eq 0 ]
}

@test "canary-rings.json: initiative-driver is recorded as unmanaged, not a channel agent" {
# no reusable -> not channel-promotable -> excluded from .agents so the
# automation never tries to gate it.
run jq -e '.agents["initiative-driver"] == null' "$RINGS"
[ "$status" -eq 0 ]
# but explicitly recorded under .unmanaged with a rationale + health source.
run jq -e '.unmanaged["initiative-driver"].reason | type == "string" and (. | length > 0)' "$RINGS"
[ "$status" -eq 0 ]
run jq -e '.unmanaged["initiative-driver"].health_source == ".github/workflows/initiative-driver-canary.yml"' "$RINGS"
[ "$status" -eq 0 ]
}

@test "canary-rings.json: blocked/hold reusables are NOT onboarded" {
# idea-enhancer (zero tags/callers) and feature-ideation (open bug #571 + missing
# channel tags) must stay out of the SoT entirely — neither agent nor unmanaged.
run jq -e '.agents["idea-enhancer"] == null and .agents["feature-ideation"] == null' "$RINGS"
[ "$status" -eq 0 ]
run jq -e '.unmanaged["idea-enhancer"] == null and .unmanaged["feature-ideation"] == null' "$RINGS"
[ "$status" -eq 0 ]
}

# ── orchestrator: resolve_members (host-relative tokens) ──────────────────────
@test "orchestrator: resolve_members expands \$host / \$org_infra / * " {
run bash -c "source '$ORCH' && CANARY_RINGS='$RINGS' resolve_members dev-lead next"
Expand Down Expand Up @@ -220,6 +283,74 @@ GHEOF
[[ "$output" == *"stable"* ]]
}

@test "orchestrator: evaluate uses next_tier_health_signal when prev_on is empty (soak_start_ring agent)" {
_make_stub_bin
# initiative-planner/next is the candidate; ring1 is not yet on it → prev_on=[]
cat > "$STUB_BIN/git" <<'GITEOF'
#!/usr/bin/env bash
case "$*" in
*"rev-parse"*"initiative-planner/next"*) echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;
*"rev-parse"*"initiative-planner/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;
*"rev-parse"*"initiative-planner/stable"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;;
*) : ;;
esac
GITEOF
chmod +x "$STUB_BIN/git"
# Healthy runs from the host canary; ring member queries return empty (low traffic)
cat > "$STUB_BIN/gh" <<'GHEOF'
#!/usr/bin/env bash
case "$*" in
*"run list"*"initiative-planner-canary"*)
printf '['; for i in $(seq 1 15); do [ "$i" -gt 1 ] && printf ','; printf '{"conclusion":"success"}'; done; printf ']'
;;
*"run list"*) echo "[]" ;;
*) echo "{}" ;;
esac
GHEOF
chmod +x "$STUB_BIN/gh"

run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate initiative-planner
[ "$status" -eq 0 ]
[[ "$output" == *"initiative-planner"* ]]
[[ "$output" == *"ring1"* ]]
# 15 healthy runs from canary, baseline=0 → ceil(0/7)=0 min needed → PROMOTE
[[ "$output" == *"PROMOTE"* ]]
}

@test "orchestrator: evaluate uses ring1 health when soak_start_ring is the candidate source (no next tag)" {
_make_stub_bin
# initiative-planner has soak_start_ring=ring1 and no @next channel caller.
# ring1 tag exists (aaaa = candidate); stable tag is on a different commit (bbbb).
# _frontier_state must treat ring1 as the candidate source and identify stable as
# the frontier (not get stuck or falsely report COMPLETE).
cat > "$STUB_BIN/git" <<'GITEOF'
#!/usr/bin/env bash
case "$*" in
*"rev-parse"*"initiative-planner/ring1"*) echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;
*"rev-parse"*"initiative-planner/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;
*) : ;;
esac
GITEOF
chmod +x "$STUB_BIN/git"
# ring1 members return healthy runs; stable frontier members are wildcards (skipped)
cat > "$STUB_BIN/gh" <<'GHEOF'
#!/usr/bin/env bash
case "$*" in
*"run list"*) echo '[{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"}]' ;;
*) echo "{}" ;;
esac
GHEOF
chmod +x "$STUB_BIN/gh"

run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate initiative-planner
[ "$status" -eq 0 ]
[[ "$output" == *"initiative-planner"* ]]
# stable is the frontier (ring1 = soak_start_ring = already on candidate)
[[ "$output" == *"stable"* ]]
# 10+ healthy ring1 runs, baseline (stable/*) = 0 → min_healthy=0, healthy>0 → PROMOTE
[[ "$output" == *"PROMOTE"* ]]
}

@test "orchestrator: promote --override --dry-run shows the move but never pushes" {
_make_stub_bin
cat > "$STUB_BIN/gh" <<'GHEOF'
Expand Down
Loading