fix(combos): a spent pool account must not cool the whole target (carry #6234)

docs-site/src/content/docs/guides/combos.md

@@ -354,6 +354,10 @@ five seconds. A valid immediate
 `Retry-After: 0` remains an immediate upstream directive rather than being replaced by a configured
 cooldown.
 
+For an Anthropic OAuth or Codex pool, a 429 tied to one account that the pool has cooled does
+not cool the whole combo target. Other accounts behind that target remain available. A 429 with
+no identified, cooled pool account still cools the target, as do provider-wide failures.
+
 ### Last-resort targets
 
 A brief cooldown on a preferred target otherwise routes straight to whatever

scripts/test-layout/layout.json

@@ -100,6 +100,7 @@
     "ambiguous-resend-composition.test.ts": "lib",
     "ambiguous-resend-gate.test.ts": "lib",
     "anthropic-account-pool.test.ts": "adapters/anthropic",
+    "anthropic-combo-account-cooldown.test.ts": "adapters/anthropic",
     "anthropic-model-routes.test.ts": "adapters/anthropic",
     "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic",
     "anthropic-baseurl-override.test.ts": "adapters/anthropic",

src/combos/resolve.ts

@@ -380,13 +380,28 @@ export function advanceComboAfterFailure(
     status?: number;
     code?: string | null;
     message?: string;
+    /** Account-qualified provider label of the account this failure came from, when a pool resolved one. */
+    failedAccount?: string;
   } = {},
 ): ComboPick | null {
   noteComboFailure(pick.comboId, pick.target, pick.writerGeneration);
   const combo = getCombo(config, pick.comboId);
+  // A 429 from a pooled provider names the ACCOUNT that is spent, not the target. The pool
+  // records that account's own cooldown and rotates past it, so cooling the target here blacks
+  // out the accounts that still answer: on 2026-09-28 one Anthropic account replied
+  // `Retry-After: 318747` (88.5h) and anthropic/claude-opus-5-5 left selection for four hours
+  // while four of five accounts still returned 200.
+  //
+  // Only a pool-resolved account qualifies, and an unidentified account still cools the target --
+  // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks
+  // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is
+  // holding the target back, the target cooldown still does.
+  const accountScoped = options.cooldownScope === "target"
+    && options.status === 429
+    && options.failedAccount !== undefined;
   // "none" records no cooldown at all: the failure described the request, not the target, so
   // the target must stay immediately selectable for the next (differently shaped) request.
-  if (options.cooldownScope !== "none") {
+  if (options.cooldownScope !== "none" && !accountScoped) {
     const cooldownTargets = options.cooldownScope === "provider" && combo
       ? combo.targets.filter(target => target.provider === pick.target.provider)
       : [pick.target];

src/oauth/anthropic-routing.ts

@@ -736,13 +736,41 @@ export function rotateAnthropicAccountOn429(
   rateLimitHeaders?: AnthropicRateLimitHeaders | null,
   decision: AnthropicRouteDecision | null = null,
 ): string | null {
+  if (!recordAnthropicAccount429(config, failedAccountId, retryAfterHeader, now, rateLimitHeaders)) return null;
+
+  // The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive
+  // presence-only recovery must not silently reactivate round-robin/fill-first merely
+  // because those dormant values remain in config. The quota picker is the neutral
+  // recovery policy already used by the default strategy.
+  const next = isAnthropicAccountPoolEnabled(config)
+    ? pickAlternateAnthropicAccount(config, failedAccountId, now, decision)
+    : pickLowestUsage(config, failedAccountId, now);
+  if (!next) {
+    console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning 429`);
+    return null;
+  }
+
+  console.warn(
+    `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`,
+  );
+  return next;
+}
+
+/** Record the account refusal even when this request has no remaining retry sends. */
+export function recordAnthropicAccount429(
+  config: OcxConfig,
+  failedAccountId: string,
+  retryAfterHeader: string | null | undefined,
+  now = Date.now(),
+  rateLimitHeaders?: AnthropicRateLimitHeaders | null,
+): boolean {
   // Reactive 429 failover is NOT gated on the pool flag. That flag buys PROACTIVE routing --
   // session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all
   // of which move a HEALTHY request and stay opt-in. Rotating away from an account upstream has
   // just rate-limited is a different thing: it only ever runs after a refusal, and stranding a
   // 429 while a second logged-in account sits idle is a defect, not a configuration choice.
   // Presence is the activation rule, the same one an apiKeyPool of two keys already uses.
-  if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return null;
+  if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return false;
 
   // Retry-After first: it is the header written FOR this decision. The rejected window's
   // reset is the fallback, because a 429 that omits Retry-After still carries it -- and
@@ -760,26 +788,10 @@ export function rotateAnthropicAccountOn429(
   sweepExpiredOnWrite(now);
   clearAnthropicSessionAffinityForAccount(failedAccountId);
   notePoolRotationFailure(POOL_KEY_ANTHROPIC, failedAccountId);
-  // A rotation means the roster in use just changed; do not answer the next activation question
-  // from a count read taken before the failure.
+  // The refused account changed the eligible roster even when no retry send remains.
   quorumCache = null;
 
-  // The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive
-  // presence-only recovery must not silently reactivate round-robin/fill-first merely
-  // because those dormant values remain in config. The quota picker is the neutral
-  // recovery policy already used by the default strategy.
-  const next = isAnthropicAccountPoolEnabled(config)
-    ? pickAlternateAnthropicAccount(config, failedAccountId, now, decision)
-    : pickLowestUsage(config, failedAccountId, now);
-  if (!next) {
-    console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning 429`);
-    return null;
-  }
-
-  console.warn(
-    `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`,
-  );
-  return next;
+  return true;
 }
 
 export interface AnthropicSelectionRoutingOptions {

src/providers/label.ts

@@ -1,4 +1,4 @@
-import { CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label";
+import { ACCOUNT_LOG_LABEL_RE, CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label";
 import type { OcxProviderConfig } from "../types";
 
 export function canonicalUsageProviderLabel(provider: string): string {
@@ -25,6 +25,25 @@ export function stampApiKeyAccountLabel(
   }
 }
 
+/**
+ * The account-qualified provider label a pool dispatch stamped for this request, or undefined.
+ *
+ * An account pool rewrites the logged provider to name the account that served the turn
+ * (`anthropic-p2e22d0`, `openai-main`); the configured provider name never does, so a label that
+ * differs from the configured one AND ends in an account label identifies exactly one account.
+ *
+ * Deliberately NOT `accountLogLabel`: that is stamped for a lone configured API key too, so it
+ * cannot tell a pooled account apart from a provider that has no pool to rotate within.
+ */
+export function poolAccountProviderLabel(
+  loggedProvider: string | undefined,
+  configuredProvider: string,
+): string | undefined {
+  if (!loggedProvider?.startsWith(`${configuredProvider}-`)) return undefined;
+  return ACCOUNT_LOG_LABEL_RE.test(loggedProvider.slice(configuredProvider.length + 1))
+    ? loggedProvider : undefined;
+}
+
 export function baseProviderLabel(provider: string): string {
   const canonical = canonicalUsageProviderLabel(provider);
   if (canonical !== provider) return canonical;

src/server/responses/adapter-continuation.ts

@@ -35,6 +35,7 @@ import { bindRouteReasoningReplayScope } from "./core-replay";
 import {
   ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
   rotateAnthropicAccountOn429,
+  recordAnthropicAccount429,
   getAnthropicPoolAccessSnapshot,
   formatAnthropicProviderForLog,
 } from "../../oauth/anthropic-routing";
@@ -425,6 +426,11 @@ export function createAdapterContinuations(
           }
         }
       }
+      if (response.status === 429 && transportState.anthropicPoolAccountId
+        && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) {
+        recordAnthropicAccount429(config, transportState.anthropicPoolAccountId,
+          response.headers.get("retry-after"), Date.now(), response.headers);
+      }
       // Generic OAuth rotation for the continuation loop. The streaming loop grew this arm with
       // #2568 and this one did not, so an xAI/Cursor/Kimi/Copilot/Antigravity/Nous continuation
       // 429 stayed terminal even with failover fully active -- the same class of divergence the

src/server/responses/adapter-dispatch.ts

@@ -56,6 +56,7 @@ import { bindRouteReasoningReplayScope } from "./core-replay";
 import {
   ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
   rotateAnthropicAccountOn429,
+  recordAnthropicAccount429,
   getAnthropicPoolAccessSnapshot,
   formatAnthropicProviderForLog,
 } from "../../oauth/anthropic-routing";
@@ -1000,6 +1001,11 @@ export async function prepareAdapterExchange(
           break;
         }
       }
+      if (upstreamResponse.status === 429 && transportState.anthropicPoolAccountId
+        && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) {
+        recordAnthropicAccount429(config, transportState.anthropicPoolAccountId,
+          upstreamResponse.headers.get("retry-after"), Date.now(), upstreamResponse.headers);
+      }
       // Generic OAuth account failover (#2568) rotates reactively after a refusal when
       // two accounts are stored. Kiro additionally classifies bounded 400/403 refusals;
       // all other providers retain the original 429 loop below.

src/server/responses/core-combo.ts

@@ -46,6 +46,11 @@ import {
 import { hasUnreadableEncryptedAgentTask } from "./encrypted-payload";
 import { routeConcreteModel, comboRouteDecisionTrace } from "../../router";
 import { memoryModelRouteReason } from "./memory-models";
+import { poolAccountProviderLabel } from "../../providers/label";
+import { getAccountSet } from "../../oauth/store";
+import { formatAnthropicProviderForLog, getAnthropicAccountHealthSnapshot } from "../../oauth/anthropic-routing";
+import { codexAccountLogLabel } from "../../codex/account-label";
+import { codexQuotaScopeForModel, getCodexQuotaHealthSnapshot } from "../../codex/routing";
 import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
 import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery";
 import {
@@ -97,6 +102,22 @@ import { clientWireOf } from "../inference/client-wire";
  */
 export const COMBO_TARGET_BASE_SENDS = CODEX_TEXT_GUARDED_BUDGET_POLICY.baseSendAllowance;
 
+function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId: string, label: string | undefined): string | undefined {
+  if (!label) return undefined;
+  if (providerName === "anthropic") {
+    const matches = getAccountSet("anthropic")?.accounts.filter(account =>
+      formatAnthropicProviderForLog("anthropic", account.id) === label) ?? [];
+    return matches.length === 1 && getAnthropicAccountHealthSnapshot(matches[0]!.id) ? label : undefined;
+  }
+  const provider = config.providers[providerName];
+  if (!provider || !isCanonicalOpenAiForwardProvider(provider)) return undefined;
+  const matches = (config.codexAccounts ?? []).filter(account =>
+    `${providerName}-${codexAccountLogLabel(account)}` === label);
+  return matches.length === 1
+    && getCodexQuotaHealthSnapshot(matches[0]!.id, codexQuotaScopeForModel(modelId))
+    ? label : undefined;
+}
+
 
 /**
  * A combo's execution policy is DECLARED by the combo, not inherited from the single-target
@@ -977,6 +998,13 @@ export async function executeComboResponses(
       status: failure.response.status,
       code: failure.upstreamCode,
       message: failure.classificationText,
+      // The dispatch rewrote this to name the pool account that actually served the turn.
+      failedAccount: cooledPoolAccountLabel(
+        config,
+        pick.target.provider,
+        pick.target.model,
+        poolAccountProviderLabel(childLog.provider, pick.target.provider),
+      ),
       onCooldownRecorded: target => {
         failedTargetCooldownRecorded ||= targetKey(target) === failedTargetKey;
       },

structure/providers-and-adapters.md

@@ -2,6 +2,8 @@
 
 For Anthropic OAuth, `src/oauth/anthropic-routing.ts` applies the first matching `anthropicAccountPool.routes` rule to every eligible pick. The declared account order is stable while its candidates remain eligible; active, manual, affinity, quota and strategy preferences only choose inside that set. A healthy session affinity outside a model route is ignored for that request and retained for later unrouted or differently routed models; the routed commit does not overwrite it. An explicit fallback widens an empty route to the ordinary pool, and fill-first then advances in ordinary pool order from the active account. A missing eligible route fails locally without that fallback. The rules are operator allowlists, not provider entitlement evidence. Request logs use `route:#<n>` for the 1-based rule position, not the operator name.
 
+An Anthropic 429 records the served account's cooldown even when the request has used its allowed retry sends. That final account remains excluded on the next request; combo target cooling is skipped only after the matching account cooldown is present.
+
 GitHub Copilot `modelContextTiers` is selected per upstream model. The Chat and Responses
 adapters set `contextTier` only when the canonical routed provider is `github-copilot`
 and a tier is configured. Otherwise passthrough retains caller-supplied values. The server carries provider identity

structure/runtime.md

@@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str
 
 Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`.
 
-`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes.
+`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown.
 
 ## Combo default effort precedence