fix(client): a slow server no longer mints a new relay credential on every reconnect

packages/client-runtime/src/authorization/layer.test.ts

@@ -46,12 +46,19 @@ const BOOTSTRAP: RelayEnvironmentConnectResponse = {
   expiresAt: "2026-06-06T01:00:00.000Z",
 };
 
-function recordedFetch(responses: ReadonlyArray<Response>) {
+/** A response slot for a request that a stalled server never answers. */
+const STALLED = "stalled";
+
+function recordedFetch(responses: ReadonlyArray<Response | typeof STALLED>, onStall: () => void) {
   const calls: Array<readonly [RequestInfo | URL, RequestInit]> = [];
   let responseIndex = 0;
   const fetchFn = ((input, init) => {
     calls.push([input, init ?? {}]);
     const response = responses[responseIndex++];
+    if (response === STALLED) {
+      onStall();
+      return new Promise<Response>(() => {});
+    }
     return response === undefined
       ? Promise.reject(new Error(`Unexpected fetch call to ${String(input)}`))
       : Promise.resolve(response);
@@ -105,7 +112,7 @@ const persistedToken = (
 
 const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (input: {
   readonly initialToken?: TokenStore.RemoteDpopAccessToken;
-  readonly responses: ReadonlyArray<Response>;
+  readonly responses: ReadonlyArray<Response | typeof STALLED>;
   readonly bootstrap?: RelayEnvironmentConnectResponse;
   readonly beforeBootstrap?: Effect.Effect<void, ManagedRelay.ManagedRelayClientError>;
   readonly beforePut?: Effect.Effect<void>;
@@ -134,7 +141,8 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
       readonly accessToken?: string;
     }>
   >([]);
-  const fetch = recordedFetch(input.responses);
+  const stalls = yield* Queue.unbounded<void>();
+  const fetch = recordedFetch(input.responses, () => Queue.offerUnsafe(stalls, undefined));
 
   const tokenStore = TokenStore.RemoteDpopAccessTokenStore.of({
     get: (environmentId) =>
@@ -224,6 +232,7 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
     fetch,
     relayInputs,
     session,
+    stalls,
     thumbprint,
     tokenReads,
   };
@@ -440,7 +449,7 @@ describe("RemoteEnvironmentAuthorization", () => {
     }),
   );
 
-  it.effect("refreshes a cached endpoint after its first transient failure", () =>
+  it.effect("refreshes a cached endpoint after a server error", () =>
     Effect.gen(function* () {
       const cached = new TokenStore.RemoteDpopAccessToken({
         environmentId: ENVIRONMENT_ID,
@@ -479,6 +488,84 @@ describe("RemoteEnvironmentAuthorization", () => {
     }),
   );
 
+  // connection/supervisor.test.ts covers the limit: the third timeout in a row asks the relay.
+  it.effect("resets the cached ticket timeout count when a retry succeeds", () =>
+    Effect.gen(function* () {
+      // An attempt sends the cached ticket request and, after it times out, one retry.
+      const timedOutAttempt = [STALLED, STALLED] as const;
+      const recoveredAttempt = [STALLED, websocketTicket("cached-ticket")] as const;
+      const harness = yield* makeHarness({
+        initialToken: persistedToken(),
+        responses: [
+          ...timedOutAttempt,
+          ...timedOutAttempt,
+          ...recoveredAttempt,
+          ...timedOutAttempt,
+        ],
+      });
+
+      const [timeouts, recovered] = yield* Effect.gen(function* () {
+        const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
+        const timeOut = Effect.gen(function* () {
+          const pending = yield* remote
+            .authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID })
+            .pipe(Effect.flip, Effect.forkChild);
+          yield* Queue.take(harness.stalls);
+          yield* TestClock.adjust("3 seconds");
+          yield* Queue.take(harness.stalls);
+          yield* TestClock.adjust("7 seconds");
+          return yield* Fiber.join(pending);
+        });
+        const recover = Effect.gen(function* () {
+          const pending = yield* remote
+            .authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID })
+            .pipe(Effect.forkChild);
+          yield* Queue.take(harness.stalls);
+          yield* TestClock.adjust("3 seconds");
+          return yield* Fiber.join(pending);
+        });
+        const timeouts = [yield* timeOut, yield* timeOut];
+        const recovered = yield* recover;
+        // Without the reset, this would be the third timeout in a row and ask the relay.
+        timeouts.push(yield* timeOut);
+        return [timeouts, recovered] as const;
+      }).pipe(Effect.provide(harness.layer));
+
+      for (const failure of timeouts) {
+        expect(failure).toMatchObject({ _tag: "ConnectionTransientError", reason: "timeout" });
+      }
+      expect(recovered.socketUrl).toContain("wsTicket=cached-ticket");
+      expect(yield* Ref.get(harness.bootstrapCalls)).toBe(0);
+    }),
+  );
+
+  it.effect("keeps a newly refreshed token when its websocket ticket request times out", () =>
+    Effect.gen(function* () {
+      const harness = yield* makeHarness({
+        responses: [
+          Response.json(DESCRIPTOR),
+          accessToken("fresh-access-token"),
+          STALLED,
+          websocketTicket("second-ticket"),
+        ],
+      });
+
+      const [failure, authorized] = yield* Effect.gen(function* () {
+        const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
+        const authorize = () => remote.authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID });
+        const pending = yield* authorize().pipe(Effect.flip, Effect.forkChild);
+        yield* Queue.take(harness.stalls);
+        yield* TestClock.adjust("10 seconds");
+        return [yield* Fiber.join(pending), yield* authorize()] as const;
+      }).pipe(Effect.provide(harness.layer));
+
+      expect(failure).toMatchObject({ _tag: "ConnectionTransientError", reason: "timeout" });
+      expect(authorized.socketUrl).toContain("wsTicket=second-ticket");
+      expect(authorized.httpAuthorization).toMatchObject({ accessToken: "fresh-access-token" });
+      expect(yield* Ref.get(harness.bootstrapCalls)).toBe(1);
+    }),
+  );
+
   it.effect("removes a newly refreshed token when websocket ticket authorization fails", () =>
     Effect.gen(function* () {
       const harness = yield* makeHarness({

packages/client-runtime/src/authorization/service.ts

@@ -80,6 +80,11 @@ export class RemoteEnvironmentAuthorization extends Context.Service<
 >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}
 
 const CACHED_ENDPOINT_SOCKET_TIMEOUT_MS = 3_000;
+// 3 s + 7 s ends the cached ticket step by 10 s, the default request budget. That leaves
+// 5 s of the supervisor's 15 s setup deadline for the descriptor check and websocket open.
+const CACHED_ENDPOINT_SOCKET_RETRY_TIMEOUT_MS = 7_000;
+/** Cached ticket attempts in a row that time out before the client asks the relay again. */
+const CACHED_TICKET_TIMEOUT_LIMIT = 3;
 const BEARER_DESCRIPTOR_CACHE_TTL_MS = 10_000;
 const DPOP_AUTHORIZATION_TIMEOUT_MS = 30_000;
 
@@ -112,6 +117,12 @@ export const make = Effect.gen(function* () {
     EnvironmentId,
     { readonly accessToken: string; readonly identity: ClientCapabilities.CloudSessionIdentity }
   >();
+  // Timed-out cached ticket steps in a row, per environment and token. It lives as long as
+  // the service, so supervisor retries and a mobile foreground resume do not reset it.
+  const cachedTicketTimeouts = new Map<
+    EnvironmentId,
+    { readonly accessToken: string; readonly count: number }
+  >();
   const pendingTokens = new Map<
     EnvironmentId,
     {
@@ -465,27 +476,59 @@ export const make = Effect.gen(function* () {
   ) {
     let selected = yield* getDpopToken(input);
     if (selected.fromCache) {
+      const cachedToken = selected.token;
+      // A timeout means a lost request or a slow server, not a bad token. Retry the same
+      // token once, so a lost request recovers in about 3 s without a new credential.
       const cachedSocket = yield* createDpopSocketUrl(
-        selected.token,
+        cachedToken,
         CACHED_ENDPOINT_SOCKET_TIMEOUT_MS,
-      ).pipe(Effect.result);
+      ).pipe(
+        Effect.catchTags({
+          RemoteEnvironmentAuthTimeoutError: () =>
+            createDpopSocketUrl(cachedToken, CACHED_ENDPOINT_SOCKET_RETRY_TIMEOUT_MS),
+        }),
+        Effect.result,
+      );
       if (Result.isSuccess(cachedSocket)) {
+        cachedTicketTimeouts.delete(input.expectedEnvironmentId);
         yield* assertSession(selected.identity);
-        return { ...httpAuthorization(selected.token), socketUrl: cachedSocket.success };
+        return { ...httpAuthorization(cachedToken), socketUrl: cachedSocket.success };
       }
       if (cachedSocket.failure._tag === "ConnectionBlockedError") {
         return yield* mapDpopSocketError(cachedSocket.failure);
       }
+      // When the retry also times out, keep the token and fail as transient. The relay returns
+      // the same hostname for an environment, so a new credential would only add an auth
+      // session on that slow server. Leave out the network hint: the relay and tunnel answered
+      // for this token before. At the limit, drop the token and ask the relay again in case
+      // the endpoint moved. That mint runs in the service scope, so it lands even if this
+      // attempt times out.
+      if (cachedSocket.failure._tag === "RemoteEnvironmentAuthTimeoutError") {
+        const accessToken = cachedToken.accessToken;
+        const previous = cachedTicketTimeouts.get(input.expectedEnvironmentId);
+        const count = previous?.accessToken === accessToken ? previous.count + 1 : 1;
+        if (count < CACHED_TICKET_TIMEOUT_LIMIT) {
+          cachedTicketTimeouts.set(input.expectedEnvironmentId, { accessToken, count });
+          return yield* new ConnectionTransientError({
+            reason: "timeout",
+            detail: `${cachedToken.label} did not respond during connection setup.`,
+          });
+        }
+        cachedTicketTimeouts.delete(input.expectedEnvironmentId);
+      }
       selected = yield* getDpopToken({
         ...input,
-        rejectedAccessToken: selected.token.accessToken,
+        rejectedAccessToken: cachedToken.accessToken,
       });
     }
     const socket = yield* createDpopSocketUrl(selected.token).pipe(Effect.result);
     if (Result.isFailure(socket)) {
-      yield* tokenLock.withPermits(1)(
-        removeRejectedToken(input.expectedEnvironmentId, selected.token.accessToken),
-      );
+      // A timeout does not mean the new token is bad. Keep it so the next attempt reuses it.
+      if (socket.failure._tag !== "RemoteEnvironmentAuthTimeoutError") {
+        yield* tokenLock.withPermits(1)(
+          removeRejectedToken(input.expectedEnvironmentId, selected.token.accessToken),
+        );
+      }
       return yield* mapDpopSocketError(socket.failure);
     }
     yield* assertSession(selected.identity);