fix(core): time out API requests so a dead socket can't hang a call

API client requests had no timeout, so a request reusing a half-open keep-alive connection could hang until the runtime's socket default (minutes) rather than failing fast, and the retry logic never engaged because a hang produces no response to react to. Requests now run under a default 30s timeout (configurable per request via timeoutInMs); on timeout the request is aborted and the retry reconnects on a fresh socket. Caller cancellations are not retried.

Author: ericallam

.changeset/api-client-request-timeout.md

@@ -0,0 +1,13 @@
+---
+"@trigger.dev/core": patch
+---
+
+API requests now run under a request timeout (default 30s), so a half-open keep-alive connection can no longer hang a call until the runtime's socket default (minutes). On timeout the request aborts and retries on a fresh connection, and the retry is duplicate-safe via the request idempotency key. This mostly affects long-lived processes that reuse connections, including tasks that trigger other tasks.
+
+Set the timeout per request, per client, or globally (most specific wins, `0` disables):
+
+```ts
+tasks.trigger("my-task", payload, undefined, { timeoutInMs: 10_000 }); // per request
+new TriggerClient({ accessToken, requestOptions: { timeoutInMs: 10_000 } }); // per client
+// or globally: TRIGGER_API_REQUEST_TIMEOUT_MS=10000
+```

packages/core/src/v3/apiClient/core.test.ts

@@ -0,0 +1,131 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { z } from "zod";
+import { zodfetch } from "./core.js";
+
+vi.setConfig({ testTimeout: 5_000 });
+
+const schema = z.object({ ok: z.boolean() });
+
+describe("zodfetch request timeout", () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    vi.restoreAllMocks();
+    delete process.env.TRIGGER_API_REQUEST_TIMEOUT_MS;
+  });
+
+  // Never settles unless the request is aborted (models a half-open keep-alive socket).
+  function neverResponds(init?: RequestInit): Promise<Response> {
+    return new Promise<Response>((_resolve, reject) => {
+      init?.signal?.addEventListener("abort", () =>
+        reject(init.signal!.reason ?? new DOMException("aborted", "AbortError"))
+      );
+    });
+  }
+
+  it("aborts a hung request after the timeout instead of hanging forever", async () => {
+    globalThis.fetch = vi.fn().mockImplementation((_url: string, init?: RequestInit) =>
+      neverResponds(init)
+    );
+
+    const start = Date.now();
+    await expect(
+      zodfetch(schema, "http://localhost/x", { method: "POST" }, {
+        timeoutInMs: 150,
+        retry: { maxAttempts: 1 },
+      })
+    ).rejects.toThrow();
+    expect(Date.now() - start).toBeLessThan(2_000);
+  });
+
+  it("retries on a fresh connection after a timeout and recovers", async () => {
+    let calls = 0;
+    globalThis.fetch = vi.fn().mockImplementation((_url: string, init?: RequestInit) => {
+      calls++;
+      // First connection hangs; the retry's fresh connection responds.
+      if (calls === 1) {
+        return neverResponds(init);
+      }
+      return Promise.resolve(
+        new Response(JSON.stringify({ ok: true }), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        })
+      );
+    });
+
+    const result = await zodfetch(schema, "http://localhost/x", { method: "POST" }, {
+      timeoutInMs: 150,
+      retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 50, factor: 1, randomize: false },
+    });
+
+    expect(result).toEqual({ ok: true });
+    expect(calls).toBe(2);
+  });
+
+  it("reuses the same request idempotency key across a timeout retry so the server can dedupe", async () => {
+    const keys: Array<string | null> = [];
+    let calls = 0;
+    globalThis.fetch = vi.fn().mockImplementation((_url: string, init?: RequestInit) => {
+      calls++;
+      keys.push(new Headers(init?.headers).get("x-trigger-request-idempotency-key"));
+      if (calls === 1) {
+        return neverResponds(init);
+      }
+      return Promise.resolve(
+        new Response(JSON.stringify({ ok: true }), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        })
+      );
+    });
+
+    await zodfetch(schema, "http://localhost/x", { method: "POST" }, {
+      timeoutInMs: 150,
+      retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 50, factor: 1, randomize: false },
+    });
+
+    expect(keys).toHaveLength(2);
+    expect(keys[0]).toBeTruthy();
+    expect(keys[0]).toBe(keys[1]);
+  });
+
+  it("does not time out when timeoutInMs is 0 (for long-lived requests)", async () => {
+    globalThis.fetch = vi.fn().mockImplementation(
+      () =>
+        new Promise<Response>((resolve) =>
+          setTimeout(
+            () =>
+              resolve(
+                new Response(JSON.stringify({ ok: true }), {
+                  status: 200,
+                  headers: { "content-type": "application/json" },
+                })
+              ),
+            300
+          )
+        )
+    );
+
+    const result = await zodfetch(schema, "http://localhost/x", { method: "POST" }, {
+      timeoutInMs: 0,
+      retry: { maxAttempts: 1 },
+    });
+
+    expect(result).toEqual({ ok: true });
+  });
+
+  it("uses TRIGGER_API_REQUEST_TIMEOUT_MS as the default when no timeout is passed", async () => {
+    process.env.TRIGGER_API_REQUEST_TIMEOUT_MS = "120";
+    globalThis.fetch = vi.fn().mockImplementation((_url: string, init?: RequestInit) =>
+      neverResponds(init)
+    );
+
+    const start = Date.now();
+    await expect(
+      zodfetch(schema, "http://localhost/x", { method: "POST" }, { retry: { maxAttempts: 1 } })
+    ).rejects.toThrow();
+    expect(Date.now() - start).toBeLessThan(2_000);
+  });
+});

packages/core/src/v3/apiClient/core.ts

@@ -19,6 +19,7 @@ import {
 } from "./pagination.js";
 import { EventSource, type ErrorEvent } from "eventsource";
 import { randomUUID } from "../utils/crypto.js";
+import { getNumberEnvVar } from "../utils/getEnv.js";
 
 export const defaultRetryOptions = {
   maxAttempts: 3,
@@ -28,8 +29,11 @@ export const defaultRetryOptions = {
   randomize: false,
 } satisfies RetryOptions;
 
+export const defaultRequestTimeoutInMs = 30_000;
+
 export type ZodFetchOptions<TData = any> = {
   retry?: RetryOptions;
+  timeoutInMs?: number;
   tracer?: TriggerTracer;
   name?: string;
   attributes?: Attributes;
@@ -40,7 +44,7 @@ export type ZodFetchOptions<TData = any> = {
 
 export type AnyZodFetchOptions = ZodFetchOptions<any>;
 
-export type ApiRequestOptions = Pick<ZodFetchOptions, "retry"> & {
+export type ApiRequestOptions = Pick<ZodFetchOptions, "retry" | "timeoutInMs"> & {
   additionalHeaders?: Record<string, string>;
 };
 
@@ -51,6 +55,7 @@ type KeysEnum<T> = { [P in keyof Required<T>]: true };
 // compiler such that any missing / extraneous keys will cause an error.
 const requestOptionsKeys: KeysEnum<ApiRequestOptions> = {
   retry: true,
+  timeoutInMs: true,
   additionalHeaders: true,
 };
 
@@ -230,9 +235,28 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
   options?: ZodFetchOptions,
   attempt = 1
 ): Promise<ZodFetchResult<z.output<TResponseBodySchema>>> {
+  // Precedence: per-request option > per-client requestOptions > env var > built-in default.
+  const timeoutInMs =
+    options?.timeoutInMs ?? getNumberEnvVar("TRIGGER_API_REQUEST_TIMEOUT_MS") ?? defaultRequestTimeoutInMs;
+  const controller = new AbortController();
+  const callerSignal = requestInit?.signal ?? undefined;
+  const onCallerAbort = () => controller.abort(callerSignal?.reason);
+  if (callerSignal) {
+    if (callerSignal.aborted) controller.abort(callerSignal.reason);
+    else callerSignal.addEventListener("abort", onCallerAbort, { once: true });
+  }
+  // timeoutInMs <= 0 disables the timeout, for long-lived requests that set their own bound.
+  const timeout =
+    timeoutInMs > 0
+      ? setTimeout(
+          () => controller.abort(new Error(`Request to ${url} timed out after ${timeoutInMs}ms`)),
+          timeoutInMs
+        )
+      : undefined;
+
   try {
     const response = await context.with(suppressTracing(context.active()), () =>
-      fetch(url, requestInitWithCache(requestInit))
+      fetch(url, { ...requestInitWithCache(requestInit), signal: controller.signal })
     );
 
     const responseHeaders = createResponseHeaders(response.headers);
@@ -277,6 +301,11 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
     if (error instanceof ValidationError) {
     }
 
+    // A caller-supplied signal aborting is a cancellation, not a transient failure.
+    if (callerSignal?.aborted) {
+      throw error;
+    }
+
     if (options?.retry) {
       const retry = { ...defaultRetryOptions, ...options.retry };
 
@@ -290,6 +319,9 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
     }
 
     throw new ApiConnectionError({ cause: castToError(error) });
+  } finally {
+    clearTimeout(timeout);
+    callerSignal?.removeEventListener("abort", onCallerAbort);
   }
 }