feat(cli,webapp): enforce instance base images end to end

Reject initialize-deployment requests from CLIs that cannot apply the instance's base images, so the setting is enforced rather than advisory. The CLI declares support on the paths that can honour it, and fails with a clear error on --native-build and --local-bundle, which cannot.

Return the base images on the get-deployment response as well, so deploys that attach to an existing deployment build on them too.

Validate image refs with one shared schema in core on both the server and the CLI: a single token pinned by digest. Reject the runtime alias node in favour of the concrete runtime keys, and report every invalid env entry in one error at startup.

Build the custom build stage with the same customization block as the base stage, so instructions and package installs run in the same order.

Author: nicktrn

.changeset/instance-deploy-base-images.md

@@ -3,4 +3,4 @@
 "trigger.dev": patch
 ---
 
-Self-hosted instances can require custom base images for deploys, such as FIPS-validated or hardened Node images, with the new `DEPLOY_BASE_IMAGES` webapp setting. The CLI builds on the base images the instance specifies.
+Self-hosted instances can require custom deploy base images per runtime via the new `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES` webapp settings. The CLI builds on the images the instance specifies, and older CLIs are rejected with an upgrade message.

apps/webapp/app/env.server.ts

@@ -15,6 +15,18 @@ function durationString() {
   return z.string().refine(isValidDuration, "must be a duration like 7d, 30d, 365d, 1h, 1y");
 }
 
+const parseDeployBaseImagesEnv = (
+  value: string | undefined,
+  envVarName: string,
+  ctx: z.RefinementCtx
+) => {
+  const { images, errors } = parseDeployBaseImages(value, envVarName);
+  for (const message of errors) {
+    ctx.addIssue({ code: z.ZodIssueCode.custom, message });
+  }
+  return errors.length > 0 ? z.NEVER : images;
+};
+
 // Parses a CSV of machine preset names (e.g. "small-1x,small-2x") into a
 // non-empty array of MachinePresetName. Used by COMPUTE_TEMPLATE_MACHINE_PRESETS
 // and its _REQUIRED variant. Adds zod issues for empty input or unknown names.
@@ -905,11 +917,11 @@ const EnvironmentSchema = z
     DEPLOY_BASE_IMAGES: z
       .string()
       .optional()
-      .transform((v) => parseDeployBaseImages(v, "DEPLOY_BASE_IMAGES")),
+      .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BASE_IMAGES", ctx)),
     DEPLOY_BUILD_BASE_IMAGES: z
       .string()
       .optional()
-      .transform((v) => parseDeployBaseImages(v, "DEPLOY_BUILD_BASE_IMAGES")),
+      .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BUILD_BASE_IMAGES", ctx)),
     DEPLOY_TIMEOUT_MS: z.coerce
       .number()
       .int()

apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts

@@ -2,6 +2,8 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
 import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
 import { authenticateApiKeyWithScope } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 
@@ -65,6 +67,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
       externalId: deployment.externalId ?? undefined,
       externalBuildData:
         deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"],
+      baseImages: resolveDeployBaseImages(deployment.runtime, {
+        base: env.DEPLOY_BASE_IMAGES,
+        buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
+      }),
       errorData: deployment.errorData as GetDeploymentResponseBody["errorData"],
       canceledReason: deployment.canceledReason,
       worker: deployment.worker

apps/webapp/app/v3/deployBaseImages.server.ts

@@ -1,58 +1,66 @@
-import { BuildRuntime } from "@trigger.dev/core/v3";
+import { BuildRuntime, DeployBaseImageRef } from "@trigger.dev/core/v3";
 
 type BaseImageMap = Partial<Record<BuildRuntime, string>>;
 
-const DIGEST_PINNED = /@sha256:[a-f0-9]{64}$/;
+export type ParsedDeployBaseImages = { images: BaseImageMap; errors: string[] };
 
-function invalidSegment(envVarName: string, segment: string, reason: string): Error {
-  return new Error(`${envVarName}: ${reason} in "${segment}"`);
-}
-
-export function parseDeployBaseImages(value: string | undefined, envVarName: string): BaseImageMap {
-  const result: BaseImageMap = {};
+export function parseDeployBaseImages(
+  value: string | undefined,
+  envVarName: string
+): ParsedDeployBaseImages {
+  const images: BaseImageMap = {};
+  const errors: string[] = [];
 
   if (!value) {
-    return result;
+    return { images, errors };
   }
 
   for (const segment of value.split(",").map((s) => s.trim())) {
     if (!segment) {
       continue;
     }
 
+    const fail = (reason: string) => errors.push(`${envVarName}: ${reason} in "${segment}"`);
+
     const separator = segment.indexOf("=");
     if (separator === -1) {
-      throw invalidSegment(envVarName, segment, "expected runtime=image");
+      fail("expected runtime=image");
+      continue;
     }
 
     const runtimeName = segment.slice(0, separator).trim();
     const image = segment.slice(separator + 1).trim();
 
+    if (runtimeName === "node") {
+      fail('runtime "node" is an alias; use the concrete runtime (node-22, node-24, node-26)');
+      continue;
+    }
+
     const runtime = BuildRuntime.safeParse(runtimeName);
     if (!runtime.success) {
-      throw invalidSegment(
-        envVarName,
-        segment,
-        `unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})`
-      );
+      fail(`unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})`);
+      continue;
     }
 
     if (!image) {
-      throw invalidSegment(envVarName, segment, "missing image");
+      fail("missing image");
+      continue;
     }
 
-    if (!DIGEST_PINNED.test(image)) {
-      throw invalidSegment(envVarName, segment, "image must be pinned by digest (@sha256:<64 hex chars>)");
+    if (!DeployBaseImageRef.safeParse(image).success) {
+      fail("image must be image@sha256:<64 hex chars> with no whitespace before the digest");
+      continue;
     }
 
-    if (runtime.data in result) {
-      throw invalidSegment(envVarName, segment, `duplicate runtime "${runtimeName}"`);
+    if (runtime.data in images) {
+      fail(`duplicate runtime "${runtimeName}"`);
+      continue;
     }
 
-    result[runtime.data] = image;
+    images[runtime.data] = image;
   }
 
-  return result;
+  return { images, errors };
 }
 
 export function resolveDeployBaseImages(

apps/webapp/app/v3/services/initializeDeployment.server.ts

@@ -14,6 +14,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { resolveDeployBaseImages } from "../deployBaseImages.server";
 import { getDeploymentImageRef } from "../getDeploymentImageRef.server";
 import { tryCatch } from "@trigger.dev/core";
 import { getRegistryConfig } from "../registryConfig.server";
@@ -147,6 +148,19 @@ export class InitializeDeploymentService extends BaseService {
         throw new ServiceValidationError("UNMANAGED deployments are not supported");
       }
 
+      if (
+        resolveDeployBaseImages(runtime, {
+          base: env.DEPLOY_BASE_IMAGES,
+          buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
+        }) &&
+        payload.supportsInstanceBaseImages !== true
+      ) {
+        throw new ServiceValidationError(
+          "This instance requires custom deploy base images, which this version of the CLI cannot apply. Upgrade the trigger.dev CLI and deploy again.",
+          400
+        );
+      }
+
       // Upgrade the project to engine "V2" if it's not already. This should cover cases where people deploy to V2 without running dev first.
       if (payload.type === "MANAGED" && environment.project.engine === "V1") {
         await this._prisma.project.update({

apps/webapp/test/deployBaseImages.test.ts

@@ -5,11 +5,14 @@ const digestA = `sha256:${"a".repeat(64)}`;
 const digestB = `sha256:${"b".repeat(64)}`;
 const digestC = `sha256:${"c".repeat(64)}`;
 
+const hex64 = "a".repeat(64);
+
 describe("parseDeployBaseImages", () => {
   it("returns an empty map for undefined and empty values", () => {
-    expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual({});
-    expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual({});
-    expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual({});
+    const empty = { images: {}, errors: [] };
+    expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual(empty);
+    expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual(empty);
+    expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual(empty);
   });
 
   it("parses multiple entries and trims whitespace", () => {
@@ -19,31 +22,61 @@ describe("parseDeployBaseImages", () => {
         "DEPLOY_BASE_IMAGES"
       )
     ).toEqual({
-      "node-24": `acme/node-fips:24@${digestA}`,
-      bun: `acme/bun:1@${digestB}`,
+      images: {
+        "node-24": `acme/node-fips:24@${digestA}`,
+        bun: `acme/bun:1@${digestB}`,
+      },
+      errors: [],
+    });
+  });
+
+  it("accepts a registry with a port and a tag before the digest", () => {
+    const image = `registry.example.com:5000/ns/img:tag@sha256:${hex64}`;
+    expect(parseDeployBaseImages(`node-24=${image}`, "DEPLOY_BASE_IMAGES")).toEqual({
+      images: { "node-24": image },
+      errors: [],
     });
   });
 
   it.each([
     ["missing =", "garbage"],
     ["unknown runtime", `node-23=acme/node:23@${digestA}`],
+    ["node alias", `node=acme/node:24@${digestA}`],
     ["empty image", "node-24="],
     ["missing digest", "node-24=acme/node:24"],
+    ["flag before image", `node-24=--platform=linux/arm64 acme/node@sha256:${hex64}`],
+    ["bare digest", `node-24=@sha256:${hex64}`],
+    ["newline in image", `node-24=acme/node\nx@sha256:${hex64}`],
     ["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`],
-  ])("throws naming the env var and segment: %s", (_name, value) => {
+  ])("reports an error naming the env var and segment: %s", (_name, value) => {
     const segments = value.split(",");
-    const offending = segments[segments.length - 1]!;
-
-    let error: Error | undefined;
-    try {
-      parseDeployBaseImages(`node-22=acme/ok@${digestC},${value}`, "DEPLOY_BUILD_BASE_IMAGES");
-    } catch (e) {
-      error = e as Error;
-    }
-
-    expect(error).toBeInstanceOf(Error);
-    expect(error?.message).toContain("DEPLOY_BUILD_BASE_IMAGES");
-    expect(error?.message).toContain(offending);
+    const offending = segments[segments.length - 1]!.trim();
+
+    const { images, errors } = parseDeployBaseImages(
+      `node-22=acme/ok@${digestC},${value}`,
+      "DEPLOY_BUILD_BASE_IMAGES"
+    );
+
+    expect(images["node-22"]).toBe(`acme/ok@${digestC}`);
+    expect(errors).toHaveLength(1);
+    expect(errors[0]).toContain("DEPLOY_BUILD_BASE_IMAGES");
+    expect(errors[0]).toContain(offending);
+  });
+
+  it("explains that node is an alias", () => {
+    const { errors } = parseDeployBaseImages(`node=acme/node@${digestA}`, "DEPLOY_BASE_IMAGES");
+    expect(errors[0]).toContain('runtime "node" is an alias; use the concrete runtime');
+  });
+
+  it("reports every bad segment", () => {
+    const { errors } = parseDeployBaseImages(
+      `garbage,node-23=acme/node@${digestA},bun=acme/bun`,
+      "DEPLOY_BASE_IMAGES"
+    );
+    expect(errors).toHaveLength(3);
+    expect(errors[0]).toContain("garbage");
+    expect(errors[1]).toContain("node-23");
+    expect(errors[2]).toContain("bun=acme/bun");
   });
 });
 

docs/self-hosting/env/webapp.mdx

@@ -97,7 +97,7 @@ mode: "wide"
 | `DEPLOY_REGISTRY_NAMESPACE`                      | No       | trigger               | Deploy registry namespace.                                                                                         |
 | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY`  | No       | —                     | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. |
 | `DEPLOY_IMAGE_PLATFORM`                          | No       | linux/amd64           | Deploy image platform, same values as docker `--platform` flag.                                                    |
-| `DEPLOY_BASE_IMAGES`                             | No       | —                     | Base images every deploy must build on. Comma-separated `runtime=image@sha256:<digest>` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node`, `node-22`, `node-24`, `node-26`, `bun`. The digest is required. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). |
+| `DEPLOY_BASE_IMAGES`                             | No       | —                     | Base images every deploy must build on. Comma-separated `runtime=image@sha256:<digest>` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node-22`, `node-24`, `node-26`, `bun`. The digest is required. Projects with `runtime: "node"` resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an upgrade message, and deploys using `--native-build` or `--local-bundle` fail when base images are configured. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). |
 | `DEPLOY_BUILD_BASE_IMAGES`                       | No       | —                     | Build-stage images per runtime. Same format and validation as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. |
 | `DEPLOY_TIMEOUT_MS`                              | No       | 480000 (8m)           | Deploy timeout (ms).                                                                                               |
 | `DEPLOY_QUEUE_TIMEOUT_MS`                        | No       | 900000 (15m)          | Deploy queue timeout (ms).                                                                                         |

docs/self-hosting/overview.mdx

@@ -108,7 +108,9 @@ Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debi
 DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>"
 ```
 
-Entries are comma-separated `runtime=image@sha256:<digest>`. The runtimes are `node`, `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`.
+Entries are comma-separated `runtime=image@sha256:<digest>`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`.
+
+Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade. Deploys using `--native-build` or `--local-bundle` fail with an error when base images are configured, since those paths cannot apply them.
 
 The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the bundle's Containerfile with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud.
 

packages/cli-v3/src/commands/deploy.ts

@@ -591,6 +591,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
       triggeredVia: getTriggeredVia(),
       externalId: options.externalId,
       force: options.force,
+      supportsInstanceBaseImages: true,
     },
     envVars.TRIGGER_EXISTING_DEPLOYMENT_ID
   );
@@ -1497,6 +1498,13 @@ async function handleNativeBuildServerDeploy({
 
   const deployment = initializeDeploymentResult.data;
 
+  if (deployment.baseImages) {
+    $deploymentSpinner.stop("Failed to initialize deployment");
+    throw new Error(
+      "This instance requires custom deploy base images, which cannot be applied with --native-build. Deploy without that flag."
+    );
+  }
+
   const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`;
   const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${
     options.env === "prod" ? "prod" : "stg"
@@ -1830,6 +1838,13 @@ async function handleLocalBundleDeploy({
 
   const deployment = initializeDeploymentResult.data;
 
+  if (deployment.baseImages) {
+    $deploymentSpinner.stop("Failed to initialize deployment");
+    throw new Error(
+      "This instance requires custom deploy base images, which cannot be applied with --local-bundle. Deploy without that flag."
+    );
+  }
+
   const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`;
   const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${
     options.env === "prod" ? "prod" : "stg"
@@ -2304,6 +2319,7 @@ async function handleFromBundleDeploy({
       isLocalBuild: true,
       isNativeBuild: false,
       triggeredVia: getTriggeredVia(),
+      supportsInstanceBaseImages: true,
     },
     existingDeploymentId
   );

packages/cli-v3/src/deploy/buildImage.test.ts

@@ -289,12 +289,27 @@ describe("generateContainerfile", () => {
     expect(containerfile).toContain("FROM acme/node:26-dev@sha256:def AS build");
     expect(containerfile).not.toContain("FROM base AS build");
     expect(containerfile).not.toContain(TOOLCHAIN_PACKAGES);
-    expect(buildStage).toContain("apt-get install -y --no-install-recommends --allow-downgrades jq");
+    expect(buildStage).toContain(
+      "apt-get install -y --no-install-recommends --allow-downgrades jq"
+    );
     expect(buildStage).toContain("RUN echo first > /etc/first");
     expect(buildStage).toContain("RUN echo second > /etc/second");
     expect(containerfile.indexOf("RUN echo first > /etc/first")).toBeLessThan(
       containerfile.indexOf("AS build")
     );
+
+    const buildFrom = "FROM acme/node:26-dev@sha256:def AS build";
+    const baseEnv = "ENV DEBIAN_FRONTEND=noninteractive\n\n";
+    const baseStart = containerfile.indexOf(baseEnv) + baseEnv.length;
+    const baseCustomization = containerfile.slice(
+      baseStart,
+      containerfile.indexOf(buildFrom) - "\n\n".length
+    );
+
+    expect(buildStage.indexOf("RUN echo second > /etc/second")).toBeLessThan(
+      buildStage.indexOf("apt-get install")
+    );
+    expect(containerfile).toContain(`${buildFrom}\n\n${baseEnv}${baseCustomization}\n\n`);
   });
 
   it("builds from the base stage when instructions have no configured build image", async () => {