feat(container): first-class BuildKit secrets support

First-class BuildKit secrets support for BuildKit in-cluster building, Garden Cloud Builder and building locally. Kaniko is not supported due to lack of support (See also GoogleContainerTools/kaniko#3028)
garden-io · Jul 16, 2024 · 5a8fded · 5a8fded
1 parent 52e919a
commit 5a8fded
Show file tree

Hide file tree

Showing 15 changed files with 383 additions and 39 deletions.
diff --git a/core/src/plugins/container/build.ts b/core/src/plugins/container/build.ts
@@ -11,7 +11,7 @@ import { ConfigurationError, toGardenError } from "../../exceptions.js"
 import type { PrimitiveMap } from "../../config/common.js"
 import split2 from "split2"
 import type { BuildActionHandler } from "../../plugin/action-types.js"
-import type { ContainerBuildAction, ContainerBuildOutputs } from "./config.js"
+import type { ContainerBuildAction, ContainerBuildActionSpec, ContainerBuildOutputs } from "./config.js"
 import { defaultDockerfileName } from "./config.js"
 import { joinWithPosix } from "../../util/fs.js"
 import type { Resolved } from "../../actions/types.js"
@@ -153,6 +153,9 @@ async function buildContainerLocally({
 
   const dockerFlags = [...getDockerBuildFlags(action, ctx.provider.config), ...extraDockerOpts]
 
+  const { secretArgs, secretEnvVars } = getDockerSecrets(action.getSpec())
+  dockerFlags.push(...secretArgs)
+
   // If there already is a --tag flag, another plugin like the Kubernetes plugin already decided how to tag the image.
   // In this case, we don't want to add another local tag.
   // TODO: it would be nice to find a better way to become aware of the parent plugin's concerns in the container plugin.
@@ -175,6 +178,7 @@ async function buildContainerLocally({
       stderr: outputStream,
       timeout,
       ctx,
+      env: secretEnvVars,
     })
   } catch (e) {
     const error = toGardenError(e)
@@ -262,6 +266,38 @@ export function getContainerBuildActionOutputs(action: Resolved<ContainerBuildAc
   return containerHelpers.getBuildActionOutputs(action, undefined)
 }
 
+export function getDockerSecrets(actionSpec: ContainerBuildActionSpec): {
+  secretArgs: string[]
+  secretEnvVars: Record<string, string>
+} {
+  const args: string[] = []
+  const env: Record<string, string> = {}
+
+  for (const [secretKey, secretValue] of Object.entries(actionSpec.secrets || {})) {
+    if (!secretKey.match(/^[a-zA-Z0-9\._-]+$/)) {
+      throw new ConfigurationError({
+        message: `Invalid secret ID '${secretKey}'. Only alphanumeric characters (a-z, A-Z, 0-9), underscores (_), dashes (-) and dots (.) are allowed.`,
+      })
+    }
+
+    // determine env var names. There can be name collisions due to the fact that we replace special characters with underscores.
+    let envVarname: string
+    let i = 0
+    do {
+      envVarname = `GARDEN_BUILD_SECRET_${secretKey.toUpperCase().replaceAll(/[-\.]/, "_")}${i > 0 ? `_${i}` : ""}`
+      i += 1
+    } while (env[envVarname] !== undefined)
+
+    env[envVarname] = secretValue
+    args.push("--secret", `id=${secretKey},env=${envVarname}`)
+  }
+
+  return {
+    secretArgs: args,
+    secretEnvVars: env,
+  }
+}
+
 export function getDockerBuildFlags(
   action: Resolved<ContainerBuildAction>,
   containerProviderConfig: ContainerProviderConfig

diff --git a/core/src/plugins/container/config.ts b/core/src/plugins/container/config.ts
@@ -1013,6 +1013,7 @@ export interface ContainerBuildActionSpec {
   buildArgs: PrimitiveMap
   dockerfile: string
   extraFlags: string[]
+  secrets?: Record<string, string>
   localId?: string
   publishId?: string
   targetStage?: string
@@ -1061,9 +1062,30 @@ export const containerCommonBuildSpecKeys = memoize(() => ({
     Specify extra flags to use when building the container image.
     Note that arguments may not be portable across implementations.`),
   platforms: joi.sparseArray().items(joi.string()).description(dedent`
-    Specify the platforms to build the image for. This is useful when building multi-platform images.
-    The format is \`os/arch\`, e.g. \`linux/amd64\`, \`linux/arm64\`, etc.
-  `),
+      Specify the platforms to build the image for. This is useful when building multi-platform images.
+      The format is \`os/arch\`, e.g. \`linux/amd64\`, \`linux/arm64\`, etc.
+    `),
+  secrets: joi
+    .object()
+    .pattern(/.+/, joi.string())
+    .description(
+      dedent`
+      Specify secret values that can be mounted during the build process but become part of the resulting image filesystem or image manifest, for example private registry auth tokens.
+
+      Build arguments and environment variables are inappropriate for secrets, as they persist in the final image.
+
+      The secret can later be consumed in the Dockerfile like so:
+      \`\`\`
+        RUN --mount=type=secret,id=mytoken \
+            TOKEN=$(cat /run/secrets/mytoken) ...
+      \`\`\`
+
+      See also https://docs.docker.com/build/building/secrets/
+    `
+    )
+    .example({
+      mytoken: "supersecret",
+    }),
 }))
 
 export const containerBuildSpecSchema = createSchema({

diff --git a/core/src/plugins/container/helpers.ts b/core/src/plugins/container/helpers.ts
@@ -382,6 +382,7 @@ const helpers = {
     stdout,
     stderr,
     timeout,
+    env,
   }: {
     cwd: string
     args: string[]
@@ -391,14 +392,15 @@ const helpers = {
     stdout?: Writable
     stderr?: Writable
     timeout?: number
+    env?: { [key: string]: string }
   }) {
     const docker = ctx.tools["container.docker"]
 
     try {
       const res = await docker.spawnAndWait({
         args,
         cwd,
-        env: { ...process.env, DOCKER_CLI_EXPERIMENTAL: "enabled" },
+        env,
         ignoreError,
         log,
         stdout,

diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -36,7 +36,7 @@ import {
 import { getNamespaceStatus } from "../../namespace.js"
 import { sleep } from "../../../../util/util.js"
 import type { ContainerBuildAction, ContainerModuleOutputs } from "../../../container/moduleConfig.js"
-import { getDockerBuildArgs } from "../../../container/build.js"
+import { getDockerBuildArgs, getDockerSecrets } from "../../../container/build.js"
 import type { Resolved } from "../../../../actions/types.js"
 import { PodRunner } from "../../run.js"
 import { prepareSecrets } from "../../secrets.js"
@@ -280,6 +280,8 @@ export function makeBuildkitBuildCommand({
   contextPath: string
   dockerfile: string
 }): string[] {
+  const { secretArgs, secretEnvVars } = getDockerSecrets(action.getSpec())
+
   const buildctlCommand = [
     "buildctl",
     "build",
@@ -290,6 +292,7 @@ export function makeBuildkitBuildCommand({
     "dockerfile=" + contextPath,
     "--opt",
     "filename=" + dockerfile,
+    ...secretArgs,
     ...getBuildkitImageFlags(
       provider.config.clusterBuildkit!.cache,
       outputs,
@@ -298,7 +301,11 @@ export function makeBuildkitBuildCommand({
     ...getBuildkitFlags(action),
   ]
 
-  return ["sh", "-c", `cd ${contextPath} && ${commandListToShellScript(buildctlCommand)}`]
+  return [
+    "sh",
+    "-c",
+    `cd ${contextPath} && ${commandListToShellScript({ command: buildctlCommand, env: secretEnvVars })}`,
+  ]
 }
 
 export function getBuildkitFlags(action: Resolved<ContainerBuildAction>) {

diff --git a/core/src/plugins/kubernetes/container/build/kaniko.ts b/core/src/plugins/kubernetes/container/build/kaniko.ts
@@ -91,6 +91,18 @@ export const kanikoBuild: BuildHandler = async (params) => {
   const projectNamespace = (await getNamespaceStatus({ log, ctx: k8sCtx, provider })).namespaceName
 
   const spec = action.getSpec()
+
+  if (spec.secrets) {
+    throw new ConfigurationError({
+      message: dedent`
+        Unfortunately Kaniko does not support secret build arguments.
+        Garden Cloud Builder and the Kubernetes BuildKit in-cluster builder both support secrets.
+
+        See also https://github.com/GoogleContainerTools/kaniko/issues/3028
+      `,
+    })
+  }
+
   const outputs = k8sGetContainerBuildActionOutputs({ provider, action })
 
   const localId = outputs.localImageId
@@ -318,7 +330,7 @@ export function getKanikoBuilderPodManifest({
             n=0
             until [ "$n" -ge 30 ]
             do
-              rsync ${commandListToShellScript(syncArgs)} && break
+              rsync ${commandListToShellScript({ command: syncArgs })} && break
               n=$((n+1))
               sleep 1
             done
@@ -352,9 +364,9 @@ export function getKanikoBuilderPodManifest({
           "/bin/sh",
           "-c",
           dedent`
-            ${commandListToShellScript(kanikoCommand)};
+            ${commandListToShellScript({ command: kanikoCommand })};
             export exitcode=$?;
-            ${commandListToShellScript(["touch", `${sharedMountPath}/done`])};
+            ${commandListToShellScript({ command: ["touch", `${sharedMountPath}/done`] })};
             exit $exitcode;
           `,
         ],

diff --git a/core/src/plugins/kubernetes/jib-container.ts b/core/src/plugins/kubernetes/jib-container.ts
@@ -74,6 +74,7 @@ async function buildAndPushViaRemote(params: BuildActionParams<"build", Containe
 
   // Build the tarball with the base handler
   const spec: any = action.getSpec()
+
   spec.tarOnly = true
   spec.tarFormat = "oci"
 

diff --git a/core/src/plugins/kubernetes/run.ts b/core/src/plugins/kubernetes/run.ts
@@ -455,7 +455,7 @@ exec 2<&-
 exec 1<>/tmp/output
 exec 2>&1
 
-${commandListToShellScript(cmd)}
+${commandListToShellScript({ command: cmd })}
 `
 }
 
@@ -469,20 +469,20 @@ ${commandListToShellScript(cmd)}
  */
 function getArtifactsTarScript(artifacts: ArtifactSpec[]) {
   const directoriesToCreate = artifacts.map((a) => a.target).filter((target) => !!target && target !== ".")
-  const tmpPath = commandListToShellScript(["/tmp/.garden-artifacts-" + randomString(8)])
+  const tmpPath = commandListToShellScript({ command: ["/tmp/.garden-artifacts-" + randomString(8)] })
 
   const createDirectoriesCommands = directoriesToCreate.map((target) =>
-    commandListToShellScript(["mkdir", "-p", target])
+    commandListToShellScript({ command: ["mkdir", "-p", target] })
   )
 
   const copyArtifactsCommands = artifacts.map(({ source, target }) => {
-    const escapedTarget = commandListToShellScript([target || "."])
+    const escapedTarget = commandListToShellScript({ command: [target || "."] })
 
     // Allow globs (*) in the source path
     // Note: This works because `commandListToShellScript` wraps every parameter in single quotes, escaping contained single quotes.
     // The string `bin/*` will be transformed to `'bin/*'` by `commandListToShellScript`. The shell would treat `*` as literal and not expand it.
     // `replaceAll` transforms that string then to `'bin/'*''`, which allows the shell to expand the glob, everything else is treated as literal.
-    const escapedSource = commandListToShellScript([source]).replaceAll("*", "'*'")
+    const escapedSource = commandListToShellScript({ command: [source] }).replaceAll("*", "'*'")
 
     return `cp -r ${escapedSource} ${escapedTarget} >/dev/null || true`
   })

diff --git a/core/src/plugins/kubernetes/sync.ts b/core/src/plugins/kubernetes/sync.ts
@@ -464,7 +464,11 @@ export async function configureSyncMode({
       const initContainer = {
         name: "garden-dev-init",
         image: k8sSyncUtilImageName,
-        command: ["/bin/sh", "-c", commandListToShellScript(["cp", "/usr/local/bin/mutagen-agent", mutagenAgentPath])],
+        command: [
+          "/bin/sh",
+          "-c",
+          commandListToShellScript({ command: ["cp", "/usr/local/bin/mutagen-agent", mutagenAgentPath] }),
+        ],
         imagePullPolicy: "IfNotPresent",
         volumeMounts: [gardenVolumeMount],
       }

diff --git a/core/src/util/escape.ts b/core/src/util/escape.ts
@@ -6,47 +6,76 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import { InternalError } from "../exceptions.js"
+
 /**
  * Wraps every parameter in single quotes, escaping contained single quotes (for use in bash scripts). Joins the elements with a space character.
  *
  * Examples:
  *
  *   // returns `echo 'hello world'`
- *   commandListToShellScript(["echo", "hello world"])
+ *   commandListToShellScript({ command: ["echo", "hello world"] })
  *
  *   // returns `echo 'hello'"'"'world'`
- *   commandListToShellScript(["echo", "hello'world"])
+ *   commandListToShellScript({ command: ["echo", "hello'world"] })
  *
  *   // returns `echo ''"'"'; exec ls /'`
- *   commandListToShellScript(["echo", "'; exec ls /"])
+ *   commandListToShellScript({ command: ["echo", "'; exec ls /"] })
  *
  * Caveat: This is only safe if the command is directly executed. It is not safe, if you wrap the output of this in double quotes, for instance.
  *
  * // SAFE
- *    exec(["sh", "-c", ${commandListToShellScript(["some", "command", "--with" untrustedInput])}])
+ *    exec(["sh", "-c", ${commandListToShellScript({ command: ["some", "command", "--with" untrustedInput] })}])
  *    exec(["sh", "-c", dedent`
  *       set -e
  *       echo "running command..."
- *       ${commandListToShellScript(["some", "command", "--with" untrustedInput])}
+ *       ${commandListToShellScript({ command: ["some", "command", "--with" untrustedInput] })}
  *       echo "done"
  *    `])
  *
  * // UNSAFE! don't do this
  *
- *    const commandWithUntrustedInput = commandListToShellScript(["some", "command", "--with" untrustedInput])
- *    exec(["sh", "-c", `some_var="${commandWithUntrustedInput}"; echo "$some_var"`])
+ *    const UNSAFE_commandWithUntrustedInput = commandListToShellScript({ command: ["some", "UNSAFE", "command", "--with" untrustedInput] })
+ *    exec(["sh", "-c", `UNSAFE_some_var="${UNSAFE_commandWithUntrustedInput}"; echo "$UNSAFE_some_var"`])
  *
  * The second is UNSAFE, because we can't know that the /double quotes/ need to be escaped here.
  *
  * If you can, use environment variables instead of this, to pass untrusted values to shell scripts, e.g. if you do not need to construct a command with untrusted input.
  *
- * // SAFE
+ * // SAFE (preferred, if possible)
  *
  *    exec(["sh", "-c", `some_var="$UNTRUSTED_INPUT"; echo "$some_var"`], { env: { UNTRUSTED_INPUT: untrustedInput } })
  *
+ * // ALSO SAFE
+ *
+ *    exec([
+ *      "sh",
+ *      "-c",
+ *      commandListToShellScript({
+ *        command: ["some", "command", "--with" untrustedInput],
+ *        env: { UNTRUSTED_ENV_VAR: "moreUntrustedInput" },
+ *      }),
+ *    ])
+ *
  * @param command array of command line arguments
  * @returns string to be used as shell script statement to execute the given command.
  */
-export function commandListToShellScript(command: string[]) {
-  return command.map((c) => `'${c.replaceAll("'", `'"'"'`)}'`).join(" ")
+export function commandListToShellScript({ command, env }: { command: string[]; env?: Record<string, string> }) {
+  const wrapInSingleQuotes = (s: string) => `'${s.replaceAll("'", `'"'"'`)}'`
+
+  const escapedCommand = command.map(wrapInSingleQuotes).join(" ")
+
+  const escapedEnv =
+    env !== undefined
+      ? Object.entries(env).map(([k, v]) => {
+          if (!k.match(/^[0-9a-zA-Z_]$/)) {
+            throw new InternalError({
+              message: `Invalid environment variable name ${k}. Alphanumeric letters and underscores are allowed.`,
+            })
+          }
+          return `${k}=${wrapInSingleQuotes(v)}`
+        })
+      : undefined
+
+  return `${escapedEnv ? `${escapedEnv} ` : ""}${escapedCommand}`
 }