From 05a998065b3333854715c456b20b7cc5d5daac67 Mon Sep 17 00:00:00 2001 From: CaerusKaru Date: Mon, 11 Jan 2021 05:13:58 -0600 Subject: [PATCH] feat(cdk-assets): add external asset support (#12259) In the event that assets are not actually available at synthesis time, we still want to support JIT (just-in-time) asset generation via external tooling. This would, for instance, enable a third party tool to fetch additional resources prior to bundling/building and subsequent uploading. This adds a new interface for both File and Docker asset types that allows users to specify an executable. The executable, depending on the asset type, must then reply with a specific key on stdout, which will then get picked up and used by CDK Assets. This also updates the default stack synthesizer to support adding external sources directly. This is technically a breaking change for anyone who currently extends the class. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- allowed-breaking-changes.txt | 4 + .../lib/assets/docker-image-asset.ts | 18 +++- .../lib/assets/file-asset.ts | 15 ++- .../schema/assets.schema.json | 36 ++++--- .../schema/cloud-assembly.version.json | 2 +- .../cloud-assembly-schema/test/assets.test.ts | 46 ++++++++- packages/@aws-cdk/core/lib/assets.ts | 36 ++++++- .../stack-synthesizers/default-synthesizer.ts | 38 +++++++- .../core/lib/stack-synthesizers/legacy.ts | 8 ++ packages/cdk-assets/README.md | 35 +++++++ .../lib/private/handlers/container-images.ts | 95 +++++++++++++++---- .../cdk-assets/lib/private/handlers/files.ts | 65 +++++++++---- .../cdk-assets/lib/private/handlers/index.ts | 2 +- .../cdk-assets/test/docker-images.test.ts | 67 ++++++++++++- packages/cdk-assets/test/files.test.ts | 64 ++++++++++++- .../cdk-assets/test/mock-child_process.ts | 8 +- 16 files changed, 470 insertions(+), 69 deletions(-) diff --git a/allowed-breaking-changes.txt b/allowed-breaking-changes.txt index 9120903b01912..2ca2ca5b6067f 100644 --- a/allowed-breaking-changes.txt +++ b/allowed-breaking-changes.txt @@ -52,3 +52,7 @@ incompatible-argument:@aws-cdk/aws-ecs.FargateTaskDefinition. incompatible-argument:@aws-cdk/aws-ecs.FargateTaskDefinition.addVolume incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition. incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.addVolume + +# We made properties optional and it's really fine but our differ doesn't think so. +weakened:@aws-cdk/cloud-assembly-schema.DockerImageSource +weakened:@aws-cdk/cloud-assembly-schema.FileSource diff --git a/packages/@aws-cdk/cloud-assembly-schema/lib/assets/docker-image-asset.ts b/packages/@aws-cdk/cloud-assembly-schema/lib/assets/docker-image-asset.ts index ebec6ab166fbb..654e1aa032926 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/lib/assets/docker-image-asset.ts +++ b/packages/@aws-cdk/cloud-assembly-schema/lib/assets/docker-image-asset.ts @@ -23,12 +23,24 @@ export interface DockerImageSource { * The directory containing the Docker image build instructions. * * This path is relative to the asset manifest location. + * + * @default - Exactly one of `directory` and `executable` is required + */ + readonly directory?: string; + + /** + * A command-line executable that returns the name of a local + * Docker image on stdout after being run. + * + * @default - Exactly one of `directory` and `executable` is required */ - readonly directory: string; + readonly executable?: string[]; /** * The name of the file with build instructions * + * Only allowed when `directory` is set. + * * @default "Dockerfile" */ readonly dockerFile?: string; @@ -36,6 +48,8 @@ export interface DockerImageSource { /** * Target build stage in a Dockerfile with multiple build stages * + * Only allowed when `directory` is set. + * * @default - The last stage in the Dockerfile */ readonly dockerBuildTarget?: string; @@ -43,6 +57,8 @@ export interface DockerImageSource { /** * Additional build arguments * + * Only allowed when `directory` is set. + * * @default - No additional build arguments */ readonly dockerBuildArgs?: { [name: string]: string }; diff --git a/packages/@aws-cdk/cloud-assembly-schema/lib/assets/file-asset.ts b/packages/@aws-cdk/cloud-assembly-schema/lib/assets/file-asset.ts index efa6cd4384bbe..58c7e0cc93ebc 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/lib/assets/file-asset.ts +++ b/packages/@aws-cdk/cloud-assembly-schema/lib/assets/file-asset.ts @@ -34,16 +34,27 @@ export enum FileAssetPackaging { * Describe the source of a file asset */ export interface FileSource { + /** + * External command which will produce the file asset to upload. + * + * @default - Exactly one of `executable` and `path` is required. + */ + readonly executable?: string[]; + /** * The filesystem object to upload * * This path is relative to the asset manifest location. + * + * @default - Exactly one of `executable` and `path` is required. */ - readonly path: string; + readonly path?: string; /** * Packaging method * + * Only allowed when `path` is specified. + * * @default FILE */ readonly packaging?: FileAssetPackaging; @@ -62,4 +73,4 @@ export interface FileDestination extends AwsDestination { * The destination object key */ readonly objectKey: string; -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json b/packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json index bbd61aae66813..995a895ad824d 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json @@ -53,22 +53,26 @@ "description": "Describe the source of a file asset", "type": "object", "properties": { + "executable": { + "description": "External command which will produce the file asset to upload. (Default - Exactly one of `executable` and `path` is required.)", + "type": "array", + "items": { + "type": "string" + } + }, "path": { - "description": "The filesystem object to upload\n\nThis path is relative to the asset manifest location.", + "description": "The filesystem object to upload\n\nThis path is relative to the asset manifest location. (Default - Exactly one of `executable` and `path` is required.)", "type": "string" }, "packaging": { - "description": "Packaging method (Default FILE)", + "description": "Packaging method\n\nOnly allowed when `path` is specified. (Default FILE)", "enum": [ "file", "zip" ], "type": "string" } - }, - "required": [ - "path" - ] + } }, "FileDestination": { "description": "Where in S3 a file asset needs to be published", @@ -126,28 +130,32 @@ "type": "object", "properties": { "directory": { - "description": "The directory containing the Docker image build instructions.\n\nThis path is relative to the asset manifest location.", + "description": "The directory containing the Docker image build instructions.\n\nThis path is relative to the asset manifest location. (Default - Exactly one of `directory` and `executable` is required)", "type": "string" }, + "executable": { + "description": "A command-line executable that returns the name of a local\nDocker image on stdout after being run. (Default - Exactly one of `directory` and `executable` is required)", + "type": "array", + "items": { + "type": "string" + } + }, "dockerFile": { - "description": "The name of the file with build instructions (Default Dockerfile)", + "description": "The name of the file with build instructions\n\nOnly allowed when `directory` is set. (Default Dockerfile)", "type": "string" }, "dockerBuildTarget": { - "description": "Target build stage in a Dockerfile with multiple build stages (Default - The last stage in the Dockerfile)", + "description": "Target build stage in a Dockerfile with multiple build stages\n\nOnly allowed when `directory` is set. (Default - The last stage in the Dockerfile)", "type": "string" }, "dockerBuildArgs": { - "description": "Additional build arguments (Default - No additional build arguments)", + "description": "Additional build arguments\n\nOnly allowed when `directory` is set. (Default - No additional build arguments)", "type": "object", "additionalProperties": { "type": "string" } } - }, - "required": [ - "directory" - ] + } }, "DockerImageDestination": { "description": "Where to publish docker images", diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json index bdc5a9f306dec..e6bb766b23585 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json @@ -1 +1 @@ -{"version":"7.0.0"} \ No newline at end of file +{"version":"8.0.0"} diff --git a/packages/@aws-cdk/cloud-assembly-schema/test/assets.test.ts b/packages/@aws-cdk/cloud-assembly-schema/test/assets.test.ts index 62aebfa26e6ee..24ddd465484b7 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/test/assets.test.ts +++ b/packages/@aws-cdk/cloud-assembly-schema/test/assets.test.ts @@ -21,6 +21,18 @@ describe('Docker image asset', () => { }, }, }, + externalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + dest: { + region: 'us-north-20', + repositoryName: 'REPO', + imageTag: 'TAG', + }, + }, + }, }, }); }).not.toThrow(); @@ -32,12 +44,18 @@ describe('Docker image asset', () => { version: Manifest.version(), dockerImages: { asset: { + source: { + directory: true, + }, + destinations: {}, + }, + externalAsset: { source: {}, destinations: {}, }, }, }); - }).toThrow(/instance\.dockerImages\.asset\.source requires property \"directory\"/); + }).toThrow(/instance\.dockerImages\.asset\.source\.directory is not of a type\(s\) string/); }); }); @@ -60,6 +78,18 @@ describe('File asset', () => { }, }, }, + externalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + dest: { + region: 'us-north-20', + bucketName: 'Bouquet', + objectKey: 'key', + }, + }, + }, }, }); }).not.toThrow(); @@ -109,6 +139,18 @@ describe('File asset', () => { }, }, }, + externalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + dest: { + region: 'us-north-20', + bucketName: 'Bouquet', + objectKey: 'key', + }, + }, + }, }, }); }).toThrow(/instance\.files\.asset\.source\.path is not of a type\(s\) string/); @@ -149,4 +191,4 @@ function validate(manifest: any) { fs.unlinkSync(filePath); fs.rmdirSync(dir); } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/core/lib/assets.ts b/packages/@aws-cdk/core/lib/assets.ts index 17d3b9d93e53f..d992546dbfdb3 100644 --- a/packages/@aws-cdk/core/lib/assets.ts +++ b/packages/@aws-cdk/core/lib/assets.ts @@ -106,17 +106,30 @@ export interface FileAssetSource { */ readonly sourceHash: string; + /** + * An external command that will produce the packaged asset. + * + * The command should produce the location of a ZIP file on `stdout`. + * + * @default - Exactly one of `directory` and `executable` is required + */ + readonly executable?: string[]; + /** * The path, relative to the root of the cloud assembly, in which this asset * source resides. This can be a path to a file or a directory, dependning on the * packaging type. + * + * @default - Exactly one of `directory` and `executable` is required */ - readonly fileName: string; + readonly fileName?: string; /** * Which type of packaging to perform. + * + * @default - Required if `fileName` is specified. */ - readonly packaging: FileAssetPackaging; + readonly packaging?: FileAssetPackaging; } export interface DockerImageAssetSource { @@ -130,11 +143,22 @@ export interface DockerImageAssetSource { */ readonly sourceHash: string; + /** + * An external command that will produce the packaged asset. + * + * The command should produce the name of a local Docker image on `stdout`. + * + * @default - Exactly one of `directoryName` and `executable` is required + */ + readonly executable?: string[]; + /** * The directory where the Dockerfile is stored, must be relative * to the cloud assembly root. + * + * @default - Exactly one of `directoryName` and `executable` is required */ - readonly directoryName: string; + readonly directoryName?: string; /** * Build args to pass to the `docker build` command. @@ -143,6 +167,8 @@ export interface DockerImageAssetSource { * values cannot refer to unresolved tokens (such as `lambda.functionArn` or * `queue.queueUrl`). * + * Only allowed when `directoryName` is specified. + * * @default - no build args are passed */ readonly dockerBuildArgs?: { [key: string]: string }; @@ -150,6 +176,8 @@ export interface DockerImageAssetSource { /** * Docker target to build to * + * Only allowed when `directoryName` is specified. + * * @default - no target */ readonly dockerBuildTarget?: string; @@ -157,6 +185,8 @@ export interface DockerImageAssetSource { /** * Path to the Dockerfile (relative to the directory). * + * Only allowed when `directoryName` is specified. + * * @default - no file */ readonly dockerFile?: string; diff --git a/packages/@aws-cdk/core/lib/stack-synthesizers/default-synthesizer.ts b/packages/@aws-cdk/core/lib/stack-synthesizers/default-synthesizer.ts index 734e4915c5b95..f90ae86dbf584 100644 --- a/packages/@aws-cdk/core/lib/stack-synthesizers/default-synthesizer.ts +++ b/packages/@aws-cdk/core/lib/stack-synthesizers/default-synthesizer.ts @@ -1,7 +1,7 @@ -import * as cxschema from '@aws-cdk/cloud-assembly-schema'; -import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; +import * as cxschema from '@aws-cdk/cloud-assembly-schema'; +import * as cxapi from '@aws-cdk/cx-api'; import { DockerImageAssetLocation, DockerImageAssetSource, FileAssetLocation, FileAssetPackaging, FileAssetSource } from '../assets'; import { Fn } from '../cfn-fn'; import { CfnParameter } from '../cfn-parameter'; @@ -9,8 +9,8 @@ import { CfnRule } from '../cfn-rule'; import { ISynthesisSession } from '../construct-compat'; import { Stack } from '../stack'; import { Token } from '../token'; -import { StackSynthesizer } from './stack-synthesizer'; import { assertBound, contentHash } from './_shared'; +import { StackSynthesizer } from './stack-synthesizer'; export const BOOTSTRAP_QUALIFIER_CONTEXT = '@aws-cdk/core:bootstrapQualifier'; @@ -289,12 +289,15 @@ export class DefaultStackSynthesizer extends StackSynthesizer { public addFileAsset(asset: FileAssetSource): FileAssetLocation { assertBound(this.stack); assertBound(this.bucketName); + validateFileAssetSource(asset); + const objectKey = this.bucketPrefix + asset.sourceHash + (asset.packaging === FileAssetPackaging.ZIP_DIRECTORY ? '.zip' : ''); // Add to manifest this.files[asset.sourceHash] = { source: { path: asset.fileName, + executable: asset.executable, packaging: asset.packaging, }, destinations: { @@ -325,12 +328,14 @@ export class DefaultStackSynthesizer extends StackSynthesizer { public addDockerImageAsset(asset: DockerImageAssetSource): DockerImageAssetLocation { assertBound(this.stack); assertBound(this.repositoryName); + validateDockerImageAssetSource(asset); const imageTag = asset.sourceHash; // Add to manifest this.dockerImages[asset.sourceHash] = { source: { + executable: asset.executable, directory: asset.directoryName, dockerBuildArgs: asset.dockerBuildArgs, dockerBuildTarget: asset.dockerBuildTarget, @@ -565,4 +570,31 @@ function range(startIncl: number, endExcl: number) { ret.push(i); } return ret; +} + + +function validateFileAssetSource(asset: FileAssetSource) { + if (!!asset.executable === !!asset.fileName) { + throw new Error(`Exactly one of 'fileName' or 'executable' is required, got: ${JSON.stringify(asset)}`); + } + + if (!!asset.packaging !== !!asset.fileName) { + throw new Error(`'packaging' is expected in combination with 'fileName', got: ${JSON.stringify(asset)}`); + } +} + +function validateDockerImageAssetSource(asset: DockerImageAssetSource) { + if (!!asset.executable === !!asset.directoryName) { + throw new Error(`Exactly one of 'directoryName' or 'executable' is required, got: ${JSON.stringify(asset)}`); + } + + check('dockerBuildArgs'); + check('dockerBuildTarget'); + check('dockerFile'); + + function check(key: K) { + if (asset[key] && !asset.directoryName) { + throw new Error(`'${key}' is only allowed in combination with 'directoryName', got: ${JSON.stringify(asset)}`); + } + } } \ No newline at end of file diff --git a/packages/@aws-cdk/core/lib/stack-synthesizers/legacy.ts b/packages/@aws-cdk/core/lib/stack-synthesizers/legacy.ts index e6dfd63235b8c..bf699b271878d 100644 --- a/packages/@aws-cdk/core/lib/stack-synthesizers/legacy.ts +++ b/packages/@aws-cdk/core/lib/stack-synthesizers/legacy.ts @@ -120,6 +120,10 @@ export class LegacyStackSynthesizer extends StackSynthesizer { // only add every image (identified by source hash) once for each stack that uses it. if (!this.addedImageAssets.has(assetId)) { + if (!asset.directoryName) { + throw new Error(`LegacyStackSynthesizer does not support this type of file asset: ${JSON.stringify(asset)}`); + } + const metadata: cxschema.ContainerImageAssetMetadataEntry = { repositoryName, imageTag, @@ -149,6 +153,10 @@ export class LegacyStackSynthesizer extends StackSynthesizer { if (!params) { params = new FileAssetParameters(this.assetParameters, asset.sourceHash); + if (!asset.fileName || !asset.packaging) { + throw new Error(`LegacyStackSynthesizer does not support this type of file asset: ${JSON.stringify(asset)}`); + } + const metadata: cxschema.FileAssetMetadataEntry = { path: asset.fileName, id: asset.sourceHash, diff --git a/packages/cdk-assets/README.md b/packages/cdk-assets/README.md index 2eb10ae621947..c40afcd00c42d 100644 --- a/packages/cdk-assets/README.md +++ b/packages/cdk-assets/README.md @@ -28,6 +28,7 @@ Currently the following asset types are supported: * Files and archives, uploaded to S3 * Docker Images, uploaded to ECR +* Files, archives, and Docker images built by external utilities S3 buckets and ECR repositories to upload to are expected to exist already. @@ -41,6 +42,13 @@ itself in the following behaviors: image in the local Docker cache) already exists named after the asset's ID, it will not be packaged, but will be uploaded directly to the destination location. + +For assets build by external utilities, the contract is such that cdk-assets +expects the utility to manage dedupe detection as well as path/image tag generation. +This means that cdk-assets will call the external utility every time generation +is warranted, and it is up to the utility to a) determine whether to do a +full rebuild; and b) to return only one thing on stdout: the path to the file/archive +asset, or the name of the local Docker image. ## Usage @@ -82,6 +90,19 @@ An asset manifest looks like this: } } }, + "3dfe2b80b050e7e4e168f84feff678d4": { + "source": { + "executable": ["myzip"] + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "bucketName": "MySpecialBucket", + "objectKey": "3dfe2b80b050e7e4e168f84feff678d4.zip" + } + } + }, }, "dockerImages": { "b48783c58a86f7b8c68a4591c4f9be31": { @@ -97,6 +118,20 @@ An asset manifest looks like this: "imageUri": "123456789012.dkr.ecr.us-east-1.amazonaws.com/MyRepository:1234567891b48783c58a86f7b8c68a4591c4f9be31", } } + }, + "d92753c58a86f7b8c68a4591c4f9cf28": { + "source": { + "executable": ["mytool", "package", "dockerdir"], + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "repositoryName": "MyRepository2", + "imageTag": "d92753c58a86f7b8c68a4591c4f9cf28", + "imageUri": "123456789987.dkr.ecr.us-east-1.amazonaws.com/MyRepository2:1234567891b48783c58a86f7b8c68a4591c4f9be31", + } + } } } } diff --git a/packages/cdk-assets/lib/private/handlers/container-images.ts b/packages/cdk-assets/lib/private/handlers/container-images.ts index bd755a52f139b..a3b6756ecb18d 100644 --- a/packages/cdk-assets/lib/private/handlers/container-images.ts +++ b/packages/cdk-assets/lib/private/handlers/container-images.ts @@ -1,73 +1,128 @@ import * as path from 'path'; +import { DockerImageDestination } from '@aws-cdk/cloud-assembly-schema'; import { DockerImageManifestEntry } from '../../asset-manifest'; import { EventType } from '../../progress'; import { IAssetHandler, IHandlerHost } from '../asset-handler'; import { Docker } from '../docker'; import { replaceAwsPlaceholders } from '../placeholders'; +import { shell } from '../shell'; export class ContainerImageAssetHandler implements IAssetHandler { - private readonly localTagName: string; private readonly docker = new Docker(m => this.host.emitMessage(EventType.DEBUG, m)); constructor( private readonly workDir: string, private readonly asset: DockerImageManifestEntry, private readonly host: IHandlerHost) { - - this.localTagName = `cdkasset-${this.asset.id.assetId.toLowerCase()}`; } public async publish(): Promise { const destination = await replaceAwsPlaceholders(this.asset.destination, this.host.aws); - const ecr = await this.host.aws.ecrClient(destination); - const account = (await this.host.aws.discoverCurrentAccount()).accountId; - const repoUri = await repositoryUri(ecr, destination.repositoryName); + if (!repoUri) { throw new Error(`No ECR repository named '${destination.repositoryName}' in account ${account}. Is this account bootstrapped?`); } const imageUri = `${repoUri}:${destination.imageTag}`; - this.host.emitMessage(EventType.CHECK, `Check ${imageUri}`); - if (await imageExists(ecr, destination.repositoryName, destination.imageTag)) { - this.host.emitMessage(EventType.FOUND, `Found ${imageUri}`); - return; - } - + if (await this.destinationAlreadyExists(ecr, destination, imageUri)) { return; } if (this.host.aborted) { return; } // Login before build so that the Dockerfile can reference images in the ECR repo await this.docker.login(ecr); - await this.buildImage(); + + const localTagName = this.asset.source.executable + ? await this.buildExternalAsset(this.asset.source.executable) + : await this.buildDirectoryAsset(); + + if (localTagName === undefined || this.host.aborted) { + return; + } this.host.emitMessage(EventType.UPLOAD, `Push ${imageUri}`); if (this.host.aborted) { return; } - await this.docker.tag(this.localTagName, imageUri); + await this.docker.tag(localTagName, imageUri); await this.docker.push(imageUri); } - private async buildImage(): Promise { - if (await this.docker.exists(this.localTagName)) { - this.host.emitMessage(EventType.CACHED, `Cached ${this.localTagName}`); - return; + /** + * Build a (local) Docker asset from a directory with a Dockerfile + * + * Tags under a deterministic, unique, local identifier wich will skip + * the build if it already exists. + */ + private async buildDirectoryAsset(): Promise { + const localTagName = `cdkasset-${this.asset.id.assetId.toLowerCase()}`; + + if (!(await this.isImageCached(localTagName))) { + if (this.host.aborted) { return undefined; } + + await this.buildImage(localTagName); + } + + return localTagName; + } + + /** + * Build a (local) Docker asset by running an external command + * + * External command is responsible for deduplicating the build if possible, + * and is expected to return the generated image identifier on stdout. + */ + private async buildExternalAsset(executable: string[]): Promise { + this.host.emitMessage(EventType.BUILD, `Building Docker image using command '${executable}'`); + if (this.host.aborted) { return undefined; } + + return (await shell(executable, { quiet: true })).trim(); + } + + + /** + * Check whether the image already exists in the ECR repo + * + * Use the fields from the destination to do the actual check. The imageUri + * should correspond to that, but is only used to print Docker image location + * for user benefit (the format is slightly different). + */ + private async destinationAlreadyExists(ecr: AWS.ECR, destination: DockerImageDestination, imageUri: string): Promise { + this.host.emitMessage(EventType.CHECK, `Check ${imageUri}`); + if (await imageExists(ecr, destination.repositoryName, destination.imageTag)) { + this.host.emitMessage(EventType.FOUND, `Found ${imageUri}`); + return true; } + return false; + } + + private async buildImage(localTagName: string): Promise { const source = this.asset.source; + if (!source.directory) { + throw new Error(`'directory' is expected in the DockerImage asset source, got: ${JSON.stringify(source)}`); + } const fullPath = path.resolve(this.workDir, source.directory); this.host.emitMessage(EventType.BUILD, `Building Docker image at ${fullPath}`); await this.docker.build({ directory: fullPath, - tag: this.localTagName, + tag: localTagName, buildArgs: source.dockerBuildArgs, target: source.dockerBuildTarget, file: source.dockerFile, }); } + + private async isImageCached(localTagName: string): Promise { + if (await this.docker.exists(localTagName)) { + this.host.emitMessage(EventType.CACHED, `Cached ${localTagName}`); + return true; + } + + return false; + } } async function imageExists(ecr: AWS.ECR, repositoryName: string, imageTag: string) { @@ -93,4 +148,4 @@ async function repositoryUri(ecr: AWS.ECR, repositoryName: string): Promise { - const source = this.asset.source; - const fullPath = path.resolve(this.workDir, this.asset.source.path); + private async packageFile(source: FileSource): Promise { + if (!source.path) { + throw new Error(`'path' is expected in the File asset source, got: ${JSON.stringify(source)}`); + } + + const fullPath = path.resolve(this.workDir, source.path); if (source.packaging === FileAssetPackaging.ZIP_DIRECTORY) { + const contentType = 'application/zip'; + await fs.mkdir(this.fileCacheRoot, { recursive: true }); - const ret = path.join(this.fileCacheRoot, `${this.asset.id.assetId}.zip`); + const packagedPath = path.join(this.fileCacheRoot, `${this.asset.id.assetId}.zip`); - if (await pathExists(ret)) { - this.host.emitMessage(EventType.CACHED, `From cache ${ret}`); - return ret; + if (await pathExists(packagedPath)) { + this.host.emitMessage(EventType.CACHED, `From cache ${path}`); + return { packagedPath, contentType }; } - this.host.emitMessage(EventType.BUILD, `Zip ${fullPath} -> ${ret}`); - await zipDirectory(fullPath, ret); - return ret; + this.host.emitMessage(EventType.BUILD, `Zip ${fullPath} -> ${path}`); + await zipDirectory(fullPath, packagedPath); + return { packagedPath, contentType }; } else { - return fullPath; + return { packagedPath: fullPath }; } } + + private async externalPackageFile(executable: string[]): Promise { + this.host.emitMessage(EventType.BUILD, `Building asset source using command: '${executable}'`); + + return { + packagedPath: (await shell(executable, { quiet: true })).trim(), + contentType: 'application/zip', + }; + } } enum BucketOwnership { @@ -109,3 +124,21 @@ async function objectExists(s3: AWS.S3, bucket: string, key: string) { const response = await s3.listObjectsV2({ Bucket: bucket, Prefix: key, MaxKeys: 1 }).promise(); return response.Contents != null && response.Contents.some(object => object.Key === key); } + + +/** + * A packaged asset which can be uploaded (either a single file or directory) + */ +interface PackagedFileAsset { + /** + * Path of the file or directory + */ + readonly packagedPath: string; + + /** + * Content type to be added in the S3 upload action + * + * @default - No content type + */ + readonly contentType?: string; +} diff --git a/packages/cdk-assets/lib/private/handlers/index.ts b/packages/cdk-assets/lib/private/handlers/index.ts index 2e4d406ce5b0b..97ec7354279df 100644 --- a/packages/cdk-assets/lib/private/handlers/index.ts +++ b/packages/cdk-assets/lib/private/handlers/index.ts @@ -12,4 +12,4 @@ export function makeAssetHandler(manifest: AssetManifest, asset: IManifestEntry, } throw new Error(`Unrecognized asset type: '${asset}'`); -} \ No newline at end of file +} diff --git a/packages/cdk-assets/test/docker-images.test.ts b/packages/cdk-assets/test/docker-images.test.ts index 3f0aeaabf474c..3b608a1e63ffe 100644 --- a/packages/cdk-assets/test/docker-images.test.ts +++ b/packages/cdk-assets/test/docker-images.test.ts @@ -9,6 +9,8 @@ import { mockSpawn } from './mock-child_process'; let aws: ReturnType; const absoluteDockerPath = '/simple/cdk.out/dockerdir'; beforeEach(() => { + jest.resetAllMocks(); + mockfs({ '/simple/cdk.out/assets.json': JSON.stringify({ version: Manifest.version(), @@ -28,6 +30,24 @@ beforeEach(() => { }, }, }), + '/external/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theExternalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'ghijkl', + }, + }, + }, + }, + }), '/simple/cdk.out/dockerdir/Dockerfile': 'FROM scratch', '/abs/cdk.out/assets.json': JSON.stringify({ version: Manifest.version(), @@ -92,7 +112,7 @@ describe('with a complete manifest', () => { ], }); - mockSpawn( + const expectAllSpawns = mockSpawn( { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, { commandLine: ['docker', 'inspect', 'cdkasset-theasset'] }, { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:abcdef'] }, @@ -100,6 +120,9 @@ describe('with a complete manifest', () => { ); await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter }); test('build and upload docker image if not exists anywhere', async () => { @@ -110,7 +133,7 @@ describe('with a complete manifest', () => { ], }); - mockSpawn( + const expectAllSpawns = mockSpawn( { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: absoluteDockerPath }, @@ -119,6 +142,41 @@ describe('with a complete manifest', () => { ); await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); +}); + +describe('external assets', () => { + let pub: AssetPublishing; + const externalTag = 'external:tag'; + beforeEach(() => { + pub = new AssetPublishing(AssetManifest.fromPath('/external/cdk.out'), { aws }); + }); + + test('upload externally generated Docker image', async () => { + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['sometool'], stdout: externalTag }, + { commandLine: ['docker', 'tag', externalTag, '12345.amazonaws.com/repo:ghijkl'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:ghijkl'] }, + ); + + await pub.publish(); + + expect(aws.ecrClient).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); + expectAllSpawns(); }); }); @@ -132,7 +190,7 @@ test('correctly identify Docker directory if path is absolute', async () => { ], }); - mockSpawn( + const expectAllSpawns = mockSpawn( // Only care about the 'build' command line { commandLine: ['docker', 'login'], prefix: true }, { commandLine: ['docker', 'inspect'], exitCode: 1, prefix: true }, @@ -142,4 +200,7 @@ test('correctly identify Docker directory if path is absolute', async () => { ); await pub.publish(); + + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + expectAllSpawns(); }); diff --git a/packages/cdk-assets/test/files.test.ts b/packages/cdk-assets/test/files.test.ts index e8c7247ef7f42..42cb8a71c05ad 100644 --- a/packages/cdk-assets/test/files.test.ts +++ b/packages/cdk-assets/test/files.test.ts @@ -1,10 +1,17 @@ +jest.mock('child_process'); + import { Manifest } from '@aws-cdk/cloud-assembly-schema'; import * as mockfs from 'mock-fs'; import { AssetManifest, AssetPublishing } from '../lib'; import { mockAws, mockedApiResult, mockUpload } from './mock-aws'; +import { mockSpawn } from './mock-child_process'; + +const ABS_PATH = '/simple/cdk.out/some_external_file'; let aws: ReturnType; beforeEach(() => { + jest.resetAllMocks(); + mockfs({ '/simple/cdk.out/assets.json': JSON.stringify({ version: Manifest.version(), @@ -25,6 +32,7 @@ beforeEach(() => { }, }), '/simple/cdk.out/some_file': 'FILE_CONTENTS', + [ABS_PATH]: 'FILE_CONTENTS', '/abs/cdk.out/assets.json': JSON.stringify({ version: Manifest.version(), files: { @@ -36,7 +44,25 @@ beforeEach(() => { theDestination: { region: 'us-north-50', assumeRoleArn: 'arn:aws:role', - bucketName: 'some_bucket', + bucketName: 'some_other_bucket', + objectKey: 'some_key', + }, + }, + }, + }, + }), + '/external/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + externalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + bucketName: 'some_external_bucket', objectKey: 'some_key', }, }, @@ -127,4 +153,40 @@ test('correctly identify asset path if path is absolute', async () => { aws.mockS3.upload = mockUpload('FILE_CONTENTS'); await pub.publish(); + + expect(true).toBeTruthy(); // No exception, satisfy linter +}); + +describe('external assets', () => { + let pub: AssetPublishing; + beforeEach(() => { + pub = new AssetPublishing(AssetManifest.fromPath('/external/cdk.out'), { aws }); + }); + + test('do nothing if file exists already', async () => { + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key' }] }); + + await pub.publish(); + + expect(aws.mockS3.listObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_external_bucket', + Prefix: 'some_key', + MaxKeys: 1, + })); + }); + + test('upload external asset correctly', async () => { + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + const expectAllSpawns = mockSpawn({ commandLine: ['sometool'], stdout: ABS_PATH }); + + await pub.publish(); + + expect(aws.s3Client).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); + + expectAllSpawns(); + }); }); diff --git a/packages/cdk-assets/test/mock-child_process.ts b/packages/cdk-assets/test/mock-child_process.ts index da0fd27d08fe6..2cb513e24fff7 100644 --- a/packages/cdk-assets/test/mock-child_process.ts +++ b/packages/cdk-assets/test/mock-child_process.ts @@ -17,7 +17,7 @@ export interface Invocation { prefix?: boolean; } -export function mockSpawn(...invocations: Invocation[]) { +export function mockSpawn(...invocations: Invocation[]): () => void { let mock = (child_process.spawn as any); for (const _invocation of invocations) { const invocation = _invocation; // Mirror into variable for closure @@ -42,7 +42,7 @@ export function mockSpawn(...invocations: Invocation[]) { child.stderr = new events.EventEmitter(); if (invocation.stdout) { - mockEmit(child.stdout, 'data', invocation.stdout); + mockEmit(child.stdout, 'data', Buffer.from(invocation.stdout)); } mockEmit(child, 'close', invocation.exitCode ?? 0); @@ -53,6 +53,10 @@ export function mockSpawn(...invocations: Invocation[]) { mock.mockImplementation((binary: string, args: string[], _options: any) => { throw new Error(`Did not expect call of ${JSON.stringify([binary, ...args])}`); }); + + return () => { + expect(mock).toHaveBeenCalledTimes(invocations.length); + }; } /**