feat(cdk-assets): add external asset support (aws#12259)

In the event that assets are not actually available at synthesis
time, we still want to support JIT (just-in-time) asset
generation via external tooling. This would, for instance, enable
a third party tool to fetch additional resources prior to
bundling/building and subsequent uploading.

This adds a new interface for both File and Docker asset types
that allows users to specify an executable. The executable,
depending on the asset type, must then reply with a specific key
on stdout, which will then get picked up and used by CDK Assets.

This also updates the default stack synthesizer to support adding
external sources directly. This is technically a breaking change
for anyone who currently extends the class.


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

allowed-breaking-changes.txt

@@ -52,3 +52,7 @@ incompatible-argument:@aws-cdk/aws-ecs.FargateTaskDefinition.<initializer>
 incompatible-argument:@aws-cdk/aws-ecs.FargateTaskDefinition.addVolume
 incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.<initializer>
 incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.addVolume
+
+# We made properties optional and it's really fine but our differ doesn't think so.
+weakened:@aws-cdk/cloud-assembly-schema.DockerImageSource
+weakened:@aws-cdk/cloud-assembly-schema.FileSource

packages/@aws-cdk/cloud-assembly-schema/lib/assets/docker-image-asset.ts

@@ -23,26 +23,42 @@ export interface DockerImageSource {
    * The directory containing the Docker image build instructions.
    *
    * This path is relative to the asset manifest location.
+   *
+   * @default - Exactly one of `directory` and `executable` is required
+   */
+  readonly directory?: string;
+
+  /**
+   * A command-line executable that returns the name of a local
+   * Docker image on stdout after being run.
+   *
+   * @default - Exactly one of `directory` and `executable` is required
    */
-  readonly directory: string;
+  readonly executable?: string[];
 
   /**
    * The name of the file with build instructions
    *
+   * Only allowed when `directory` is set.
+   *
    * @default "Dockerfile"
    */
   readonly dockerFile?: string;
 
   /**
    * Target build stage in a Dockerfile with multiple build stages
    *
+   * Only allowed when `directory` is set.
+   *
    * @default - The last stage in the Dockerfile
    */
   readonly dockerBuildTarget?: string;
 
   /**
    * Additional build arguments
    *
+   * Only allowed when `directory` is set.
+   *
    * @default - No additional build arguments
    */
   readonly dockerBuildArgs?: { [name: string]: string };

packages/@aws-cdk/cloud-assembly-schema/lib/assets/file-asset.ts

@@ -34,16 +34,27 @@ export enum FileAssetPackaging {
  * Describe the source of a file asset
  */
 export interface FileSource {
+  /**
+   * External command which will produce the file asset to upload.
+   *
+   * @default - Exactly one of `executable` and `path` is required.
+   */
+  readonly executable?: string[];
+
   /**
    * The filesystem object to upload
    *
    * This path is relative to the asset manifest location.
+   *
+   * @default - Exactly one of `executable` and `path` is required.
    */
-  readonly path: string;
+  readonly path?: string;
 
   /**
    * Packaging method
    *
+   * Only allowed when `path` is specified.
+   *
    * @default FILE
    */
   readonly packaging?: FileAssetPackaging;
@@ -62,4 +73,4 @@ export interface FileDestination extends AwsDestination {
    * The destination object key
    */
   readonly objectKey: string;
-}
+}

packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json

@@ -53,22 +53,26 @@
             "description": "Describe the source of a file asset",
             "type": "object",
             "properties": {
+                "executable": {
+                    "description": "External command which will produce the file asset to upload. (Default - Exactly one of `executable` and `path` is required.)",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "path": {
-                    "description": "The filesystem object to upload\n\nThis path is relative to the asset manifest location.",
+                    "description": "The filesystem object to upload\n\nThis path is relative to the asset manifest location. (Default - Exactly one of `executable` and `path` is required.)",
                     "type": "string"
                 },
                 "packaging": {
-                    "description": "Packaging method (Default FILE)",
+                    "description": "Packaging method\n\nOnly allowed when `path` is specified. (Default FILE)",
                     "enum": [
                         "file",
                         "zip"
                     ],
                     "type": "string"
                 }
-            },
-            "required": [
-                "path"
-            ]
+            }
         },
         "FileDestination": {
             "description": "Where in S3 a file asset needs to be published",
@@ -126,28 +130,32 @@
             "type": "object",
             "properties": {
                 "directory": {
-                    "description": "The directory containing the Docker image build instructions.\n\nThis path is relative to the asset manifest location.",
+                    "description": "The directory containing the Docker image build instructions.\n\nThis path is relative to the asset manifest location. (Default - Exactly one of `directory` and `executable` is required)",
                     "type": "string"
                 },
+                "executable": {
+                    "description": "A command-line executable that returns the name of a local\nDocker image on stdout after being run. (Default - Exactly one of `directory` and `executable` is required)",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "dockerFile": {
-                    "description": "The name of the file with build instructions (Default Dockerfile)",
+                    "description": "The name of the file with build instructions\n\nOnly allowed when `directory` is set. (Default Dockerfile)",
                     "type": "string"
                 },
                 "dockerBuildTarget": {
-                    "description": "Target build stage in a Dockerfile with multiple build stages (Default - The last stage in the Dockerfile)",
+                    "description": "Target build stage in a Dockerfile with multiple build stages\n\nOnly allowed when `directory` is set. (Default - The last stage in the Dockerfile)",
                     "type": "string"
                 },
                 "dockerBuildArgs": {
-                    "description": "Additional build arguments (Default - No additional build arguments)",
+                    "description": "Additional build arguments\n\nOnly allowed when `directory` is set. (Default - No additional build arguments)",
                     "type": "object",
                     "additionalProperties": {
                         "type": "string"
                     }
                 }
-            },
-            "required": [
-                "directory"
-            ]
+            }
         },
         "DockerImageDestination": {
             "description": "Where to publish docker images",

packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json

@@ -1 +1 @@
-{"version":"7.0.0"}
+{"version":"8.0.0"}

packages/@aws-cdk/cloud-assembly-schema/test/assets.test.ts

@@ -21,6 +21,18 @@ describe('Docker image asset', () => {
               },
             },
           },
+          externalAsset: {
+            source: {
+              executable: ['sometool'],
+            },
+            destinations: {
+              dest: {
+                region: 'us-north-20',
+                repositoryName: 'REPO',
+                imageTag: 'TAG',
+              },
+            },
+          },
         },
       });
     }).not.toThrow();
@@ -32,12 +44,18 @@ describe('Docker image asset', () => {
         version: Manifest.version(),
         dockerImages: {
           asset: {
+            source: {
+              directory: true,
+            },
+            destinations: {},
+          },
+          externalAsset: {
             source: {},
             destinations: {},
           },
         },
       });
-    }).toThrow(/instance\.dockerImages\.asset\.source requires property \"directory\"/);
+    }).toThrow(/instance\.dockerImages\.asset\.source\.directory is not of a type\(s\) string/);
   });
 });
 
@@ -60,6 +78,18 @@ describe('File asset', () => {
                 },
               },
             },
+            externalAsset: {
+              source: {
+                executable: ['sometool'],
+              },
+              destinations: {
+                dest: {
+                  region: 'us-north-20',
+                  bucketName: 'Bouquet',
+                  objectKey: 'key',
+                },
+              },
+            },
           },
         });
       }).not.toThrow();
@@ -109,6 +139,18 @@ describe('File asset', () => {
                 },
               },
             },
+            externalAsset: {
+              source: {
+                executable: ['sometool'],
+              },
+              destinations: {
+                dest: {
+                  region: 'us-north-20',
+                  bucketName: 'Bouquet',
+                  objectKey: 'key',
+                },
+              },
+            },
           },
         });
       }).toThrow(/instance\.files\.asset\.source\.path is not of a type\(s\) string/);
@@ -149,4 +191,4 @@ function validate(manifest: any) {
     fs.unlinkSync(filePath);
     fs.rmdirSync(dir);
   }
-}
+}

packages/@aws-cdk/core/lib/assets.ts

@@ -106,17 +106,30 @@ export interface FileAssetSource {
    */
   readonly sourceHash: string;
 
+  /**
+   * An external command that will produce the packaged asset.
+   *
+   * The command should produce the location of a ZIP file on `stdout`.
+   *
+   * @default - Exactly one of `directory` and `executable` is required
+   */
+  readonly executable?: string[];
+
   /**
    * The path, relative to the root of the cloud assembly, in which this asset
    * source resides. This can be a path to a file or a directory, dependning on the
    * packaging type.
+   *
+   * @default - Exactly one of `directory` and `executable` is required
    */
-  readonly fileName: string;
+  readonly fileName?: string;
 
   /**
    * Which type of packaging to perform.
+   *
+   * @default - Required if `fileName` is specified.
    */
-  readonly packaging: FileAssetPackaging;
+  readonly packaging?: FileAssetPackaging;
 }
 
 export interface DockerImageAssetSource {
@@ -130,11 +143,22 @@ export interface DockerImageAssetSource {
    */
   readonly sourceHash: string;
 
+  /**
+   * An external command that will produce the packaged asset.
+   *
+   * The command should produce the name of a local Docker image on `stdout`.
+   *
+   * @default - Exactly one of `directoryName` and `executable` is required
+   */
+  readonly executable?: string[];
+
   /**
    * The directory where the Dockerfile is stored, must be relative
    * to the cloud assembly root.
+   *
+   * @default - Exactly one of `directoryName` and `executable` is required
    */
-  readonly directoryName: string;
+  readonly directoryName?: string;
 
   /**
    * Build args to pass to the `docker build` command.
@@ -143,20 +167,26 @@ export interface DockerImageAssetSource {
    * values cannot refer to unresolved tokens (such as `lambda.functionArn` or
    * `queue.queueUrl`).
    *
+   * Only allowed when `directoryName` is specified.
+   *
    * @default - no build args are passed
    */
   readonly dockerBuildArgs?: { [key: string]: string };
 
   /**
    * Docker target to build to
    *
+   * Only allowed when `directoryName` is specified.
+   *
    * @default - no target
    */
   readonly dockerBuildTarget?: string;
 
   /**
    * Path to the Dockerfile (relative to the directory).
    *
+   * Only allowed when `directoryName` is specified.
+   *
    * @default - no file
    */
   readonly dockerFile?: string;