feat(cloudformation): nested stacks (aws#2821)

The `NestedStack` construct is a special kind of `Stack`. Any resource defined within its scope will be included in a separate template from the parent stack. 

The template for the nested stack is synthesized into the cloud assembly but not treated as a deployable unit but rather as a file asset. This will cause the CLI to upload it to S3 and wire it's coordinates to the parent stack so we can reference its S3 URL.

To support references between the parent stack and the nested stack, we abstracted the concept of preparing cross references by inverting the control of `consumeReference` from the reference object itself to the `Stack` object. This allows us to override it at the `NestedStack` level (through `prepareCrossReference`) and mutate the token accordingly. 

When an outside resource is referenced within the nested stack, it is wired through a synthesized CloudFormation parameter. When a resource inside the nested stack is referenced from outside, it will be wired through a synthesized CloudFormation output. This works for arbitrarily deep nesting.

When an asset is referenced within a nested stack, it will be added to the top-level stack and wired through the asset parameter reference (like any other reference).

Fixes aws#239
Fixes aws#395
Related aws#3437 
Related aws#1439 
Related aws#3463

allowed-breaking-changes.txt

@@ -23,3 +23,7 @@ incompatible-argument:@aws-cdk/aws-apigateway.ProxyResource.addProxy
 incompatible-argument:@aws-cdk/aws-apigateway.Resource.addProxy
 incompatible-argument:@aws-cdk/aws-apigateway.ResourceBase.addProxy
 incompatible-argument:@aws-cdk/aws-apigateway.IResource.addProxy
+removed:@aws-cdk/core.ConstructNode.addReference
+removed:@aws-cdk/core.ConstructNode.references
+removed:@aws-cdk/core.OutgoingReference
+

packages/@aws-cdk/assert/lib/expect.ts

@@ -5,6 +5,6 @@ import { SynthUtils } from './synth-utils';
 
 export function expect(stack: api.CloudFormationStackArtifact | cdk.Stack, skipValidation = false): StackInspector {
   // if this is already a synthesized stack, then just inspect it.
-  const artifact = stack instanceof api.CloudFormationStackArtifact ? stack : SynthUtils.synthesize(stack, { skipValidation });
+  const artifact = stack instanceof api.CloudFormationStackArtifact ? stack : SynthUtils._synthesizeWithNested(stack, { skipValidation });
   return new StackInspector(artifact);
 }

packages/@aws-cdk/assert/lib/inspector.ts

@@ -27,11 +27,20 @@ export abstract class Inspector {
 }
 
 export class StackInspector extends Inspector {
-  constructor(public readonly stack: api.CloudFormationStackArtifact) {
+
+  private readonly template: { [key: string]: any };
+
+  constructor(public readonly stack: api.CloudFormationStackArtifact | object) {
     super();
+
+    this.template = stack instanceof api.CloudFormationStackArtifact ? stack.template : stack;
   }
 
   public at(path: string | string[]): StackPathInspector {
+    if (!(this.stack instanceof api.CloudFormationStackArtifact)) {
+      throw new Error(`Cannot use "expect(stack).at(path)" for a raw template, only CloudFormationStackArtifact`);
+    }
+
     const strPath = typeof path === 'string' ? path : path.join('/');
     return new StackPathInspector(this.stack, strPath);
   }
@@ -41,7 +50,7 @@ export class StackInspector extends Inspector {
   }
 
   public get value(): { [key: string]: any } {
-    return this.stack.template;
+    return this.template;
   }
 }
 

packages/@aws-cdk/assert/lib/synth-utils.ts

@@ -1,29 +1,36 @@
-import { ConstructNode, Stack, SynthesisOptions } from '@aws-cdk/core';
+import { App, ConstructNode, Stack, SynthesisOptions } from '@aws-cdk/core';
 import cxapi = require('@aws-cdk/cx-api');
+import fs = require('fs');
+import path = require('path');
 
 export class SynthUtils {
-  /**
-   * Synthesizes the stack and returns a `CloudFormationStackArtifact` which can be inspected.
-   */
   public static synthesize(stack: Stack, options: SynthesisOptions = { }): cxapi.CloudFormationStackArtifact {
     // always synthesize against the root (be it an App or whatever) so all artifacts will be included
     const root = stack.node.root;
-    const assembly = ConstructNode.synth(root.node, options);
+
+    // if the root is an app, invoke "synth" to avoid double synthesis
+    const assembly = root instanceof App ? root.synth() : ConstructNode.synth(root.node, options);
+
     return assembly.getStack(stack.stackName);
   }
 
   /**
    * Synthesizes the stack and returns the resulting CloudFormation template.
    */
   public static toCloudFormation(stack: Stack, options: SynthesisOptions = { }): any {
-    return this.synthesize(stack, options).template;
+    const synth = this._synthesizeWithNested(stack, options);
+    if (synth instanceof cxapi.CloudFormationStackArtifact) {
+      return synth.template;
+    } else {
+      return synth;
+    }
   }
 
   /**
    * @returns Returns a subset of the synthesized CloudFormation template (only specific resource types).
    */
   public static subset(stack: Stack, options: SubsetOptions): any {
-    const template = SynthUtils.synthesize(stack).template;
+    const template = this.toCloudFormation(stack);
     if (template.Resources) {
       for (const [key, resource] of Object.entries(template.Resources)) {
         if (options.resourceTypes && !options.resourceTypes.includes((resource as any).Type)) {
@@ -34,6 +41,28 @@ export class SynthUtils {
 
     return template;
   }
+
+  /**
+   * Synthesizes the stack and returns a `CloudFormationStackArtifact` which can be inspected.
+   * Supports nested stacks as well as normal stacks.
+   *
+   * @return CloudFormationStackArtifact for normal stacks or the actual template for nested stacks
+   * @internal
+   */
+  public static _synthesizeWithNested(stack: Stack, options: SynthesisOptions = { }): cxapi.CloudFormationStackArtifact | object {
+    // always synthesize against the root (be it an App or whatever) so all artifacts will be included
+    const root = stack.node.root;
+
+    // if the root is an app, invoke "synth" to avoid double synthesis
+    const assembly = root instanceof App ? root.synth() : ConstructNode.synth(root.node, options);
+
+    // if this is a nested stack (it has a parent), then just read the template as a string
+    if (stack.parentStack) {
+      return JSON.parse(fs.readFileSync(path.join(assembly.directory, stack.templateFile)).toString('utf-8'));
+    }
+
+    return assembly.getStack(stack.stackName);
+  }
 }
 
 export interface SubsetOptions {

packages/@aws-cdk/aws-cloudformation/README.md

@@ -51,3 +51,44 @@ See the following section of the docs on details to write Custom Resources:
 * [Introduction](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html)
 * [Reference](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-ref.html)
 * [Code Reference](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html)
+
+### Nested Stacks
+
+[Nested stacks](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html) are stacks created as part of other stacks. You create a nested stack within another stack by using the `NestedStack` construct.
+
+As your infrastructure grows, common patterns can emerge in which you declare the same components in multiple templates. You can separate out these common components and create dedicated templates for them. Then use the resource in your template to reference other templates, creating nested stacks.
+
+For example, assume that you have a load balancer configuration that you use for most of your stacks. Instead of copying and pasting the same configurations into your templates, you can create a dedicated template for the load balancer. Then, you just use the resource to reference that template from within other templates.
+
+The following example will define a single top-level stack that contains two nested stacks: each one with a single Amazon S3 bucket:
+
+```ts
+import { Stack, Construct, StackProps } from '@aws-cdk/core';
+import cfn = require('@aws-cdk/aws-cloudformation');
+import s3 = require('@aws-cdk/aws-s3');
+
+class MyNestedStack extends cfn.NestedStack {
+  constructor(scope: Construct, id: string, props: cfn.NestedStackProps) {
+    super(scope, id, props);
+
+    new s3.Bucket(this, 'NestedBucket');  
+  }
+}
+
+class MyParentStack extends Stack {
+  constructor(scope: Construct, id: string, props: StackProps) {
+    super(scope, id, props);
+
+    new MyNestedStack(scope, 'Nested1');
+    new MyNestedStack(scope, 'Nested2');
+  }
+}
+```
+
+Resources references across nested/parent boundaries (even with multiple levels of nesting) will be wired by the AWS CDK
+through CloudFormation parameters and outputs. When a resource from a parent stack is referenced by a nested stack,
+a CloudFormation parameter will automatically be added to the nested stack and assigned from the parent; when a resource
+from a nested stack is referenced by a parent stack, a CloudFormation output will be automatically be added to the
+nested stack and referenced using `Fn::GetAtt "Outputs.Xxx"` from the parent.
+
+Nested stacks also support the use of Docker image and file assets.

packages/@aws-cdk/aws-cloudformation/lib/index.ts

@@ -1,5 +1,6 @@
 export * from './cloud-formation-capabilities';
 export * from './custom-resource';
+export * from './nested-stack';
 
 // AWS::CloudFormation CloudFormation Resources:
 export * from './cloudformation.generated';