fix(cli): add Cloud Assembly backwards compat tests (aws#4625)

* fix(cli): add Cloud Assembly backwards compat tests

Make sure that the CLI will continue to be able to load old cloud
assemblies. We do this by using the CLI feature that it can directly
load Cloud Assemblies through its `--app` argument, hard copying
a couple of cloud assemblies into the test set and trying to synth
those.

In addition to adding a test for loading v0.36.0 assemblies, also
add tests for 2 common context provider requests (AZs and VPC),
to make sure new CLIs will continue to be able to handle these
context lookup requests.

Fixes aws#4475, pre-emptively adds a regression test for aws#4544.

* Address review comments

packages/aws-cdk/.gitignore

@@ -16,3 +16,4 @@ coverage
 *.snk
 
 !test/integ/run-wrappers/dist
+!test/integ/cli/**/*

packages/aws-cdk/test/integ/cli/.gitignore

@@ -0,0 +1 @@
+cdk.context.json

packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/InitStack.template.json

@@ -0,0 +1 @@
+{}

packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/cdk.out

@@ -0,0 +1 @@
+{"version":"0.36.0"}

packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/manifest.json

@@ -0,0 +1,19 @@
+{
+  "version": "0.36.0",
+  "artifacts": {
+    "InitStack": {
+      "type": "aws:cloudformation:stack",
+      "environment": "aws://unknown-account/unknown-region",
+      "properties": {
+        "templateFile": "InitStack.template.json"
+      }
+    }
+  },
+  "runtime": {
+    "libraries": {
+      "@aws-cdk/core": "1.12.0",
+      "@aws-cdk/cx-api": "1.12.0",
+      "jsii-runtime": "node.js/v8.11.4"
+    }
+  }
+}

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/InitStack.template.json

@@ -0,0 +1,2 @@
+{
+}

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/cdk.out

@@ -0,0 +1 @@
+{"version":"1.10.0"}

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/manifest.json.js

@@ -0,0 +1,37 @@
+process.stdout.write(JSON.stringify({
+  "version": "1.10.0",
+  "artifacts": {
+    "InitStack": {
+      "type": "aws:cloudformation:stack",
+      "environment": `aws://${process.env.TEST_ACCOUNT}/${process.env.TEST_REGION}`,
+      "properties": {
+        "templateFile": "InitStack.template.json"
+      }
+    }
+  },
+  "runtime": {
+    "libraries": {
+      "@aws-cdk/core": "1.14.0",
+      "@aws-cdk/cx-api": "1.14.0",
+      "@aws-cdk/aws-ec2": "1.14.0",
+      "@aws-cdk/aws-iam": "1.14.0",
+      "@aws-cdk/region-info": "1.14.0",
+      "@aws-cdk/aws-ssm": "1.14.0",
+      "@aws-cdk/aws-cloudwatch": "1.14.0",
+      "jsii-runtime": "node.js/v8.11.4"
+    }
+  },
+  "missing": [
+    {
+      "key": `vpc-provider:account=${process.env.TEST_ACCOUNT}:filter.isDefault=true:region=${process.env.TEST_REGION}`,
+      "props": {
+        "account": process.env.TEST_ACCOUNT,
+        "region": process.env.TEST_REGION,
+        "filter": {
+          "isDefault": "true"
+        }
+      },
+      "provider": "vpc-provider"
+    }
+  ]
+}, undefined, 2));

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/InitStack.template.json

@@ -0,0 +1,2 @@
+{
+}

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/cdk.out

@@ -0,0 +1 @@
+{"version":"1.10.0"}

packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/manifest.json.js

@@ -0,0 +1,34 @@
+process.stdout.write(JSON.stringify({
+  "version": "1.10.0",
+  "artifacts": {
+    "InitStack": {
+      "type": "aws:cloudformation:stack",
+      "environment": `aws://${process.env.TEST_ACCOUNT}/${process.env.TEST_REGION}`,
+      "properties": {
+        "templateFile": "InitStack.template.json"
+      }
+    }
+  },
+  "runtime": {
+    "libraries": {
+      "@aws-cdk/core": "1.14.0",
+      "@aws-cdk/cx-api": "1.14.0",
+      "@aws-cdk/aws-ec2": "1.14.0",
+      "@aws-cdk/aws-iam": "1.14.0",
+      "@aws-cdk/region-info": "1.14.0",
+      "@aws-cdk/aws-ssm": "1.14.0",
+      "@aws-cdk/aws-cloudwatch": "1.14.0",
+      "jsii-runtime": "node.js/v8.11.4"
+    }
+  },
+  "missing": [
+    {
+      "key": `availability-zones:account=${process.env.TEST_ACCOUNT}:region=${process.env.TEST_REGION}`,
+      "props": {
+        "account": process.env.TEST_ACCOUNT,
+        "region": process.env.TEST_REGION,
+      },
+      "provider": "availability-zones"
+    }
+  ]
+}, undefined, 2));

packages/aws-cdk/test/integ/cli/common.bash

@@ -3,8 +3,11 @@ scriptdir=$(cd $(dirname $0) && pwd)
 cd ${scriptdir}
 
 if [[ -z "${CREDS_SET:-}" ]]; then
-    # Check that credentials are configured
-    aws sts get-caller-identity > /dev/null
+    # Check that credentials are configured (will error & abort if not)
+    creds=$(aws sts get-caller-identity)
+
+    export TEST_ACCOUNT=$(node -p "($creds).Account")
+    export TEST_REGION=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}
     export CREDS_SET=1
 fi
 

packages/aws-cdk/test/integ/cli/test-backwards-compat.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -euo pipefail
+scriptdir=$(cd $(dirname $0) && pwd)
+source ${scriptdir}/common.bash
+# ----------------------------------------------------------
+
+tmpdir=$(dirname $(mktemp -u))
+casmdir=$tmpdir/cdk-integ-cx
+stderrfile=$casmdir/cdk.err
+
+# Copy the files from a cloud assembly source dir into the
+# tempdir. Evaluate .js files found their (interpret them
+# as templates).
+function prepare_cloud_assembly() {
+    local asmdir=$1
+    echo "ASSEMBLY ${asmdir}"
+    rm -rf $casmdir
+    mkdir -p $casmdir
+    cp -R $asmdir/* $casmdir
+
+    # Execute templates to produce file with the same name
+    # but without .js extension
+    shopt -s nullglob
+    for template in $casmdir/*.js; do
+        node $template > ${template%.js}
+    done
+}
+
+# Assert that there was no providerError in CDK's stderr
+# Because we rely on the app/framework to actually error in case the
+# provider fails, we inspect the logs here.
+function assert_no_error() {
+    local asmdir=$1
+
+    if grep '$providerError' $stderrfile; then
+        cat $stderrfile >&2
+        echo "There was an error executing the context provider for assembly ${asmdir}!" >&2
+        exit 1
+    fi
+}
+
+# Echo the TEST_ACCOUNT, TEST_REGION vars to make bash abort if they're not set.
+echo "Running backwards compatibility test (account ${TEST_ACCOUNT}, region ${TEST_REGION})"
+
+for assembly in ${scriptdir}/cloud-assemblies/*; do
+    prepare_cloud_assembly $assembly
+
+    rm -f cdk.context.json
+    cdk -a $casmdir -v synth > /dev/null 2> $stderrfile
+
+    assert_no_error $assembly
+done
+
+echo "✅  success"

packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh

@@ -7,15 +7,13 @@ source ${scriptdir}/common.bash
 setup
 
 parameterName=/does/not/exist
-account=$(node -p "($(aws sts get-caller-identity)).Account")
-region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}
 
 function cdk_synth() {
     (cdk synth $@ 2>&1 || true) | strip_color_codes
 }
 
 assert "cdk_synth ${STACK_NAME_PREFIX}-missing-ssm-parameter -c test:ssm-parameter-name=${parameterName}" <<HERE
-[Error at /${STACK_NAME_PREFIX}-missing-ssm-parameter] SSM parameter not available in account ${account}, region ${region}: /does/not/exist
+[Error at /${STACK_NAME_PREFIX}-missing-ssm-parameter] SSM parameter not available in account ${TEST_ACCOUNT}, region ${TEST_REGION}: /does/not/exist
 Found errors
 HERE