chore(cli): add CLI integration tests into aws-cdk package (aws#3336)

Add the CLI's integration tests into the CLI package, in order to version the tests
together with the code they're testing.

We will use these to run pipeline integration tests and canaries.

CONTRIBUTING.md

@@ -33,6 +33,7 @@ and let us know if it's not up-to-date (even better, submit a PR with your  corr
   - [Adding Dependencies](#adding-dependencies)
   - [Finding dependency cycles between packages](#finding-dependency-cycles-between-packages)
   - [Updating all Dependencies](#updating-all-dependencies)
+  - [Running CLI integration tests](#running-cli-integration-tests)
 - [Troubleshooting](#troubleshooting)
 - [Related Repositories](#related-repositories)
 
@@ -472,6 +473,13 @@ To update all dependencies (without bumping major versions):
 3. Run `./scripts/update-dependencies.sh --mode full` (use `--mode semver` to avoid bumping major versions)
 4. Submit a Pull Request.
 
+### Running CLI integration tests
+
+The CLI package (`packages/aws-cdk`) has some integration tests that aren't
+run as part of the regular build, since they have some particular requirements.
+See the [CLI CONTRIBUTING.md file](packages/aws-cdk/CONTRIBUTING.md) for
+more information on running those tests.
+
 ## Troubleshooting
 
 Most build issues can be solved by doing a full clean rebuild:

packages/aws-cdk/CONTRIBUTING.md

@@ -0,0 +1,52 @@
+## Integration Tests
+
+Unit tests are automatically run as part of the regular build. Integration tests
+aren't run automatically since they have nontrivial requirements to run.
+
+### CLI integration tests
+
+CLI tests will exercise a number of common CLI scenarios, and deploy actual
+stacks to your AWS account.
+
+REQUIREMENTS
+
+* All packages have been compiled.
+* Shell has been preloaded with AWS credentials.
+
+Run:
+
+```
+npm run integ-cli
+```
+
+### Init template integration tests
+
+Init template tests will initialize and compile the init templates that the
+CLI ships with.
+
+REQUIREMENTS
+
+* Running on a machine that has all language tools available (JDK, .NET Core,
+  Python installed).
+* All packages have been compiled.
+* All packages have been packaged to their respective languages (`pack.sh`).
+
+Run:
+
+```
+npm run integ-init
+```
+
+## Integration test modes
+
+These two sets of integration tests have 3 running modes:
+
+- Developer mode, when called through `npm run`. Will use the source tree.
+- Integration test, when called from a directory with the build artifacts
+  (the `dist` directory).
+- Canaries, when called with `IS_CANARY=true`. Will use the build artifacts
+  up on the respective package managers.
+
+The integration test and canary modes are used in the CDK publishing pipeline
+and the CDK canaries, respectively. You wouldn't normally need to run
+them directly that way.

packages/aws-cdk/lib/init-templates/sample-app/python/hello/hello_stack.py

@@ -6,7 +6,7 @@
     core
 )
 
-from hello_construct import HelloConstruct
+from .hello_construct import HelloConstruct
 
 
 class MyStack(core.Stack):

packages/aws-cdk/package.json

@@ -15,7 +15,9 @@
     "test": "cdk-test",
     "package": "cdk-package",
     "build+test+package": "npm run build+test && npm run package",
-    "build+test": "npm run build && npm test"
+    "build+test": "npm run build && npm test",
+    "integ-cli": "CDK_REPO=$PWD/../.. test/integ/cli/test.sh",
+    "integ-init": "CDK_DIST=$PWD/../../dist test/integ/init/test-all.sh"
   },
   "cdk-build": {
     "pre": [

packages/aws-cdk/test/integ/cli/app/.gitignore

@@ -0,0 +1 @@
+!*.js

packages/aws-cdk/test/integ/cli/app/app.js

@@ -0,0 +1,126 @@
+const path = require('path');
+const cdk = require('@aws-cdk/core');
+const ssm = require('@aws-cdk/aws-ssm');
+const iam = require('@aws-cdk/aws-iam');
+const sns = require('@aws-cdk/aws-sns');
+const lambda = require('@aws-cdk/aws-lambda');
+const docker = require('@aws-cdk/aws-ecr-assets');
+
+class MyStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+    new sns.Topic(this, 'topic');
+
+    if (cdk.AvailabilityZoneProvider) { // <= 0.34.0
+      new cdk.AvailabilityZoneProvider(this).availabilityZones;
+    } else if (cdk.Context) { // <= 0.35.0
+      cdk.Context.getAvailabilityZones(this);
+    } else {
+      this.availabilityZones;
+    }
+
+    const parameterName = '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2';
+    getSsmParameterValue(this, parameterName);
+  }
+}
+
+function getSsmParameterValue(scope, parameterName) {
+  return ssm.StringParameter.valueFromLookup(scope, parameterName);
+}
+
+class YourStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+    new sns.Topic(this, 'topic1');
+    new sns.Topic(this, 'topic2');
+  }
+}
+
+class IamStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    new iam.Role(this, 'SomeRole', {
+      assumedBy: new iam.ServicePrincipal('ec2.amazon.aws.com')
+    });
+  }
+}
+
+class ProvidingStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    this.topic = new sns.Topic(this, 'BogusTopic'); // Some filler
+  }
+}
+
+
+
+class ConsumingStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    new sns.Topic(this, 'BogusTopic');  // Some filler
+    new cdk.CfnOutput(this, 'IConsumedSomething', { value: props.providingStack.topic.topicArn });
+  }
+}
+
+class MissingSSMParameterStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    const parameterName = this.node.tryGetContext('test:ssm-parameter-name');
+    if (parameterName) {
+      const param = getSsmParameterValue(this, parameterName);
+      new iam.Role(this, 'PhonyRole', { assumedBy: new iam.AccountPrincipal(param) });
+    }
+  }
+}
+
+class LambdaStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    const fn = new lambda.Function(this, 'my-function', {
+      code: lambda.Code.asset(path.join(__dirname, 'lambda')),
+      runtime: lambda.Runtime.NODEJS_8_10,
+      handler: 'index.handler'
+    });
+
+    new cdk.CfnOutput(this, 'FunctionArn', { value: fn.functionArn });
+  }
+}
+
+class DockerStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    new docker.DockerImageAsset(this, 'image', {
+      directory: path.join(__dirname, 'docker')
+    });
+  }
+}
+
+const stackPrefix = process.env.STACK_NAME_PREFIX || 'cdk-toolkit-integration';
+
+const app = new cdk.App();
+
+const defaultEnv = {
+  account: process.env.CDK_DEFAULT_ACCOUNT,
+  region: process.env.CDK_DEFAULT_REGION
+};
+
+// Deploy all does a wildcard ${stackPrefix}-test-*
+new MyStack(app, `${stackPrefix}-test-1`, { env: defaultEnv });
+new YourStack(app, `${stackPrefix}-test-2`);
+// Not included in wildcard
+new IamStack(app, `${stackPrefix}-iam-test`);
+const providing = new ProvidingStack(app, `${stackPrefix}-order-providing`);
+new ConsumingStack(app, `${stackPrefix}-order-consuming`, { providingStack: providing });
+
+new MissingSSMParameterStack(app, `${stackPrefix}-missing-ssm-parameter`, { env: defaultEnv });
+
+new LambdaStack(app, `${stackPrefix}-lambda`);
+new DockerStack(app, `${stackPrefix}-docker`);
+
+app.synth();

packages/aws-cdk/test/integ/cli/app/cdk.json

@@ -0,0 +1,4 @@
+{
+  "app": "node app.js",
+  "versionReporting": false
+}

packages/aws-cdk/test/integ/cli/app/docker/Dockerfile

@@ -0,0 +1,2 @@
+FROM alpine
+

packages/aws-cdk/test/integ/cli/app/lambda/index.js

@@ -0,0 +1,4 @@
+exports.handler = async function(event) {
+  const response = require('./response.json');
+  return response;
+};

packages/aws-cdk/test/integ/cli/app/lambda/response.json

@@ -0,0 +1,3 @@
+{
+  "response": "hello, dear asset!"
+}

packages/aws-cdk/test/integ/cli/common.bash

@@ -0,0 +1,147 @@
+set -eu
+scriptdir=$(cd $(dirname $0) && pwd)
+source $scriptdir/../common/util.bash
+
+CDK_REPO="${CDK_REPO:-}"
+
+# If CDK_REPO is set to point to the root of the CDK source repository
+# the CLI and modules will be taken from there. Otherwise, we will honor
+# the usual IS_CANARY flag for integration test-vs-canary mode switch.
+if [ -z "${CDK_REPO}" -a -z "${TESTS_PREPARED:-}" ]; then
+    prepare_toolkit
+    preload_npm_packages
+
+    # Only do this once
+    export TESTS_PREPARED=1
+fi
+
+if [[ "${CDK_REPO}" != "" ]]; then
+    export CDK_REPO=$(cd $CDK_REPO && pwd)
+    alias cdk=$CDK_REPO/packages/aws-cdk/bin/cdk
+fi
+
+if [[ -z "${CREDS_SET:-}" ]]; then
+    # Check that credentials are configured
+    aws sts get-caller-identity > /dev/null
+    export CREDS_SET=1
+fi
+
+cd ${scriptdir}
+
+if ${IS_CANARY:-false}; then
+  export STACK_NAME_PREFIX=cdk-toolkit-canary
+else
+  export STACK_NAME_PREFIX=cdk-toolkit-integration
+fi
+
+function cleanup_stack() {
+  local stack_arn=$1
+  echo "| ensuring ${stack_arn} is cleaned up"
+  if aws cloudformation describe-stacks --stack-name ${stack_arn} 2> /dev/null; then
+    aws cloudformation delete-stack --stack-name ${stack_arn}
+  fi
+}
+
+function cleanup() {
+  cleanup_stack ${STACK_NAME_PREFIX}-test-1
+  cleanup_stack ${STACK_NAME_PREFIX}-test-2
+  cleanup_stack ${STACK_NAME_PREFIX}-iam-test
+}
+
+function install_dep() {
+  local dep=$1
+
+  if [ -n "${CDK_REPO}" ]; then
+    mkdir -p node_modules/@aws-cdk
+    local source="${CDK_REPO}/packages/${dep}"
+    local target="$PWD/node_modules/${dep}"
+    echo "| symlinking dependency ${target} => ${source}"
+    ln -s ${source} ${target}
+  else
+    echo "| installing dependency ${dep}"
+    npm i --no-save ${dep}
+  fi
+}
+
+function setup() {
+  cleanup
+  rm -rf /tmp/cdk-integ-test
+  mkdir -p /tmp/cdk-integ-test
+  cp -R app/* /tmp/cdk-integ-test
+  cd /tmp/cdk-integ-test
+
+  if [ -n "${CDK_REPO}" ]; then
+    local cdk_bin="${CDK_REPO}/packages/aws-cdk/bin"
+    echo "| adding ${cdk_bin} to PATH"
+    export PATH=${cdk_bin}:$PATH
+  fi
+
+  install_dep @aws-cdk/core
+  install_dep @aws-cdk/aws-sns
+  install_dep @aws-cdk/aws-iam
+  install_dep @aws-cdk/aws-lambda
+  install_dep @aws-cdk/aws-ssm
+  install_dep @aws-cdk/aws-ecr-assets
+
+  echo "| setup complete at: $PWD"
+  echo "| 'cdk' is: $(which cdk)"
+}
+
+function fail() {
+  echo "❌  $@"
+  exit 1
+}
+
+function assert_diff() {
+  local test=$1
+  local actual=$2
+  local expected=$3
+
+  diff -w ${actual} ${expected} || {
+    echo
+    echo "+-----------"
+    echo "| expected:"
+    cat ${expected}
+    echo "|--"
+    echo
+    echo "+-----------"
+    echo "| actual:"
+    cat ${actual}
+    echo "|--"
+    echo
+    fail "assertion failed. ${test}"
+  }
+}
+
+function assert() {
+  local command="$1"
+
+  local expected=$(mktemp)
+  local actual=$(mktemp)
+
+  echo "| running ${command}"
+
+  eval "$command" > ${actual} || {
+    fail "command ${command} non-zero exit code"
+  }
+
+  cat > ${expected}
+
+  assert_diff "command: ${command}" "${actual}" "${expected}"
+}
+
+function assert_lines() {
+  local data="$1"
+  local expected="$2"
+  echo "| assert that last command returned ${expected} line(s)"
+
+  local lines="$(echo "${data}" | wc -l)"
+  if [ "${lines}" -ne "${expected}" ]; then
+    echo "${data}"
+    fail "response has ${lines} lines and we expected ${expected} lines to be returned"
+  fi
+}
+
+function strip_color_codes() {
+    perl -pe 's/\e\[?.*?[\@-~]//g'
+}