chore: change CLI integ tests from Bash to TypeScript

Convert most tests from Bash to TypeScript. It's much more comfortable
to write complex tests in a programming language than bash. Plus, `jest`
is a better test runner than bash, too.

Even though tests are now written in TypeScript, this does not
conceptually change their SUT! They are still testing the CLI via
running it as a subprocess, they are NOT reaching directly into the CLI
code to test its private methods. It's just that setup/teardown/error
handling/assertions/AWS calls are much more convenient to do in a real
programming language.

Compilation of the tests is done as part of the normal package build, at
which point it is using the dependencies brought in by the containing
`aws-cdk` package's `package.json`.

When run in a non-develompent repo (as done during integ tests or canary
runs), the required dependencies are brought in just-in-time via
`test-jest.sh`. Using a separate `package.json` just for the tests
makes it a package in its own right for `yarn`, and I didn't want
to go there.

packages/aws-cdk/test/integ/cli/.gitignore

@@ -1 +1,4 @@
 cdk.context.json
+node_modules
+*.d.ts
+*.js

packages/aws-cdk/test/integ/cli/README.md

@@ -6,9 +6,31 @@ CLI on a simple JavaScript CDK app (stored in `app/`).
 ## Entry point
 
 ```
-./test.sh
+../run-against-repo ./test.sh
 ```
 
+## Adding tests
+
+Older tests were written in bash; new tests should be written in
+TypeScript/Jest, that is much more comfortable to write in.
+
+Even though tests are now written in TypeScript, this does not
+conceptually change their SUT! They are still testing the CLI via
+running it as a subprocess, they are NOT reaching directly into the CLI
+code to test its private methods. It's just that setup/teardown/error
+handling/assertions/AWS calls are much more convenient to do in a real
+programming language.
+
+Compilation of the tests is done as part of the normal package build, at
+which point it is using the dependencies brought in by the containing
+`aws-cdk` package's `package.json`.
+
+When run in a non-develompent repo (as done during integ tests or canary runs),
+the required dependencies are brought in just-in-time via `test-jest.sh`. Any
+new dependencies added for the tests should be added there as well. But, better
+yet, don't add any dependencies at all. You shouldn't need to, these tests
+are simple.
+
 ## Configuration
 
 AWS credentials must be configured.

packages/aws-cdk/test/integ/cli/aws-helpers.ts

@@ -0,0 +1,143 @@
+import * as AWS from 'aws-sdk';
+import { log } from './cdk-helpers';
+
+interface Env {
+  account: string;
+  region: string;
+}
+
+export let testEnv = async (): Promise<Env> => {
+  const response = await new AWS.STS().getCallerIdentity().promise();
+
+  const ret: Env = {
+    account: response.Account!,
+    region: process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? 'us-east-1',
+  };
+
+  testEnv = () => Promise.resolve(ret);
+  return ret;
+};
+
+export const cloudFormation = makeAwsCaller(AWS.CloudFormation);
+export const sns = makeAwsCaller(AWS.SNS);
+export const iam = makeAwsCaller(AWS.IAM);
+export const lambda = makeAwsCaller(AWS.Lambda);
+export const sts = makeAwsCaller(AWS.STS);
+
+/**
+ * Perform an AWS call from nothing
+ *
+ * Create the correct client, do the call and resole the promise().
+ */
+async function awsCall<
+  A extends AWS.Service,
+  B extends keyof ServiceCalls<A>,
+>(ctor: new (config: any) => A, call: B, request: First<ServiceCalls<A>[B]>): Promise<Second<ServiceCalls<A>[B]>> {
+  const env = await testEnv();
+  const cfn = new ctor({ region: env.region });
+  const response = cfn[call](request);
+  try {
+    return await response.promise();
+  } catch (e) {
+    const newErr = new Error(`${call}(${JSON.stringify(request)}): ${e.message}`);
+    (newErr as any).code = e.code;
+    throw newErr;
+  }
+}
+
+/**
+ * Factory function to invoke 'awsCall' for specific services.
+ *
+ * Not strictly necessary but calling this replaces a whole bunch of annoying generics you otherwise have to type:
+ *
+ * ```ts
+ * export function cloudFormation<
+ *   C extends keyof ServiceCalls<AWS.CloudFormation>,
+ * >(call: C, request: First<ServiceCalls<AWS.CloudFormation>[C]>): Promise<Second<ServiceCalls<AWS.CloudFormation>[C]>> {
+ *   return awsCall(AWS.CloudFormation, call, request);
+ * }
+ * ```
+ */
+function makeAwsCaller<A extends AWS.Service>(ctor: new (config: any) => A) {
+  return <B extends keyof ServiceCalls<A>>(call: B, request: First<ServiceCalls<A>[B]>): Promise<Second<ServiceCalls<A>[B]>> => {
+    return awsCall(ctor, call, request);
+  };
+}
+
+type ServiceCalls<T> = NoNayNever<SimplifiedService<T>>;
+// Map ever member in the type to the important AWS call overload, or to 'never'
+type SimplifiedService<T> = {[k in keyof T]: AwsCallIO<T[k]>};
+// Remove all 'never' types from an object type
+type NoNayNever<T> = Pick<T, {[k in keyof T]: T[k] extends never ? never : k }[keyof T]>;
+
+// Because of the overloads an AWS handler type looks like this:
+//
+//   {
+//      (params: INPUTSTRUCT, callback?: ((err: AWSError, data: {}) => void) | undefined): Request<OUTPUT, ...>;
+//      (callback?: ((err: AWS.AWSError, data: {}) => void) | undefined): AWS.Request<...>;
+//   }
+//
+// Get the first overload and extract the input and output struct types
+type AwsCallIO<T> =
+  T extends {
+    (args: infer INPUT, callback?: ((err: AWS.AWSError, data: any) => void) | undefined): AWS.Request<infer OUTPUT, AWS.AWSError>;
+    (callback?: ((err: AWS.AWSError, data: {}) => void) | undefined): AWS.Request<any, any>;
+  } ? [INPUT, OUTPUT] : never;
+
+type First<T> = T extends [any, any] ? T[0] : never;
+type Second<T> = T extends [any, any] ? T[1] : never;
+
+export async function deleteStacks(...stackNames: string[]) {
+  if (stackNames.length === 0) { return; }
+
+  for (const stackName of stackNames) {
+    await cloudFormation('deleteStack', {
+      StackName: stackName,
+    });
+  }
+
+  await retry(`Deleting ${stackNames}`, afterSeconds(300), async () => {
+    for (const stackName of stackNames) {
+      if (await stackExists(stackName)) {
+        throw new Error(`Delete of '${stackName}' not complete yet`);
+      }
+    }
+  });
+}
+
+export async function stackExists(stackName: string) {
+  try {
+    await cloudFormation('describeStacks', { StackName: stackName });
+    return true;
+  } catch (e) {
+    if (e.message.indexOf('does not exist') > -1) { return false; }
+    throw e;
+  }
+}
+
+export function afterSeconds(seconds: number): Date {
+  return new Date(Date.now() + seconds * 1000);
+}
+
+export async function retry<A>(operation: string, deadline: Date, block: () => Promise<A>): Promise<A> {
+  let i = 0;
+  log(`💈 ${operation}`);
+  while (true) {
+    try {
+      i++;
+      const ret = await block();
+      log(`💈 ${operation}: succeeded after ${i} attempts`);
+      return ret;
+    } catch (e) {
+      if (Date.now() > deadline.getTime()) {
+        throw new Error(`${operation}: did not succeed after ${i} attempts: ${e}`);
+      }
+      log(`⏳ ${operation} (${e.message})`);
+      await sleep(5000);
+    }
+  }
+}
+
+export async function sleep(ms: number) {
+  return new Promise(ok => setTimeout(ok, ms));
+}

packages/aws-cdk/test/integ/cli/bootstrapping.integtest.ts

@@ -0,0 +1,49 @@
+import { cloudFormation, deleteStacks } from './aws-helpers';
+import { cdk, cleanupOldStacks, fullStackName, prepareAppFixture } from './cdk-helpers';
+
+jest.setTimeout(600 * 1000);
+
+beforeEach(async () => {
+  await prepareAppFixture();
+  await cleanupOldStacks();
+});
+
+afterEach(async () => {
+  await cleanupOldStacks();
+});
+
+test('can bootstrap without execution', async () => {
+  const bootstrapStackName = fullStackName('toolkit-stack-1');
+
+  await cdk(['bootstrap', '-v',
+    '--toolkit-stack-name', bootstrapStackName,
+    '--no-execute']);
+  try {
+
+    const resp = await cloudFormation('describeStacks', {
+      StackName: bootstrapStackName,
+    });
+
+    expect(resp.Stacks?.[0].StackStatus).toEqual('REVIEW_IN_PROGRESS');
+  } finally {
+    await deleteStacks(bootstrapStackName);
+  }
+});
+
+test('can bootstrap multiple toolkit stacks', async () => {
+  const bootstrapStackName1 = fullStackName('toolkit-stack-1');
+  const bootstrapStackName2 = fullStackName('toolkit-stack-2');
+  try {
+    // deploy two toolkit stacks into the same environment (see #1416)
+    // one with tags
+    await cdk(['bootstrap', '-v', '--toolkit-stack-name', bootstrapStackName1, '--tags', 'Foo=Bar']);
+    await cdk(['bootstrap', '-v', '--toolkit-stack-name', bootstrapStackName2]);
+
+    const response = await cloudFormation('describeStacks', { StackName: bootstrapStackName1 });
+    expect(response.Stacks?.[0].Tags).toEqual([
+      { Key: 'Foo', Value: 'Bar' },
+    ]);
+  } finally {
+    await deleteStacks(bootstrapStackName1, bootstrapStackName2);
+  }
+});