feat(aws-s3-sqs): New aws-s3-sqs pattern implementation (awslabs#27) (a…

…wslabs#105) * feat(aws-s3-sqs): New aws-s3-sqs pattern implementation (awslabs#27) * fix(aws-s3-sqs): Added logic to suppress cfn_nag warnings for S3 bucket notifications. * fix(aws-s3-sqs): Code review fixes Moved addCfnNagSuppress to core. Exposed encryption key properties in s3-sqs pattern. Improved integration and unit tests. * fix(aws-s3-sqs): Added viperlight ignore for CDK generated KMS key resource policy that allows the root account to access it. * fix(aws-s3-sqs) Executed lint with fix option to correct the indentation to use 2 spaces. * fix(aws-s3-sqs) Added documentation describing the need for customer managed CMK to encrypt the SQS Queue.
tabdunabi · Dec 16, 2020 · 80bce76 · 80bce76
1 parent d5f3d8e
commit 80bce76
Show file tree

Hide file tree

Showing 22 changed files with 3,140 additions and 41 deletions.
diff --git a/.viperlightignore b/.viperlightignore
@@ -34,4 +34,5 @@ source/patterns/@aws-solutions-constructs/aws-apigateway-iot/test/integ.override
 source/patterns/@aws-solutions-constructs/aws-apigateway-iot/test/test.apigateway-iot.test.ts:29
 source/patterns/@aws-solutions-constructs/aws-apigateway-iot/test/integ.override_auth_api_keys.expected.json:253
 source/patterns/@aws-solutions-constructs/aws-cloudfront-s3/test/test.cloudfront-s3.test.ts:106
-source/patterns/@aws-solutions-constructs/core/test/cloudfront-distribution-s3-helper.test.ts:200
+source/patterns/@aws-solutions-constructs/core/test/cloudfront-distribution-s3-helper.test.ts:200
+source/patterns/@aws-solutions-constructs/aws-s3-sqs/test/test.s3-sqs.test.ts:251
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-lambda/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-s3-lambda/lib/index.ts
@@ -14,9 +14,9 @@
 import * as lambda from '@aws-cdk/aws-lambda';
 import * as s3 from '@aws-cdk/aws-s3';
 import { Construct, Stack } from '@aws-cdk/core';
-import * as iam from '@aws-cdk/aws-iam';
 import * as defaults from '@aws-solutions-constructs/core';
 import { S3EventSourceProps, S3EventSource } from '@aws-cdk/aws-lambda-event-sources';
+import {addCfnNagS3BucketNotificationRulesToSuppress} from "@aws-solutions-constructs/core";
 
 /**
  * @summary The properties for the S3ToLambda class.
@@ -89,38 +89,7 @@ export class S3ToLambda extends Construct {
     this.lambdaFunction.addEventSource(new S3EventSource(bucket,
       defaults.S3EventSourceProps(props.s3EventSourceProps)));
 
-    this.addCfnNagSuppress();
+    // Suppress cfn-nag rules that generate warns for S3 bucket notification CDK resources
+    addCfnNagS3BucketNotificationRulesToSuppress(Stack.of(this), 'BucketNotificationsHandler050a0587b7544547bf325f094a3db834');
   }
-
-  private addCfnNagSuppress() {
-    const root = Stack.of(this);
-    const logicalId = 'BucketNotificationsHandler050a0587b7544547bf325f094a3db834';
-    const notificationsResourceHandler = root.node.tryFindChild(logicalId) as lambda.Function;
-    const notificationsResourceHandlerRoleRole = notificationsResourceHandler.node.findChild('Role') as iam.Role;
-    const notificationsResourceHandlerRolePolicy = notificationsResourceHandlerRoleRole.node.findChild('DefaultPolicy') as iam.Policy;
-
-    // Extract the CfnFunction from the Function
-    const fnResource = notificationsResourceHandler.node.findChild('Resource') as lambda.CfnFunction;
-
-    fnResource.cfnOptions.metadata = {
-      cfn_nag: {
-        rules_to_suppress: [{
-          id: 'W58',
-          reason: `Lambda function has permission to write CloudWatch Logs via AWSLambdaBasicExecutionRole policy attached to the lambda role`
-        }]
-      }
-    };
-
-    // Extract the CfnPolicy from the iam.Policy
-    const policyResource = notificationsResourceHandlerRolePolicy.node.findChild('Resource') as iam.CfnPolicy;
-
-    policyResource.cfnOptions.metadata = {
-      cfn_nag: {
-        rules_to_suppress: [{
-          id: 'W12',
-          reason: `Bucket resource is '*' due to circular dependency with bucket and role creation at the same time`
-        }]
-      }
-    };
-  }
-}
+}
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.eslintignore b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.eslintignore
@@ -0,0 +1,5 @@
+lib/*.js
+test/*.js
+*.d.ts
+coverage
+test/lambda/index.js
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.gitignore b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.gitignore
@@ -0,0 +1,15 @@
+lib/*.js
+test/*.js
+*.js.map
+*.d.ts
+node_modules
+*.generated.ts
+dist
+.jsii
+
+.LAST_BUILD
+.nyc_output
+coverage
+.nycrc
+.LAST_PACKAGE
+*.snk
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.npmignore b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/.npmignore
@@ -0,0 +1,21 @@
+# Exclude typescript source and config
+*.ts
+tsconfig.json
+coverage
+.nyc_output
+*.tgz
+*.snk
+*.tsbuildinfo
+
+# Include javascript files and typescript declarations
+!*.js
+!*.d.ts
+
+# Exclude jsii outdir
+dist
+
+# Include .jsii
+!.jsii
+
+# Include .jsii
+!.jsii
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/README.md b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/README.md
@@ -0,0 +1,98 @@
+# aws-s3-sqs module
+<!--BEGIN STABILITY BANNER-->
+
+---
+
+![Stability: Experimental](https://img.shields.io/badge/stability-Experimental-important.svg?style=for-the-badge)
+
+> All classes are under active development and subject to non-backward compatible changes or removal in any
+> future version. These are not subject to the [Semantic Versioning](https://semver.org/) model.
+> This means that while you may use them, you may need to update your source code when upgrading to a newer version of this package.
+
+---
+<!--END STABILITY BANNER-->
+
+| **Reference Documentation**:| <span style="font-weight: normal">https://docs.aws.amazon.com/solutions/latest/constructs/</span>|
+|:-------------|:-------------|
+<div style="height:8px"></div>
+
+| **Language**     | **Package**        |
+|:-------------|-----------------|
+|![Python Logo](https://docs.aws.amazon.com/cdk/api/latest/img/python32.png) Python|`aws_solutions_constructs.aws_s3_sqs`|
+|![Typescript Logo](https://docs.aws.amazon.com/cdk/api/latest/img/typescript32.png) Typescript|`@aws-solutions-constructs/aws-s3-sqs`|
+|![Java Logo](https://docs.aws.amazon.com/cdk/api/latest/img/java32.png) Java|`software.amazon.awsconstructs.services.s3sqs`|
+
+This AWS Solutions Construct implements an Amazon S3 Bucket that is configured to send notifications to an Amazon SQS queue.
+
+
+Here is a minimal deployable pattern definition in Typescript:
+
+``` typescript
+import {S3ToSqs} from "@aws-solutions-constructs/aws-s3-sqs";
+
+new S3ToSqs(stack, 'S3ToSQSPattern', {});
+```
+
+## Initializer
+
+``` text
+new S3ToSqs(scope: Construct, id: string, props: S3ToSqsProps);
+```
+
+_Parameters_
+
+* scope [`Construct`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_core.Construct.html)
+* id `string`
+* props [`S3ToSqsProps`](#pattern-construct-props)
+
+## Pattern Construct Props
+
+| **Name**     | **Type**        | **Description** |
+|:-------------|:----------------|-----------------|
+|existingBucketObj?|[`s3.Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.IBucket.html)|Existing instance of S3 Bucket object, if this is set then the bucketProps is ignored.|
+|bucketProps?|[`s3.BucketProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.BucketProps.html)|User provided props to override the default props for the S3 Bucket.|
+|s3EventTypes?|[`s3.EventType[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.EventType.html)|The S3 event types that will trigger the notification. Defaults to s3.EventType.OBJECT_CREATED.|
+|s3EventFilters?|[`s3.NotificationKeyFilter[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.NotificationKeyFilter.html)|S3 object key filter rules to determine which objects trigger this event. If not specified no filter rules will be applied.|
+|existingQueueObj?|[`sqs.Queue`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-sqs.Queue.html)|An optional, existing SQS queue to be used instead of the default queue. If an existing queue is provided, the `queueProps` property will be ignored. If the SQS queue is encrypted, the KMS key utilized for encryption must be a customer managed CMK.|
+|queueProps?|[`sqs.QueueProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-sqs.QueueProps.html)|Optional user-provided props to override the default props for the SQS queue.|
+|deadLetterQueueProps?|[`sqs.QueueProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-sqs.QueueProps.html)|Optional user-provided props to override the default props for the dead letter SQS queue.|
+|deployDeadLetterQueue?|`boolean`|Whether to create a secondary queue to be used as a dead letter queue. Defaults to true.|
+|maxReceiveCount?|`number`|The number of times a message can be unsuccessfully dequeued before being moved to the dead letter queue. Defaults to 15.|
+|enableEncryptionWithCustomerManagedKey?|`boolean`|Use a KMS Key, either managed by this CDK app, or imported. If importing an encryption key, it must be specified in the encryptionKey property for this construct.|
+|encryptionKey?|[`kms.Key`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kms.Key.html)|An optional, imported encryption key to encrypt the SQS queue.|
+|encryptionKeyProps?|[`kms.KeyProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kms.KeyProps.html)|An optional, user provided properties to override the default properties for the KMS encryption key.|
+
+## Pattern Properties
+
+| **Name**     | **Type**        | **Description** |
+|:-------------|:----------------|-----------------|
+|sqsQueue|[`sqs.Queue`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-sqs.Queue.html)|Returns an instance of the SQS queue created by the pattern.|
+|deadLetterQueue?|[`sqs.Queue`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-sqs.Queue.html)|Returns an instance of the dead-letter SQS queue created by the pattern.|
+|encryptionKey|[`kms.IKey`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kms.IKey.html)|Returns an instance of kms.Key used for the SQS queue.|
+|s3Bucket?|[`s3.Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.Bucket.html)|Returns an instance of the s3.Bucket created by the construct|
+|s3LoggingBucket?|[`s3.Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.Bucket.html)|Returns an instance of s3.Bucket created by the construct as the logging bucket for the primary bucket.|
+
+## Default settings
+
+Out of the box implementation of the Construct without any override will set the following defaults:
+
+### Amazon S3 Bucket
+* Configure Access logging for S3 Bucket
+* Enable server-side encryption for S3 Bucket using AWS managed KMS Key
+* Enforce encryption of data in transit
+* Turn on the versioning for S3 Bucket
+* Don't allow public access for S3 Bucket
+* Retain the S3 Bucket when deleting the CloudFormation stack
+* Applies Lifecycle rule to move noncurrent object versions to Glacier storage after 90 days
+
+### Amazon SQS Queue
+* Configure least privilege access permissions for SQS Queue
+* Deploy SQS dead-letter queue for the source SQS Queue
+* Enable server-side encryption for SQS Queue using Customer managed KMS Key
+* Enforce encryption of data in transit
+
+## Architecture
+![Architecture Diagram](architecture.png)
+
+***
+&copy; Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/architecture.png b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/architecture.png
diff --git a/source/patterns/@aws-solutions-constructs/aws-s3-sqs/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-s3-sqs/lib/index.ts
@@ -0,0 +1,175 @@
+/**
+ *  Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ *  with the License. A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+ *  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+ *  and limitations under the License.
+ */
+
+// Imports
+import * as kms from '@aws-cdk/aws-kms';
+import * as sqs from '@aws-cdk/aws-sqs';
+import * as s3 from '@aws-cdk/aws-s3';
+import * as defaults from '@aws-solutions-constructs/core';
+import * as s3n from '@aws-cdk/aws-s3-notifications';
+import { Construct, Stack } from '@aws-cdk/core';
+import {addCfnNagS3BucketNotificationRulesToSuppress} from "@aws-solutions-constructs/core";
+
+/**
+ * @summary The properties for the S3ToSqs class.
+ */
+export interface S3ToSqsProps {
+    /**
+     * Existing instance of S3 Bucket object, if this is set then the bucketProps is ignored.
+     *
+     * @default - None
+     */
+    readonly existingBucketObj?: s3.Bucket,
+    /**
+     * User provided props to override the default props for the S3 Bucket.
+     *
+     * @default - Default props are used
+     */
+    readonly bucketProps?: s3.BucketProps,
+    /**
+     * The S3 event types that will trigger the notification.
+     *
+     * @default - If not specified the s3.EventType.OBJECT_CREATED event will trigger the notification.
+     */
+    readonly s3EventTypes?: s3.EventType[];
+    /**
+     * S3 object key filter rules to determine which objects trigger this event.
+     *
+     * @default - If not specified no filter rules will be applied.
+     */
+    readonly s3EventFilters?: s3.NotificationKeyFilter[]
+    /**
+     * Existing instance of SQS queue object, if this is set then queueProps is ignored.
+     *
+     * @default - Default props are used
+     */
+    readonly existingQueueObj?: sqs.Queue,
+    /**
+     * Optional user provided properties
+     *
+     * @default - Default props are used
+     */
+    readonly queueProps?: sqs.QueueProps,
+    /**
+     * Optional user provided properties for the dead letter queue
+     *
+     * @default - Default props are used
+     */
+    readonly deadLetterQueueProps?: sqs.QueueProps,
+    /**
+     * Whether to deploy a secondary queue to be used as a dead letter queue.
+     *
+     * @default - true.
+     */
+    readonly deployDeadLetterQueue?: boolean,
+    /**
+     * The number of times a message can be unsuccessfully dequeued before being moved to the dead-letter queue.
+     *
+     * @default - required field if deployDeadLetterQueue=true.
+     */
+    readonly maxReceiveCount?: number
+    /**
+     * Use a KMS Key, either managed by this CDK app, or imported. If importing an encryption key, it must be specified in
+     * the encryptionKey property for this construct.
+     *
+     * @default - true (encryption enabled, managed by this CDK app).
+     */
+    readonly enableEncryptionWithCustomerManagedKey?: boolean
+    /**
+     * An optional, imported encryption key to encrypt the SQS queue.
+     *
+     * @default - not specified.
+     */
+    readonly encryptionKey?: kms.Key,
+    /**
+     * Optional user-provided props to override the default props for the encryption key.
+     *
+     * @default - Default props are used.
+     */
+    readonly encryptionKeyProps?: kms.KeyProps
+}
+
+/**
+ * @summary The S3ToSqs class.
+ */
+export class S3ToSqs extends Construct {
+    public readonly sqsQueue: sqs.Queue;
+    public readonly deadLetterQueue?: sqs.DeadLetterQueue;
+    public readonly s3Bucket?: s3.Bucket;
+    public readonly s3LoggingBucket?: s3.Bucket;
+    public readonly encryptionKey?: kms.IKey;
+
+    /**
+     * @summary Constructs a new instance of the S3ToSqs class.
+     * @param {cdk.App} scope - represents the scope for all the resources.
+     * @param {string} id - this is a a scope-unique id.
+     * @param {S3ToSqsProps} props - user provided props for the construct.
+     * @since 0.8.0
+     * @access public
+     */
+    constructor(scope: Construct, id: string, props: S3ToSqsProps) {
+      super(scope, id);
+      let bucket: s3.Bucket;
+      let enableEncryptionParam = props.enableEncryptionWithCustomerManagedKey;
+
+      if (props.enableEncryptionWithCustomerManagedKey === undefined ||
+          props.enableEncryptionWithCustomerManagedKey === true) {
+        enableEncryptionParam = true;
+      }
+
+      // Setup the S3 bucket
+      if (!props.existingBucketObj) {
+        [this.s3Bucket, this.s3LoggingBucket] = defaults.buildS3Bucket(this, {
+          bucketProps: props.bucketProps
+        });
+        bucket = this.s3Bucket;
+      } else {
+        bucket = props.existingBucketObj;
+      }
+
+      // Setup the dead letter queue, if applicable
+      this.deadLetterQueue = defaults.buildDeadLetterQueue(this, {
+        existingQueueObj: props.existingQueueObj,
+        deployDeadLetterQueue: props.deployDeadLetterQueue,
+        deadLetterQueueProps: props.deadLetterQueueProps,
+        maxReceiveCount: props.maxReceiveCount
+      });
+      // Setup the queue
+      [this.sqsQueue, this.encryptionKey] = defaults.buildQueue(this, 'queue', {
+        existingQueueObj: props.existingQueueObj,
+        queueProps: props.queueProps,
+        deadLetterQueue: this.deadLetterQueue,
+        enableEncryptionWithCustomerManagedKey: enableEncryptionParam,
+        encryptionKey: props.encryptionKey
+      });
+
+      // Setup the S3 bucket event types
+      let s3EventTypes;
+      if (!props.s3EventTypes) {
+        s3EventTypes = defaults.defaultS3NotificationEventTypes;
+      } else {
+        s3EventTypes = props.s3EventTypes;
+      }
+
+      // Setup the S3 bucket event filters
+      let s3Eventfilters: s3.NotificationKeyFilter[] = [];
+      if (props.s3EventFilters) {
+        s3Eventfilters = props.s3EventFilters;
+      }
+
+      // Setup the S3 bucket event notifications
+      s3EventTypes.forEach(type => bucket.addEventNotification(type, new s3n.SqsDestination(this.sqsQueue), ...s3Eventfilters));
+
+      addCfnNagS3BucketNotificationRulesToSuppress(Stack.of(this), 'BucketNotificationsHandler050a0587b7544547bf325f094a3db834');
+    }
+}