Skip to content
This repository has been archived by the owner on Jun 13, 2024. It is now read-only.

Commit

Permalink
feat(s3): Introduce S3 Inventory (aws#9102)
Browse files Browse the repository at this point in the history
Resolves aws#8154

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
arnulfojr authored Aug 3, 2020
1 parent 22fa426 commit b0f359e
Show file tree
Hide file tree
Showing 5 changed files with 490 additions and 3 deletions.
48 changes: 48 additions & 0 deletions packages/@aws-cdk/aws-s3/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,54 @@ const bucket = new Bucket(this, 'MyBucket', {

[S3 Server access logging]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html

### S3 Inventory

An [inventory](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html) contains a list of the objects in the source bucket and metadata for each object. The inventory lists are stored in the destination bucket as a CSV file compressed with GZIP, as an Apache optimized row columnar (ORC) file compressed with ZLIB, or as an Apache Parquet (Parquet) file compressed with Snappy.

You can configure multiple inventory lists for a bucket. You can configure what object metadata to include in the inventory, whether to list all object versions or only current versions, where to store the inventory list file output, and whether to generate the inventory on a daily or weekly basis.

```ts
const inventoryBucket = new s3.Bucket(this, 'InventoryBucket');

const dataBucket = new s3.Bucket(this, 'DataBucket', {
inventories: [
{
frequency: s3.InventoryFrequency.DAILY,
includeObjectVersions: s3.InventoryObjectVersion.CURRENT,
destination: {
bucket: inventoryBucket,
},
},
{
frequency: s3.InventoryFrequency.WEEKLY,
includeObjectVersions: s3.InventoryObjectVersion.ALL,
destination: {
bucket: inventoryBucket,
prefix: 'with-all-versions',
},
}
]
});
```

If the destination bucket is created as part of the same CDK application, the necessary permissions will be automatically added to the bucket policy.
However, if you use an imported bucket (i.e `Bucket.fromXXX()`), you'll have to make sure it contains the following policy document:

```
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "InventoryAndAnalyticsExamplePolicy",
"Effect": "Allow",
"Principal": { "Service": "s3.amazonaws.com" },
"Action": "s3:PutObject",
"Resource": ["arn:aws:s3:::destinationBucket/*"]
}
]
}
```

### Website redirection

You can use the two following properties to specify the bucket [redirection policy]. Please note that these methods cannot both be applied to the same bucket.
Expand Down
194 changes: 193 additions & 1 deletion packages/@aws-cdk/aws-s3/lib/bucket.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { EOL } from 'os';
import * as events from '@aws-cdk/aws-events';
import * as iam from '@aws-cdk/aws-iam';
import * as kms from '@aws-cdk/aws-kms';
import { Construct, Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core';
import { EOL } from 'os';
import { BucketPolicy } from './bucket-policy';
import { IBucketNotificationDestination } from './destination';
import { BucketNotifications } from './notifications-resource';
Expand Down Expand Up @@ -828,6 +828,130 @@ export interface RedirectTarget {
readonly protocol?: RedirectProtocol;
}

/**
* All supported inventory list formats.
*/
export enum InventoryFormat {
/**
* Generate the inventory list as CSV.
*/
CSV = 'CSV',
/**
* Generate the inventory list as Parquet.
*/
PARQUET = 'Parquet',
/**
* Generate the inventory list as Parquet.
*/
ORC = 'ORC',
}

/**
* All supported inventory frequencies.
*/
export enum InventoryFrequency {
/**
* A report is generated every day.
*/
DAILY = 'Daily',
/**
* A report is generated every Sunday (UTC timezone) after the initial report.
*/
WEEKLY = 'Weekly'
}

/**
* Inventory version support.
*/
export enum InventoryObjectVersion {
/**
* Includes all versions of each object in the report.
*/
ALL = 'All',
/**
* Includes only the current version of each object in the report.
*/
CURRENT = 'Current',
}

/**
* The destination of the inventory.
*/
export interface InventoryDestination {
/**
* Bucket where all inventories will be saved in.
*/
readonly bucket: IBucket;
/**
* The prefix to be used when saving the inventory.
*
* @default - No prefix.
*/
readonly prefix?: string;
/**
* The account ID that owns the destination S3 bucket.
* If no account ID is provided, the owner is not validated before exporting data.
* It's recommended to set an account ID to prevent problems if the destination bucket ownership changes.
*
* @default - No account ID.
*/
readonly bucketOwner?: string;
}

/**
* Specifies the inventory configuration of an S3 Bucket.
*
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html
*/
export interface Inventory {
/**
* The destination of the inventory.
*/
readonly destination: InventoryDestination;
/**
* The inventory will only include objects that meet the prefix filter criteria.
*
* @default - No objects prefix
*/
readonly objectsPrefix?: string;
/**
* The format of the inventory.
*
* @default InventoryFormat.CSV
*/
readonly format?: InventoryFormat;
/**
* Whether the inventory is enabled or disabled.
*
* @default true
*/
readonly enabled?: boolean;
/**
* The inventory configuration ID.
*
* @default - generated ID.
*/
readonly inventoryId?: string;
/**
* Frequency at which the inventory should be generated.
*
* @default InventoryFrequency.WEEKLY
*/
readonly frequency?: InventoryFrequency;
/**
* If the inventory should contain all the object versions or only the current one.
*
* @default InventoryObjectVersion.ALL
*/
readonly includeObjectVersions?: InventoryObjectVersion;
/**
* A list of optional fields to be included in the inventory result.
*
* @default - No optional fields.
*/
readonly optionalFields?: string[];
}

export interface BucketProps {
/**
* The kind of server-side encryption to apply to this bucket.
Expand Down Expand Up @@ -966,6 +1090,15 @@ export interface BucketProps {
* @default - No log file prefix
*/
readonly serverAccessLogsPrefix?: string;

/**
* The inventory configuration of the bucket.
*
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html
*
* @default - No inventory configuration
*/
readonly inventories?: Inventory[];
}

/**
Expand Down Expand Up @@ -1055,6 +1188,7 @@ export class Bucket extends BucketBase {
private readonly notifications: BucketNotifications;
private readonly metrics: BucketMetrics[] = [];
private readonly cors: CorsRule[] = [];
private readonly inventories: Inventory[] = [];

constructor(scope: Construct, id: string, props: BucketProps = {}) {
super(scope, id, {
Expand All @@ -1079,6 +1213,7 @@ export class Bucket extends BucketBase {
corsConfiguration: Lazy.anyValue({ produce: () => this.parseCorsConfiguration() }),
accessControl: Lazy.stringValue({ produce: () => this.accessControl }),
loggingConfiguration: this.parseServerAccessLogs(props),
inventoryConfigurations: Lazy.anyValue({ produce: () => this.parseInventoryConfiguration() }),
});

resource.applyRemovalPolicy(props.removalPolicy);
Expand Down Expand Up @@ -1107,6 +1242,10 @@ export class Bucket extends BucketBase {
props.serverAccessLogsBucket.allowLogDelivery();
}

for (const inventory of props.inventories ?? []) {
this.addInventory(inventory);
}

// Add all bucket metric configurations rules
(props.metrics || []).forEach(this.addMetric.bind(this));
// Add all cors configuration rules
Expand Down Expand Up @@ -1204,6 +1343,15 @@ export class Bucket extends BucketBase {
return this.addEventNotification(EventType.OBJECT_REMOVED, dest, ...filters);
}

/**
* Add an inventory configuration.
*
* @param inventory configuration to add
*/
public addInventory(inventory: Inventory): void {
this.inventories.push(inventory);
}

private validateBucketName(physicalName: string): void {
const bucketName = physicalName;
if (!bucketName || Token.isUnresolved(bucketName)) {
Expand Down Expand Up @@ -1460,6 +1608,50 @@ export class Bucket extends BucketBase {

this.accessControl = BucketAccessControl.LOG_DELIVERY_WRITE;
}

private parseInventoryConfiguration(): CfnBucket.InventoryConfigurationProperty[] | undefined {
if (!this.inventories || this.inventories.length === 0) {
return undefined;
}

return this.inventories.map((inventory, index) => {
const format = inventory.format ?? InventoryFormat.CSV;
const frequency = inventory.frequency ?? InventoryFrequency.WEEKLY;
const id = inventory.inventoryId ?? `${this.node.id}Inventory${index}`;

if (inventory.destination.bucket instanceof Bucket) {
inventory.destination.bucket.addToResourcePolicy(new iam.PolicyStatement({
effect: iam.Effect.ALLOW,
actions: ['s3:PutObject'],
resources: [
inventory.destination.bucket.bucketArn,
inventory.destination.bucket.arnForObjects(`${inventory.destination.prefix ?? ''}*`),
],
principals: [new iam.ServicePrincipal('s3.amazonaws.com')],
conditions: {
ArnLike: {
'aws:SourceArn': this.bucketArn,
},
},
}));
}

return {
id,
destination: {
bucketArn: inventory.destination.bucket.bucketArn,
bucketAccountId: inventory.destination.bucketOwner,
prefix: inventory.destination.prefix,
format,
},
enabled: inventory.enabled ?? true,
includedObjectVersions: inventory.includeObjectVersions ?? InventoryObjectVersion.ALL,
scheduleFrequency: frequency,
optionalFields: inventory.optionalFields,
prefix: inventory.objectsPrefix,
};
});
}
}

/**
Expand Down
Loading

0 comments on commit b0f359e

Please sign in to comment.