[Plugins] Adds support for pagination in listing objects from AWS S3 …

…buckets (ToolJet#3970) * Adds support for pagination in listing objects from s3 buckets * docs * adds Next Continuation Token * docs: Next Continuation Tokenzendesk * docs: Next Continuation Token
iamakhileshmishra · Sep 26, 2022 · 5f8b0f2 · 5f8b0f2
1 parent 0a2ba33
commit 5f8b0f2
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 4 deletions.
diff --git a/docs/docs/data-sources/s3.md b/docs/docs/data-sources/s3.md
@@ -86,6 +86,18 @@ This operation will list all the buckets in your S3. This does not require any p
 This operation will fetch the list of all the files in your bucket. It requires two parameters:
   1. **Bucket**: Bucket name (mandatory)
   2. **Prefix**: To limit the response to keys that begin with the specified prefix (optional)
+  3. **Max keys**: The maximum number of keys returned in the response body (optional). Default value is 1000.
+  4. **Offset**: The key to start with when listing objects in a bucket (optional). 
+  5. **"Next Continuation Token"**: `Next Continuation Token` indicates Amazon S3 that the list is being continued on this bucket with a token. ContinuationToken is obfuscated and is not a real key (optional).
+
+
+:::info
+**Next Continuation Token**
+For listing a bucket for objects that begin with a specific character or a prefix, then use the `Offset` parameter. For example, if you want to list all the objects that begin with `a`, then set the `Offset` parameter to `a`. Similarly, if you want to list all the objects that begin with `ab`, then set the `Offset` parameter to `ab`.
+
+The `Next Continuation Token` is used to list the next set of objects in a bucket. It is returned by the API when the response is truncated. The results will contain `Next Continuation Token` if there are more keys in the bucket that satisfy the list query. To get the next set of objects, set the `Next Continuation Token` parameter and run the query again.
+The results will continue from where the last listing finished.
+:::
 
 
 <img className="screenshot-full" src="/img/datasource-reference/aws-s3/list-objects.png" alt="aws s3 list object" />

diff --git a/plugins/packages/s3/lib/operations.json b/plugins/packages/s3/lib/operations.json
@@ -3,7 +3,9 @@
   "title": "S3 datasource",
   "description": "A schema defining S3 datasource",
   "type": "cloud-storage",
-  "defaults": {},
+  "defaults": {
+    "maxKeys": 1000
+  },
   "properties": {
     "operation": {
       "label": "Operation",
@@ -141,6 +143,36 @@
         "width": "320px",
         "height": "36px",
         "className": "codehinter-plugins"
+      },
+      "maxKeys": {
+        "label": "Max keys",
+        "key": "maxKeys",
+        "type": "codehinter",
+        "lineNumbers": false,
+        "description": "Enter max keys",
+        "width": "320px",
+        "height": "36px",
+        "className": "codehinter-plugins col-6"
+      },
+      "offset": {
+        "label": "Offset",
+        "key": "offset",
+        "type": "codehinter",
+        "lineNumbers": false,
+        "description": "Enter offset",
+        "width": "320px",
+        "height": "36px",
+        "className": "codehinter-plugins col-6"
+      }, 
+      "NextContinuationToken": {
+        "label": "Next Continuation Token",
+        "key": "continuationToken",
+        "type": "codehinter",
+        "lineNumbers": false,
+        "description": "Enter Next Continuation Token",
+        "width": "320px",
+        "height": "36px",
+        "className": "codehinter-plugins"
       }
     },
     "signed_url_for_get": {

diff --git a/plugins/packages/s3/lib/operations.ts b/plugins/packages/s3/lib/operations.ts
@@ -1,10 +1,10 @@
 import {
   GetObjectCommand,
   ListBucketsCommand,
-  ListObjectsCommand,
   PutObjectCommand,
   DeleteObjectCommand,
   S3Client,
+  ListObjectsV2Command,
 } from '@aws-sdk/client-s3';
 // https://aws.amazon.com/blogs/developer/generate-presigned-url-modular-aws-sdk-javascript/
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
@@ -15,8 +15,15 @@ export async function listBuckets(client: S3Client, options: QueryOptions): Prom
   return client.send(command);
 }
 
-export async function listObjects(client: S3Client, options: QueryOptions): Promise<object> {
-  const command = new ListObjectsCommand({ Bucket: options.bucket, Prefix: options.prefix });
+export async function listObjects(client: S3Client, options: object): Promise<object> {
+  const command = new ListObjectsV2Command({
+    Bucket: options['bucket'],
+    Prefix: options['prefix'],
+    MaxKeys: options['maxKeys'],
+    StartAfter: options['offset'],
+    ContinuationToken: options['continuationToken'],
+  });
+
   return client.send(command);
 }