[Storage] Async iterator for listing blobs and containers - storage-blob

sdk/storage/storage-blob/samples/typescript/iterators.ts

@@ -0,0 +1,89 @@
+/* 
+ Setup: Enter your storage account name and shared key in main()
+*/
+
+import {
+  ContainerClient,
+  BlobServiceClient,
+  StorageClient,
+  SharedKeyCredential,
+  BlobClient,
+  BlockBlobClient
+} from "../../src"; // Change to "@azure/storage-blob" in your package
+
+async function main() {
+  // Enter your storage account name and shared key
+  const account = "";
+  const accountKey = "";
+
+  // Use SharedKeyCredential with storage account and account key
+  const sharedKeyCredential = new SharedKeyCredential(account, accountKey);
+
+  // Use sharedKeyCredential, tokenCredential or anonymousCredential to create a pipeline
+  const pipeline = StorageClient.newPipeline(sharedKeyCredential);
+
+  // List containers
+  const blobServiceClient = new BlobServiceClient(
+    // When using AnonymousCredential, following url should include a valid SAS or support public access
+    `https://${account}.blob.core.windows.net`,
+    pipeline
+  );
+
+  // List Containers
+  let iter1 = await blobServiceClient.listContainers();
+  let i = 1;
+  for await (const container of iter1) {
+    console.log(`Container ${i++}: ${container.name}`);
+  }
+
+  // List containers - generator syntax
+  let iter2 = await blobServiceClient.listContainers();
+  i = 1;
+  let containerItem = await iter2.next();
+  do {
+    console.log(`Container ${i++}: ${containerItem.value.name}`);
+    containerItem = await iter2.next();
+  } while (containerItem.value);
+
+  // Create a container
+  const containerName = `newcontainer${new Date().getTime()}`;
+  const containerClient = ContainerClient.fromBlobServiceClient(blobServiceClient, containerName);
+
+  const createContainerResponse = await containerClient.create();
+  console.log(`Created container ${containerName} successfully`, createContainerResponse.requestId);
+
+  for (let index = 0; index < 4; index++) {
+    // Create a blob
+    let content = "hello";
+    let blobName = "newblob" + new Date().getTime();
+    let blobClient = BlobClient.fromContainerClient(containerClient, blobName);
+    let blockBlobClient = BlockBlobClient.fromBlobClient(blobClient);
+    let uploadBlobResponse = await blockBlobClient.upload(content, content.length);
+    console.log(`Uploaded block blob ${blobName} successfully`, uploadBlobResponse.requestId);
+  }
+
+  // List blobs
+  iter1 = await containerClient.listBlobsFlat();
+  i = 1;
+  for await (const blob of iter1) {
+    console.log(`Blob ${i++}: ${blob.name}`);
+  }
+
+  // List blobs - generator syntax
+  iter2 = await containerClient.listBlobsFlat();
+  i = 1;
+  let blobItem = await iter2.next();
+  do {
+    console.log(`Blob ${i++}: ${blobItem.value.name}`);
+    blobItem = await iter2.next();
+  } while (blobItem.value);
+}
+
+// An async method returns a Promise object, which is compatible with then().catch() coding style.
+main()
+  .then(() => {
+    console.log("Successfully executed the sample.");
+  })
+  .catch((err) => {
+    console.log(err.message);
+  });

sdk/storage/storage-blob/src/BlobServiceClient.ts

@@ -191,6 +191,53 @@ export class BlobServiceClient extends StorageClient {
     });
   }
 
+  /**
+   * Iterates over containers under the specified account.
+   *
+   * @param {ServiceListContainersSegmentOptions} [options={}] Options to list containers(optional)
+   * @returns {AsyncIterableIterator<Models.ContainerItem>}
+   * @memberof BlobServiceClient
+   *
+   * @example
+   * for await (const container of blobServiceClient.listContainers()) {
+   *   console.log(`Container: ${container.name}`);
+   * }
+   *
+   * @example
+   * let iter1 = blobServiceClient.listContainers();
+   * let i = 1;
+   * for await (const container of iter1) {
+   *   console.log(`${i}: ${container.name}`);
+   *   i++;
+   * }
+   *
+   * @example
+   * let iter2 = await blobServiceClient.listContainers();
+   * i = 1;
+   * let containerItem = await iter2.next();
+   * do {
+   *   console.log(`Container ${i++}: ${containerItem.value.name}`);
+   *   containerItem = await iter2.next();
+   * } while (containerItem.value);
+   *
+   */
+  public async *listContainers(
+    options: ServiceListContainersSegmentOptions = {}
+  ): AsyncIterableIterator<Models.ContainerItem> {
+    let marker = undefined;
+    const blobServiceClient = this;
+    const aborter = !options.abortSignal ? Aborter.none : options.abortSignal;
+    let listContainersResponse;
+    do {
+      listContainersResponse = await blobServiceClient.listContainersSegment(marker, {
+        ...options,
+        abortSignal: aborter
+      });
+      marker = listContainersResponse.nextMarker;
+      yield* listContainersResponse.containerItems;
+    } while (marker);
+  }
+
   /**
    * Returns a list of the containers under the specified account.
    * @see https://docs.microsoft.com/en-us/rest/api/storageservices/list-containers2

sdk/storage/storage-blob/src/ContainerClient.ts

@@ -597,6 +597,53 @@ export class ContainerClient extends StorageClient {
     });
   }
 
+  /**
+   * Iterates over blobs under the specified container.
+   *
+   * @param {ContainerListBlobsSegmentOptions} [options={}] Options to list blobs(optional)
+   * @returns {AsyncIterableIterator<Models.BlobItem>}
+   * @memberof ContainerClient
+   *
+   * @example
+   * for await (const blob of containerClient.listBlobs()) {
+   *   console.log(`Container: ${blob.name}`);
+   * }
+   *
+   * @example
+   * let iter1 = containerClient.listBlobs();
+   * let i = 1;
+   * for await (const blob of iter1) {
+   *   console.log(`${i}: ${blob.name}`);
+   *   i++;
+   * }
+   *
+   * @example
+   * let iter2 = await containerClient.listBlobs();
+   * i = 1;
+   * let blobItem = await iter2.next();
+   * do {
+   *  console.log(`Blob ${i++}: ${blobItem.value.name}`);
+   *  blobItem = await iter2.next();
+   * } while (blobItem.value);
+   *
+   */
+  public async *listBlobsFlat(
+    options: ContainerListBlobsSegmentOptions = {}
+  ): AsyncIterableIterator<Models.BlobItem> {
+    let marker = undefined;
+    const containerClient = this;
+    const aborter = !options.abortSignal ? Aborter.none : options.abortSignal;
+    let listBlobsResponse;
+    do {
+      listBlobsResponse = await containerClient.listBlobFlatSegment(marker, {
+        ...options,
+        abortSignal: aborter
+      });
+      marker = listBlobsResponse.nextMarker;
+      yield* listBlobsResponse.segment.blobItems;
+    } while (marker);
+  }
+
   /**
    * listBlobFlatSegment returns a single segment of blobs starting from the
    * specified Marker. Use an empty Marker to start enumeration from the beginning.

sdk/storage/storage-blob/test/blobserviceclient.spec.ts

@@ -74,6 +74,46 @@ describe("BlobServiceClient", () => {
     await containerClient2.delete();
   });
 
+  it("Verify AsyncIterator for ListContainers", async () => {
+    const blobServiceClient = getBSU();
+
+    const containerNamePrefix = getUniqueName("container");
+    const containerName1 = `${containerNamePrefix}x1`;
+    const containerName2 = `${containerNamePrefix}x2`;
+    const containerClient1 = blobServiceClient.createContainerClient(containerName1);
+    const containerClient2 = blobServiceClient.createContainerClient(containerName2);
+    await containerClient1.create({ metadata: { key: "val" } });
+    await containerClient2.create({ metadata: { key: "val" } });
+
+    const iterator = await blobServiceClient.listContainers({
+      include: "metadata",
+      prefix: containerNamePrefix
+    });
+
+    let containerItem = await iterator.next();
+    assert.ok(containerItem.value.name.startsWith(containerNamePrefix));
+    assert.ok(containerItem.value.properties.etag.length > 0);
+    assert.ok(containerItem.value.properties.lastModified);
+    assert.ok(!containerItem.value.properties.leaseDuration);
+    assert.ok(!containerItem.value.properties.publicAccess);
+    assert.deepEqual(containerItem.value.properties.leaseState, "available");
+    assert.deepEqual(containerItem.value.properties.leaseStatus, "unlocked");
+    assert.deepEqual(containerItem.value.metadata!.key, "val");
+
+    containerItem = await iterator.next();
+    assert.ok(containerItem.value.name.startsWith(containerNamePrefix));
+    assert.ok(containerItem.value.properties.etag.length > 0);
+    assert.ok(containerItem.value.properties.lastModified);
+    assert.ok(!containerItem.value.properties.leaseDuration);
+    assert.ok(!containerItem.value.properties.publicAccess);
+    assert.deepEqual(containerItem.value.properties.leaseState, "available");
+    assert.deepEqual(containerItem.value.properties.leaseStatus, "unlocked");
+    assert.deepEqual(containerItem.value.metadata!.key, "val");
+
+    await containerClient1.delete();
+    await containerClient2.delete();
+  });
+
   it("GetProperties", async () => {
     const blobServiceClient = getBSU();
     const result = await blobServiceClient.getProperties();

sdk/storage/storage-blob/test/containerclient.spec.ts

@@ -160,9 +160,7 @@ describe("ContainerClient", () => {
   it("listBlobFlatSegment with default parameters", async () => {
     const blobClients = [];
     for (let i = 0; i < 3; i++) {
-      const blobClient = containerClient.createBlobClient(
-        getUniqueName(`blockblob/${i}`)
-      );
+      const blobClient = containerClient.createBlobClient(getUniqueName(`blockblob/${i}`));
       const blockBlobClient = blobClient.createBlockBlobClient();
       await blockBlobClient.upload("", 0);
       blobClients.push(blobClient);
@@ -188,9 +186,7 @@ describe("ContainerClient", () => {
       keyb: "c"
     };
     for (let i = 0; i < 2; i++) {
-      const blobClient = containerClient.createBlobClient(
-        getUniqueName(`${prefix}/${i}`)
-      );
+      const blobClient = containerClient.createBlobClient(getUniqueName(`${prefix}/${i}`));
       const blockBlobClient = blobClient.createBlockBlobClient();
       await blockBlobClient.upload("", 0, {
         metadata
@@ -226,12 +222,44 @@ describe("ContainerClient", () => {
     }
   });
 
+  it("Verify AsyncIterator for listBlobsFlat", async () => {
+    const blobClients = [];
+    const prefix = "blockblob";
+    const metadata = {
+      keya: "a",
+      keyb: "c"
+    };
+    for (let i = 0; i < 2; i++) {
+      const blobClient = containerClient.createBlobClient(getUniqueName(`${prefix}/${i}`));
+      const blockBlobClient = blobClient.createBlockBlobClient();
+      await blockBlobClient.upload("", 0, {
+        metadata
+      });
+      blobClients.push(blobClient);
+    }
+
+    const iterator = await containerClient.listBlobsFlat({
+      include: ["snapshots", "metadata", "uncommittedblobs", "copy", "deleted"],
+      prefix
+    });
+
+    let blobItem = await iterator.next();
+    assert.ok(blobClients[0].url.indexOf(blobItem.value.name));
+    assert.deepStrictEqual(blobItem.value.metadata, metadata);
+
+    blobItem = await iterator.next();
+    assert.ok(blobClients[1].url.indexOf(blobItem.value.name));
+    assert.deepStrictEqual(blobItem.value.metadata, metadata);
+
+    for (const blob of blobClients) {
+      await blob.delete();
+    }
+  });
+
   it("listBlobHierarchySegment with default parameters", async () => {
     const blobClients = [];
     for (let i = 0; i < 3; i++) {
-      const blobClient = containerClient.createBlobClient(
-        getUniqueName(`blockblob${i}/${i}`)
-      );
+      const blobClient = containerClient.createBlobClient(getUniqueName(`blockblob${i}/${i}`));
       const blockBlobClient = blobClient.createBlockBlobClient();
       await blockBlobClient.upload("", 0);
       blobClients.push(blobClient);
@@ -274,45 +302,33 @@ describe("ContainerClient", () => {
       blobClients.push(blobClient);
     }
 
-    const result = await containerClient.listBlobHierarchySegment(
-      delimiter,
-      undefined,
-      {
-        include: ["metadata", "uncommittedblobs", "copy", "deleted"],
-        maxresults: 1,
-        prefix
-      }
-    );
+    const result = await containerClient.listBlobHierarchySegment(delimiter, undefined, {
+      include: ["metadata", "uncommittedblobs", "copy", "deleted"],
+      maxresults: 1,
+      prefix
+    });
     assert.ok(result.serviceEndpoint.length > 0);
     assert.ok(containerClient.url.indexOf(result.containerName));
     assert.deepStrictEqual(result.segment.blobPrefixes!.length, 1);
     assert.deepStrictEqual(result.segment.blobItems!.length, 0);
     assert.ok(blobClients[0].url.indexOf(result.segment.blobPrefixes![0].name));
 
-    const result2 = await containerClient.listBlobHierarchySegment(
-      delimiter,
-      result.nextMarker,
-      {
-        include: ["metadata", "uncommittedblobs", "copy", "deleted"],
-        maxresults: 2,
-        prefix
-      }
-    );
+    const result2 = await containerClient.listBlobHierarchySegment(delimiter, result.nextMarker, {
+      include: ["metadata", "uncommittedblobs", "copy", "deleted"],
+      maxresults: 2,
+      prefix
+    });
     assert.ok(result2.serviceEndpoint.length > 0);
     assert.ok(containerClient.url.indexOf(result2.containerName));
     assert.deepStrictEqual(result2.segment.blobPrefixes!.length, 1);
     assert.deepStrictEqual(result2.segment.blobItems!.length, 0);
     assert.ok(blobClients[0].url.indexOf(result2.segment.blobPrefixes![0].name));
 
-    const result3 = await containerClient.listBlobHierarchySegment(
-      delimiter,
-      undefined,
-      {
-        include: ["metadata", "uncommittedblobs", "copy", "deleted"],
-        maxresults: 2,
-        prefix: `${prefix}0${delimiter}`
-      }
-    );
+    const result3 = await containerClient.listBlobHierarchySegment(delimiter, undefined, {
+      include: ["metadata", "uncommittedblobs", "copy", "deleted"],
+      maxresults: 2,
+      prefix: `${prefix}0${delimiter}`
+    });
     assert.ok(result3.serviceEndpoint.length > 0);
     assert.ok(containerClient.url.indexOf(result3.containerName));
     assert.deepStrictEqual(result3.nextMarker, "");