[Storage] Async iterator for listing blobs and containers - storage-b…

…lob (#3136)

* Async iterator for container - listBlobs

* remove iter in the generator

* Add marker in the generator

* yield -> yield *

* Add sample using `.next().value`

* containerClient - *listContainers

* udpdate - remove resumable from comment

* Addressed comments

* Add description for options attribute

* remove process.env

* un-specify the type Models.xxx

* remove !options

* rectify indentation

* listBlobs -> listBlobsFlat

* Add test for ListContainers

* remove .only

* Add test for listBlobsFlat

* Add test case for listBlobsFlat - for loop syntax

* Add test case for listContainers - for loop syntax

sdk/storage/storage-blob/samples/typescript/iterators.ts

@@ -0,0 +1,89 @@
+/* 
+ Setup: Enter your storage account name and shared key in main()
+*/
+
+import {
+  ContainerClient,
+  BlobServiceClient,
+  StorageClient,
+  SharedKeyCredential,
+  BlobClient,
+  BlockBlobClient
+} from "../../src"; // Change to "@azure/storage-blob" in your package
+
+async function main() {
+  // Enter your storage account name and shared key
+  const account = "";
+  const accountKey = "";
+
+  // Use SharedKeyCredential with storage account and account key
+  const sharedKeyCredential = new SharedKeyCredential(account, accountKey);
+
+  // Use sharedKeyCredential, tokenCredential or anonymousCredential to create a pipeline
+  const pipeline = StorageClient.newPipeline(sharedKeyCredential);
+
+  // List containers
+  const blobServiceClient = new BlobServiceClient(
+    // When using AnonymousCredential, following url should include a valid SAS or support public access
+    `https://${account}.blob.core.windows.net`,
+    pipeline
+  );
+
+  // List Containers
+  let iter1 = await blobServiceClient.listContainers();
+  let i = 1;
+  for await (const container of iter1) {
+    console.log(`Container ${i++}: ${container.name}`);
+  }
+
+  // List containers - generator syntax
+  let iter2 = await blobServiceClient.listContainers();
+  i = 1;
+  let containerItem = await iter2.next();
+  do {
+    console.log(`Container ${i++}: ${containerItem.value.name}`);
+    containerItem = await iter2.next();
+  } while (containerItem.value);
+
+  // Create a container
+  const containerName = `newcontainer${new Date().getTime()}`;
+  const containerClient = ContainerClient.fromBlobServiceClient(blobServiceClient, containerName);
+
+  const createContainerResponse = await containerClient.create();
+  console.log(`Created container ${containerName} successfully`, createContainerResponse.requestId);
+
+  for (let index = 0; index < 4; index++) {
+    // Create a blob
+    let content = "hello";
+    let blobName = "newblob" + new Date().getTime();
+    let blobClient = BlobClient.fromContainerClient(containerClient, blobName);
+    let blockBlobClient = BlockBlobClient.fromBlobClient(blobClient);
+    let uploadBlobResponse = await blockBlobClient.upload(content, content.length);
+    console.log(`Uploaded block blob ${blobName} successfully`, uploadBlobResponse.requestId);
+  }
+
+  // List blobs
+  iter1 = await containerClient.listBlobsFlat();
+  i = 1;
+  for await (const blob of iter1) {
+    console.log(`Blob ${i++}: ${blob.name}`);
+  }
+
+  // List blobs - generator syntax
+  iter2 = await containerClient.listBlobsFlat();
+  i = 1;
+  let blobItem = await iter2.next();
+  do {
+    console.log(`Blob ${i++}: ${blobItem.value.name}`);
+    blobItem = await iter2.next();
+  } while (blobItem.value);
+}
+
+// An async method returns a Promise object, which is compatible with then().catch() coding style.
+main()
+  .then(() => {
+    console.log("Successfully executed the sample.");
+  })
+  .catch((err) => {
+    console.log(err.message);
+  });

sdk/storage/storage-blob/src/BlobServiceClient.ts

@@ -247,6 +247,53 @@ export class BlobServiceClient extends StorageClient {
     });
   }
 
+  /**
+   * Iterates over containers under the specified account.
+   *
+   * @param {ServiceListContainersSegmentOptions} [options={}] Options to list containers(optional)
+   * @returns {AsyncIterableIterator<Models.ContainerItem>}
+   * @memberof BlobServiceClient
+   *
+   * @example
+   * for await (const container of blobServiceClient.listContainers()) {
+   *   console.log(`Container: ${container.name}`);
+   * }
+   *
+   * @example
+   * let iter1 = blobServiceClient.listContainers();
+   * let i = 1;
+   * for await (const container of iter1) {
+   *   console.log(`${i}: ${container.name}`);
+   *   i++;
+   * }
+   *
+   * @example
+   * let iter2 = await blobServiceClient.listContainers();
+   * i = 1;
+   * let containerItem = await iter2.next();
+   * do {
+   *   console.log(`Container ${i++}: ${containerItem.value.name}`);
+   *   containerItem = await iter2.next();
+   * } while (containerItem.value);
+   *
+   */
+  public async *listContainers(
+    options: ServiceListContainersSegmentOptions = {}
+  ): AsyncIterableIterator<Models.ContainerItem> {
+    let marker = undefined;
+    const blobServiceClient = this;
+    const aborter = !options.abortSignal ? Aborter.none : options.abortSignal;
+    let listContainersResponse;
+    do {
+      listContainersResponse = await blobServiceClient.listContainersSegment(marker, {
+        ...options,
+        abortSignal: aborter
+      });
+      marker = listContainersResponse.nextMarker;
+      yield* listContainersResponse.containerItems;
+    } while (marker);
+  }
+
   /**
    * Returns a list of the containers under the specified account.
    * @see https://docs.microsoft.com/en-us/rest/api/storageservices/list-containers2

sdk/storage/storage-blob/src/ContainerClient.ts

@@ -836,6 +836,53 @@ export class ContainerClient extends StorageClient {
     });
   }
 
+  /**
+   * Iterates over blobs under the specified container.
+   *
+   * @param {ContainerListBlobsSegmentOptions} [options={}] Options to list blobs(optional)
+   * @returns {AsyncIterableIterator<Models.BlobItem>}
+   * @memberof ContainerClient
+   *
+   * @example
+   * for await (const blob of containerClient.listBlobs()) {
+   *   console.log(`Container: ${blob.name}`);
+   * }
+   *
+   * @example
+   * let iter1 = containerClient.listBlobs();
+   * let i = 1;
+   * for await (const blob of iter1) {
+   *   console.log(`${i}: ${blob.name}`);
+   *   i++;
+   * }
+   *
+   * @example
+   * let iter2 = await containerClient.listBlobs();
+   * i = 1;
+   * let blobItem = await iter2.next();
+   * do {
+   *  console.log(`Blob ${i++}: ${blobItem.value.name}`);
+   *  blobItem = await iter2.next();
+   * } while (blobItem.value);
+   *
+   */
+  public async *listBlobsFlat(
+    options: ContainerListBlobsSegmentOptions = {}
+  ): AsyncIterableIterator<Models.BlobItem> {
+    let marker = undefined;
+    const containerClient = this;
+    const aborter = !options.abortSignal ? Aborter.none : options.abortSignal;
+    let listBlobsResponse;
+    do {
+      listBlobsResponse = await containerClient.listBlobFlatSegment(marker, {
+        ...options,
+        abortSignal: aborter
+      });
+      marker = listBlobsResponse.nextMarker;
+      yield* listBlobsResponse.segment.blobItems;
+    } while (marker);
+  }
+
   /**
    * listBlobFlatSegment returns a single segment of blobs starting from the
    * specified Marker. Use an empty Marker to start enumeration from the beginning.

sdk/storage/storage-blob/test/blobserviceclient.spec.ts

@@ -74,6 +74,79 @@ describe("BlobServiceClient", () => {
     await containerClient2.delete();
   });
 
+  it("Verify AsyncIterator(generator .next() syntax) for ListContainers", async () => {
+    const blobServiceClient = getBSU();
+
+    const containerNamePrefix = getUniqueName("container");
+    const containerName1 = `${containerNamePrefix}x1`;
+    const containerName2 = `${containerNamePrefix}x2`;
+    const containerClient1 = blobServiceClient.createContainerClient(containerName1);
+    const containerClient2 = blobServiceClient.createContainerClient(containerName2);
+    await containerClient1.create({ metadata: { key: "val" } });
+    await containerClient2.create({ metadata: { key: "val" } });
+
+    const iterator = await blobServiceClient.listContainers({
+      include: "metadata",
+      prefix: containerNamePrefix
+    });
+
+    let containerItem = await iterator.next();
+    assert.ok(containerItem.value.name.startsWith(containerNamePrefix));
+    assert.ok(containerItem.value.properties.etag.length > 0);
+    assert.ok(containerItem.value.properties.lastModified);
+    assert.ok(!containerItem.value.properties.leaseDuration);
+    assert.ok(!containerItem.value.properties.publicAccess);
+    assert.deepEqual(containerItem.value.properties.leaseState, "available");
+    assert.deepEqual(containerItem.value.properties.leaseStatus, "unlocked");
+    assert.deepEqual(containerItem.value.metadata!.key, "val");
+
+    containerItem = await iterator.next();
+    assert.ok(containerItem.value.name.startsWith(containerNamePrefix));
+    assert.ok(containerItem.value.properties.etag.length > 0);
+    assert.ok(containerItem.value.properties.lastModified);
+    assert.ok(!containerItem.value.properties.leaseDuration);
+    assert.ok(!containerItem.value.properties.publicAccess);
+    assert.deepEqual(containerItem.value.properties.leaseState, "available");
+    assert.deepEqual(containerItem.value.properties.leaseStatus, "unlocked");
+    assert.deepEqual(containerItem.value.metadata!.key, "val");
+
+    await containerClient1.delete();
+    await containerClient2.delete();
+  });
+
+  it("Verify AsyncIterator(for-loop syntax) for ListContainers", async () => {
+    const containerClients = [];
+    const blobServiceClient = getBSU();
+
+    const containerNamePrefix = getUniqueName("container");
+
+    for (let i = 0; i < 4; i++) {
+      const containerName = `${containerNamePrefix}x${i}`;
+      const containerClient = blobServiceClient.createContainerClient(containerName);
+      await containerClient.create({ metadata: { key: "val" } });
+      containerClients.push(containerClient);
+    }
+
+    for await (const container of blobServiceClient.listContainers({
+      include: "metadata",
+      prefix: containerNamePrefix,
+      maxresults: 2
+    })) {
+      assert.ok(container.name.startsWith(containerNamePrefix));
+      assert.ok(container.properties.etag.length > 0);
+      assert.ok(container.properties.lastModified);
+      assert.ok(!container.properties.leaseDuration);
+      assert.ok(!container.properties.publicAccess);
+      assert.deepEqual(container.properties.leaseState, "available");
+      assert.deepEqual(container.properties.leaseStatus, "unlocked");
+      assert.deepEqual(container.metadata!.key, "val");
+    }
+
+    for (const client of containerClients) {
+      await client.delete();
+    }
+  });
+
   it("GetProperties", async () => {
     const blobServiceClient = getBSU();
     const result = await blobServiceClient.getProperties();