feat(techdocs-common): add Azure Blob Storage

app-config.yaml

@@ -80,7 +80,7 @@ techdocs:
   generators:
     techdocs: 'docker' # Alternatives - 'local'
   publisher:
-    type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives.
+    type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage'. Read documentation for using alternatives.
 
 sentry:
   organization: my-company

packages/techdocs-common/src/stages/publish/azureStorage.test.ts → packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts

@@ -16,7 +16,7 @@
 import mockFs from 'mock-fs';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '@backstage/backend-common';
-import { AzureStoragePublish } from './azureStorage';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
 import { PublisherBase } from './types';
 import type { Entity } from '@backstage/catalog-model';
 
@@ -54,8 +54,8 @@ beforeEach(async () => {
     techdocs: {
       requestUrl: 'http://localhost:7000',
       publisher: {
-        type: 'azureStorage',
-        azureStorage: {
+        type: 'azureBlobStorage',
+        azureBlobStorage: {
           credentials: {
             account: 'account',
             accountKey: 'accountKey',
@@ -66,10 +66,10 @@ beforeEach(async () => {
     },
   });
 
-  publisher = await AzureStoragePublish.fromConfig(mockConfig, logger);
+  publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
 });
 
-describe('AzureStoragePublish', () => {
+describe('AzureBlobStoragePublish', () => {
   describe('publish', () => {
     it('should publish a directory', async () => {
       const entity = createMockEntity();
@@ -117,7 +117,7 @@ describe('AzureStoragePublish', () => {
         .catch(error =>
           expect(error).toEqual(
             new Error(
-              `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
+              `Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
             ),
           ),
         );

packages/techdocs-common/src/stages/publish/azureStorage.ts → packages/techdocs-common/src/stages/publish/azureBlobStorage.ts

@@ -24,34 +24,45 @@ import { Logger } from 'winston';
 import { Entity, EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
-import { PublisherBase, PublishRequest } from './types';
+import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import limiterFactory from 'p-limit';
+import JSON5 from 'json5';
 
-export class AzureStoragePublish implements PublisherBase {
+// The number of batches that may be ongoing at the same time.
+const BATCH_CONCURRENCY = 3;
+
+export class AzureBlobStoragePublish implements PublisherBase {
   static async fromConfig(
     config: Config,
     logger: Logger,
   ): Promise<PublisherBase> {
-    let account = '';
-    let accountKey = '';
     let containerName = '';
     try {
-      account = config.getString(
-        'techdocs.publisher.azureStorage.credentials.account',
-      );
-      accountKey = config.getString(
-        'techdocs.publisher.azureStorage.credentials.accountKey',
-      );
       containerName = config.getString(
-        'techdocs.publisher.azureStorage.containerName',
+        'techdocs.publisher.azureBlobStorage.containerName',
       );
     } catch (error) {
       throw new Error(
-        "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " +
-          'credentials and containerName are required in techdocs.publisher.azureStorage ' +
-          'required to authenticate with Azure Storage.',
+        "Since techdocs.publisher.type is set to 'awsS3' in your app config, " +
+          'techdocs.publisher.awsS3.bucketName is required.',
       );
     }
 
+    // Credentials is an optional config. If missing, default AWS environment variables
+    // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate
+    // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
+    // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
+    let account = '';
+    let accountKey = '';
+    account =
+      config.getOptionalString(
+        'techdocs.publisher.azureBlobStorage.credentials.account',
+      ) || '';
+    accountKey =
+      config.getOptionalString(
+        'techdocs.publisher.azureBlobStorage.credentials.accountKey',
+      ) || '';
+
     const credential = new StorageSharedKeyCredential(account, accountKey);
     const storageClient = new BlobServiceClient(
       `https://${account}.blob.core.windows.net`,
@@ -63,20 +74,22 @@ export class AzureStoragePublish implements PublisherBase {
       .getProperties()
       .then(() => {
         logger.info(
-          `Successfully connected to the Azure Storage container ${containerName}.`,
+          `Successfully connected to the Azure Blob Storage container ${containerName}.`,
         );
       })
       .catch(reason => {
         logger.error(
-          `Could not retrieve metadata about the Azure Storage container ${containerName}. ` +
+          `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
             'Make sure the Azure project and the container exists and the access key located at the path ' +
-            "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " +
+            "techdocs.publisher.azureBlobStorage.credentials defined in app config has the role 'Storage Object Creator'. " +
             'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
         );
-        throw new Error(`from Azure Storage client library: ${reason.message}`);
+        throw new Error(
+          `from Azure Blob Storage client library: ${reason.message}`,
+        );
       });
 
-    return new AzureStoragePublish(storageClient, containerName, logger);
+    return new AzureBlobStoragePublish(storageClient, containerName, logger);
   }
 
   constructor(
@@ -90,48 +103,57 @@ export class AzureStoragePublish implements PublisherBase {
   }
 
   /**
-   * Upload all the files from the generated `directory` to the Azure Storage container.
+   * Upload all the files from the generated `directory` to the Azure Blob Storage container.
    * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
    */
   async publish({ entity, directory }: PublishRequest): Promise<void> {
     try {
-      // Note: Azure Storage manages creation of parent directories if they do not exist.
+      // Note: Azure Blob Storage manages creation of parent directories if they do not exist.
       // So collecting path of only the files is good enough.
       const allFilesToUpload = await getFileTreeRecursively(directory);
 
       const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
-      allFilesToUpload.forEach(filePath => {
+
+      // Bound the number of concurrent batches. We want a bit of concurrency for
+      // performance reasons, but not so much that we starve the connection pool
+      // or start thrashing.
+      const limiter = limiterFactory(BATCH_CONCURRENCY);
+
+      const promises = allFilesToUpload.map(filePath => {
         // Remove the absolute path prefix of the source directory
         // Path of all files to upload, relative to the root of the source directory
         // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
         const relativeFilePath = filePath.replace(`${directory}/`, '');
         const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
         const destination = path.normalize(
           `${entityRootDir}/${relativeFilePath}`,
-        ); // Azure Storage Container file relative path
+        ); // Azure Blob Storage Container file relative path
+
         // TODO: Upload in chunks of ~10 files instead of all files at once.
-        uploadPromises.push(
-          this.storageClient
-            .getContainerClient(this.containerName)
-            .getBlockBlobClient(destination)
-            .uploadFile(filePath),
-        );
+        return limiter(async () => {
+          await uploadPromises.push(
+            this.storageClient
+              .getContainerClient(this.containerName)
+              .getBlockBlobClient(destination)
+              .uploadFile(filePath),
+          );
+        });
       });
 
-      await Promise.all(uploadPromises).then(() => {
+      await Promise.all(promises).then(() => {
         this.logger.info(
           `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
         );
       });
       return;
     } catch (e) {
-      const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`;
+      const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`;
       this.logger.error(errorMessage);
       throw new Error(errorMessage);
     }
   }
 
-  download(containerName: string, path: string): Promise<string> {
+  private download(containerName: string, path: string): Promise<Buffer> {
     return new Promise((resolve, reject) => {
       const fileStreamChunks: Array<any> = [];
       this.storageClient
@@ -153,19 +175,24 @@ export class AzureStoragePublish implements PublisherBase {
               fileStreamChunks.push(chunk);
             })
             .on('end', () => {
-              resolve(Buffer.concat(fileStreamChunks).toString());
+              resolve(Buffer.concat(fileStreamChunks));
             });
         });
     });
   }
 
-  async fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
+  async fetchTechDocsMetadata(
+    entityName: EntityName,
+  ): Promise<TechDocsMetadata> {
     const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
     try {
-      return this.download(
-        this.containerName,
-        `${entityRootDir}/techdocs_metadata.json`,
-      );
+      return await new Promise<TechDocsMetadata>(resolve => {
+        const download = this.download(
+          this.containerName,
+          `${entityRootDir}/techdocs_metadata.json`,
+        );
+        resolve(JSON5.parse(download.toString()));
+      });
     } catch (e) {
       this.logger.error(e.message);
       throw e;

packages/techdocs-common/src/stages/publish/publish.test.ts

@@ -22,7 +22,7 @@ import { Publisher } from './publish';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
-import { AzureStoragePublish } from './azureStorage';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
 
 const logger = getVoidLogger();
 const discovery: jest.Mocked<PluginEndpointDiscovery> = {
@@ -107,13 +107,13 @@ describe('Publisher', () => {
     expect(publisher).toBeInstanceOf(AwsS3Publish);
   });
 
-  it('should create Azure Storage publisher from config', async () => {
+  it('should create Azure Blob Storage publisher from config', async () => {
     const mockConfig = new ConfigReader({
       techdocs: {
         requestUrl: 'http://localhost:7000',
         publisher: {
-          type: 'azureStorage',
-          azureStorage: {
+          type: 'azureBlobStorage',
+          azureBlobStorage: {
             credentials: {
               account: 'account',
               accountKey: 'accountKey',
@@ -128,6 +128,6 @@ describe('Publisher', () => {
       logger,
       discovery,
     });
-    expect(publisher).toBeInstanceOf(AzureStoragePublish);
+    expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
   });
 });

packages/techdocs-common/src/stages/publish/publish.ts

@@ -21,7 +21,7 @@ import { PublisherType, PublisherBase } from './types';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
-import { AzureStoragePublish } from './azureStorage';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
 
 type factoryOptions = {
   logger: Logger;
@@ -48,9 +48,11 @@ export class Publisher {
       case 'awsS3':
         logger.info('Creating AWS S3 Bucket publisher for TechDocs');
         return AwsS3Publish.fromConfig(config, logger);
-      case 'azureStorage':
-        logger.info('Creating Azure Storage Container publisher for TechDocs');
-        return AzureStoragePublish.fromConfig(config, logger);
+      case 'azureBlobStorage':
+        logger.info(
+          'Creating Azure Blob Storage Container publisher for TechDocs',
+        );
+        return AzureBlobStoragePublish.fromConfig(config, logger);
       case 'local':
         logger.info('Creating Local publisher for TechDocs');
         return new LocalPublish(config, logger, discovery);

packages/techdocs-common/src/stages/publish/types.ts

@@ -19,7 +19,11 @@ import express from 'express';
 /**
  * Key for all the different types of TechDocs publishers that are supported.
  */
-export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage';
+export type PublisherType =
+  | 'local'
+  | 'googleGcs'
+  | 'awsS3'
+  | 'azureBlobStorage';
 
 export type PublishRequest = {
   entity: Entity;

plugins/techdocs-backend/src/service/router.ts

@@ -148,7 +148,7 @@ export async function createRouter({
           }
           break;
         case 'awsS3':
-        case 'azureStorage':
+        case 'azureBlobStorage':
         case 'googleGcs':
           // This block should be valid for all external storage implementations. So no need to duplicate in future,
           // add the publisher type in the list here.

plugins/techdocs/config.d.ts

@@ -117,21 +117,23 @@ export interface Config {
       | {
           /**
            * attr: 'type' - accepts a string value
-           * e.g. type: 'azureStorage'
-           * alternatives: 'azureStorage' etc.
+           * e.g. type: 'azureBlobStorage'
+           * alternatives: 'azureBlobStorage' etc.
            * @see http://backstage.io/docs/features/techdocs/configuration
            */
-          type: 'azureStorage';
+          type: 'azureBlobStorage';
 
           /**
-           * azureStorage required when 'type' is set to azureStorage
+           * azureBlobStorage required when 'type' is set to azureBlobStorage
            */
-          azureStorage?: {
+          azureBlobStorage?: {
             /**
-             * Credentials used to access a storage container
+             * (Optional) Credentials used to access a storage container.
+             * If not set, environment variables will be used to authenticate.
+             * https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
              * @visibility secret
              */
-            credentials: {
+            credentials?: {
               /**
                * Account access name
                * attr: 'account' - accepts a string value
@@ -146,7 +148,7 @@ export interface Config {
               accountKey: string;
             };
             /**
-             * Cloud Storage Container Name
+             * (Required) Cloud Storage Container Name
              * attr: 'containerName' - accepts a string value
              * @visibility backend
              */