feat: add zip tree container and stream() method to tree container interface (#154)

Author: Bryan Powell

scripts/repl.js

@@ -1,13 +1,15 @@
 #!/usr/bin/env node
 
 const repl = require('repl');
-const { RegistryAccess, MetadataConverter } = require('../lib/src');
+const { Connection, AuthInfo } = require('@salesforce/core');
+const { RegistryAccess, MetadataConverter, SourceClient } = require('../lib/src');
 
 const startMessage = `
 Usage:
   registryAccess: RegistryAccess instance
   converter: MetadataConverter instance
   resolve(path): resolve components from a path
+  async client(username): create a SourceClient
 `
 console.log(startMessage);
 
@@ -16,3 +18,8 @@ replServer.setupHistory('.repl_history', (err, repl) => {});
 replServer.context.registryAccess = new RegistryAccess();
 replServer.context.converter = new MetadataConverter();
 replServer.context.resolve = (path) => replServer.context.registryAccess.getComponentsFromPath(path)
+replServer.context.client = async (username) => {
+  return new SourceClient(await Connection.create({
+    authInfo: await AuthInfo.create({ username })
+  }));
+}

src/i18n/i18n.ts

@@ -49,6 +49,9 @@ export const messages = {
   error_updating_metadata_type: "Unexpected error updating '%s'",
   error_parsing_metadata_file: 'Error parsing metadata file',
   error_parsing_xml: 'SourceComponent %s does not have an associated metadata xml to parse',
+  error_expected_file_path: '%s: path is to a directory, expected a file',
+  error_expected_directory_path: '%s: path is to a file, expected a directory',
+  error_no_directory_stream: '%s does not support readable streams on directories',
   error_static_resource_expected_archive_type:
     'A StaticResource directory must have a content type of application/zip or application/jar - found %s for %s',
   tapi_deploy_component_limit_error:

src/metadata-registry/treeContainers.ts

@@ -5,12 +5,14 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import { VirtualDirectory, TreeContainer } from '../metadata-registry';
-import { join, dirname, basename } from 'path';
+import { join, dirname, basename, normalize } from 'path';
 import { baseName } from '../utils';
 import { parseMetadataXml } from '../utils/registry';
-import { lstatSync, existsSync, readdirSync, promises as fsPromises } from 'fs';
+import { lstatSync, existsSync, readdirSync, promises as fsPromises, createReadStream } from 'fs';
 import { LibraryError } from '../errors';
 import { SourcePath } from '../common';
+import * as unzipper from 'unzipper';
+import { Readable } from 'stream';
 
 /**
  * An extendable base class for implementing the `TreeContainer` interface
@@ -35,6 +37,7 @@ export abstract class BaseTreeContainer implements TreeContainer {
   public abstract isDirectory(fsPath: SourcePath): boolean;
   public abstract readDirectory(fsPath: SourcePath): string[];
   public abstract readFile(fsPath: SourcePath): Promise<Buffer>;
+  public abstract stream(fsPath: SourcePath): Readable;
 }
 
 export class NodeFSTreeContainer extends BaseTreeContainer {
@@ -53,6 +56,83 @@ export class NodeFSTreeContainer extends BaseTreeContainer {
   public readFile(fsPath: SourcePath): Promise<Buffer> {
     return fsPromises.readFile(fsPath);
   }
+
+  public stream(fsPath: SourcePath): Readable {
+    return createReadStream(fsPath);
+  }
+}
+
+interface ZipEntry {
+  path: string;
+  stream?: () => unzipper.Entry;
+  buffer?: () => Promise<Buffer>;
+}
+
+export class ZipTreeContainer extends BaseTreeContainer {
+  private tree = new Map<SourcePath, ZipEntry[] | ZipEntry>();
+
+  private constructor(directory: unzipper.CentralDirectory) {
+    super();
+    this.populate(directory);
+  }
+
+  public static async create(buffer: Buffer): Promise<ZipTreeContainer> {
+    const directory = await unzipper.Open.buffer(buffer);
+    return new ZipTreeContainer(directory);
+  }
+
+  public exists(fsPath: string): boolean {
+    return this.tree.has(fsPath);
+  }
+
+  public isDirectory(fsPath: string): boolean {
+    if (this.exists(fsPath)) {
+      return Array.isArray(this.tree.get(fsPath));
+    }
+    throw new LibraryError('error_path_not_found', fsPath);
+  }
+
+  public readDirectory(fsPath: string): string[] {
+    if (this.isDirectory(fsPath)) {
+      return (this.tree.get(fsPath) as ZipEntry[]).map((entry) => basename(entry.path));
+    }
+    throw new LibraryError('error_expected_directory_path', fsPath);
+  }
+
+  public readFile(fsPath: string): Promise<Buffer> {
+    if (!this.isDirectory(fsPath)) {
+      return (this.tree.get(fsPath) as ZipEntry).buffer();
+    }
+    throw new LibraryError('error_expected_file_path', fsPath);
+  }
+
+  public stream(fsPath: string): Readable {
+    if (!this.isDirectory(fsPath)) {
+      return (this.tree.get(fsPath) as ZipEntry).stream();
+    }
+    throw new LibraryError('error_no_directory_stream', this.constructor.name);
+  }
+
+  private populate(directory: unzipper.CentralDirectory): void {
+    for (const { path, stream, buffer } of directory.files) {
+      // normalize path to use OS separator since zip entries always use forward slash
+      const entry = { path: normalize(path), stream, buffer };
+      this.tree.set(entry.path, entry);
+      this.ensureDirPathExists(entry);
+    }
+  }
+
+  private ensureDirPathExists(entry: ZipEntry): void {
+    const dirPath = dirname(entry.path);
+    if (dirPath === entry.path) {
+      return;
+    } else if (!this.exists(dirPath)) {
+      this.tree.set(dirPath, [entry]);
+      this.ensureDirPathExists({ path: dirPath });
+    } else {
+      (this.tree.get(dirPath) as ZipEntry[]).push(entry);
+    }
+  }
 }
 
 export class VirtualTreeContainer extends BaseTreeContainer {
@@ -93,6 +173,10 @@ export class VirtualTreeContainer extends BaseTreeContainer {
     throw new LibraryError('error_path_not_found', fsPath);
   }
 
+  public stream(fsPath: string): Readable {
+    throw new Error('Method not implemented.');
+  }
+
   private populate(virtualFs: VirtualDirectory[]): void {
     for (const dir of virtualFs) {
       const { dirPath, children } = dir;

src/metadata-registry/types.ts

@@ -7,6 +7,7 @@
 
 import { MetadataType, SourcePath } from '../common/types';
 import { SourceComponent } from '.';
+import { Readable } from 'stream';
 
 /**
  * Metadata type definitions
@@ -105,4 +106,5 @@ export interface TreeContainer {
   readDirectory(path: SourcePath): string[];
   find(fileType: 'content' | 'metadata', fullName: string, dir: SourcePath): SourcePath | undefined;
   readFile(fsPath: SourcePath): Promise<Buffer>;
+  stream(fsPath: SourcePath): Readable;
 }