feat: support decomposed components across multiple directories (#224)

src/collections/componentSet.ts

@@ -21,7 +21,7 @@ import {
   FromSourceOptions,
   FromManifestOptions,
   PackageManifestObject,
-  SourceComponentOptions,
+  ResolveOptions,
 } from './types';
 import { ComponentLike } from '../common/types';
 
@@ -46,9 +46,9 @@ export class ComponentSet implements Iterable<MetadataComponent> {
    * @param fsPath Path to resolve components from
    * @param options
    */
-  public static fromSource(fsPath: string, options?: FromSourceOptions): ComponentSet {
-    const ws = new ComponentSet(undefined, options?.registry);
-    ws.resolveSourceComponents(fsPath, { tree: options?.tree });
+  public static fromSource(fsPath: string, options: FromSourceOptions = {}): ComponentSet {
+    const ws = new ComponentSet(undefined, options.registry);
+    ws.resolveSourceComponents(fsPath, options);
     return ws;
   }
 
@@ -67,11 +67,11 @@ export class ComponentSet implements Iterable<MetadataComponent> {
    */
   public static async fromManifestFile(
     fsPath: string,
-    options?: FromManifestOptions
+    options: FromManifestOptions = {}
   ): Promise<ComponentSet> {
-    const registry = options?.registry ?? new RegistryAccess();
-    const tree = options?.tree ?? new NodeFSTreeContainer();
-    const shouldResolve = !!options?.resolve;
+    const registry = options.registry ?? new RegistryAccess();
+    const tree = options.tree ?? new NodeFSTreeContainer();
+    const shouldResolve = !!options.resolve;
 
     const ws = new ComponentSet(undefined, registry);
     const filterSet = new ComponentSet(undefined, registry);
@@ -215,35 +215,47 @@ export class ComponentSet implements Iterable<MetadataComponent> {
    * @param fsPath: File path to resolve
    * @param options
    */
-  public resolveSourceComponents(fsPath: string, options?: SourceComponentOptions): ComponentSet {
+  public resolveSourceComponents(fsPath: string, options: ResolveOptions = {}): ComponentSet {
     let filterSet: ComponentSet;
 
     if (options?.filter) {
       const { filter } = options;
       filterSet = filter instanceof ComponentSet ? filter : new ComponentSet(filter);
     }
 
-    // TODO: move filter logic to resolver W-8023153
+    // TODO: move most of this logic to resolver W-8023153
     const resolver = new MetadataResolver(this.registry, options?.tree);
     const resolved = resolver.getComponentsFromPath(fsPath);
     const sourceComponents = new ComponentSet();
 
     for (const component of resolved) {
-      const shouldResolve = !filterSet || filterSet.has(component);
-      const includedInWildcard = filterSet?.has({
-        fullName: ComponentSet.WILDCARD,
-        type: component.type,
-      });
-      if (shouldResolve || includedInWildcard) {
-        this.add(component);
-        sourceComponents.add(component);
-      } else if (filterSet) {
-        for (const childComponent of component.getChildren()) {
-          if (filterSet.has(childComponent)) {
-            this.add(childComponent);
-            sourceComponents.add(childComponent);
+      if (filterSet) {
+        const includedInWildcard = filterSet.has({
+          fullName: ComponentSet.WILDCARD,
+          type: component.type,
+        });
+        const parentInFilter =
+          component.parent &&
+          (filterSet.has(component.parent) ||
+            filterSet.has({
+              fullName: ComponentSet.WILDCARD,
+              type: component.parent.type,
+            }));
+        if (filterSet.has(component) || includedInWildcard || parentInFilter) {
+          this.add(component);
+          sourceComponents.add(component);
+        } else {
+          // have to check for any individually addressed children in the filter set
+          for (const childComponent of component.getChildren()) {
+            if (filterSet.has(childComponent)) {
+              this.add(childComponent);
+              sourceComponents.add(childComponent);
+            }
           }
         }
+      } else {
+        this.add(component);
+        sourceComponents.add(component);
       }
     }
 

src/collections/index.ts

@@ -5,4 +5,4 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 export { ComponentSet } from './componentSet';
-export { FromSourceOptions, FromManifestOptions, SourceComponentOptions } from './types';
+export { FromSourceOptions, FromManifestOptions, ResolveOptions } from './types';

src/collections/types.ts

@@ -4,7 +4,7 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-import { MetadataComponent } from '../common';
+import { ComponentLike } from '../common';
 import { RegistryAccess, TreeContainer } from '../metadata-registry';
 import { ComponentSet } from './componentSet';
 
@@ -20,11 +20,12 @@ export interface PackageManifestObject {
   };
 }
 
-export interface ComponentSetOptions {
+interface ComponentSetOptions {
   registry?: RegistryAccess;
 }
 
 export interface FromSourceOptions extends ComponentSetOptions {
+  filter?: Iterable<ComponentLike> | ComponentSet;
   tree?: TreeContainer;
 }
 
@@ -33,7 +34,4 @@ export interface FromManifestOptions extends FromSourceOptions {
   literalWildcard?: boolean;
 }
 
-export interface SourceComponentOptions {
-  tree?: TreeContainer;
-  filter?: MetadataComponent[] | ComponentSet;
-}
+export type ResolveOptions = Omit<FromSourceOptions, 'registry'>;

src/convert/convertContext.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2020, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { WriteInfo, WriterFormat } from './types';
+import { SourceComponent } from '../metadata-registry';
+import { join } from 'path';
+import { JsToXml } from './streams';
+import { MetadataComponent } from '../common';
+import { JsonArray, JsonMap } from '@salesforce/ts-types';
+import { ComponentSet } from '../collections';
+
+abstract class ConvertTransactionFinalizer<T> {
+  protected abstract _state: T;
+
+  public setState(props: (state: T) => void): void {
+    props(this._state);
+  }
+
+  get state(): T {
+    return this._state;
+  }
+
+  public abstract finalize(): Promise<WriterFormat[]>;
+}
+
+export interface RecompositionState {
+  [componentKey: string]: {
+    /**
+     * Parent component that children are rolled up into
+     */
+    component?: SourceComponent;
+    /**
+     * Children to be rolled up into the parent file
+     */
+    children?: ComponentSet;
+  };
+}
+
+/**
+ * Merges child components that share the same parent in the conversion pipeline
+ * into a single file.
+ */
+class RecompositionFinalizer extends ConvertTransactionFinalizer<RecompositionState> {
+  protected _state: RecompositionState = {};
+
+  public async finalize(): Promise<WriterFormat[]> {
+    const writerData: WriterFormat[] = [];
+
+    for (const { component: parent, children } of Object.values(this.state)) {
+      const baseObject: JsonMap = await parent.parseXml();
+      const recomposedXmlObj = await this.recompose(children, baseObject);
+      writerData.push({
+        component: parent,
+        writeInfos: [
+          {
+            source: new JsToXml(recomposedXmlObj),
+            output: join(parent.type.directoryName, `${parent.fullName}.${parent.type.suffix}`),
+          },
+        ],
+      });
+    }
+
+    return writerData;
+  }
+
+  private async recompose(children: ComponentSet, baseXmlObj: any): Promise<JsonMap> {
+    for (const child of children) {
+      const { directoryName: groupNode } = child.type;
+      const { name: parentName } = child.parent.type;
+      const xmlObj = await (child as SourceComponent).parseXml();
+      const childContents = xmlObj[child.type.name];
+
+      if (!baseXmlObj[parentName][groupNode]) {
+        baseXmlObj[parentName][groupNode] = [];
+      }
+      (baseXmlObj[parentName][groupNode] as JsonArray).push(childContents);
+    }
+    return baseXmlObj;
+  }
+}
+
+export interface DecompositionState {
+  [componentKey: string]: {
+    foundMerge?: boolean;
+    writeInfo?: WriteInfo;
+    origin?: MetadataComponent;
+  };
+}
+
+/**
+ * Creates write infos for any children that haven't been written yet. Children may
+ * delay being written in order to find potential existing children to merge
+ * with in the conversion pipeline.
+ */
+class DecompositionFinalizer extends ConvertTransactionFinalizer<DecompositionState> {
+  protected _state: DecompositionState = {};
+
+  public async finalize(): Promise<WriterFormat[]> {
+    const writerData: WriterFormat[] = [];
+
+    for (const toDecompose of Object.values(this._state)) {
+      if (!toDecompose.foundMerge) {
+        writerData.push({
+          component: toDecompose.origin.parent ?? toDecompose.origin,
+          writeInfos: [toDecompose.writeInfo],
+        });
+      }
+    }
+
+    return writerData;
+  }
+}
+
+/**
+ * A state manager over the course of a single metadata conversion call.
+ */
+export class ConvertContext {
+  public readonly decomposition = new DecompositionFinalizer();
+  public readonly recomposition = new RecompositionFinalizer();
+
+  public async *executeFinalizers(): AsyncIterable<WriterFormat[]> {
+    for (const member of Object.values(this)) {
+      if (member instanceof ConvertTransactionFinalizer) {
+        yield member.finalize();
+      }
+    }
+  }
+}

src/convert/metadataConverter.ts

@@ -90,7 +90,7 @@ export class MetadataConverter {
 
       const conversionPipeline = pipeline(
         new ComponentReader(components),
-        new ComponentConverter(targetFormat, this.registry, undefined, mergeSet),
+        new ComponentConverter(targetFormat, this.registry, mergeSet),
         writer
       );
       tasks.push(conversionPipeline);