feat: support split CustomLabels on deploy and retrieve (#278)

* feat: support split CustomLabels on deploy and retrieve

* chore: rename uniqueIdAttribute to uniqueIdElement

* fix: allow unprocessed component to be the default component

* chore: bump shelljs

Author: mdonnalley

package.json

@@ -29,6 +29,8 @@
   },
   "dependencies": {
     "@salesforce/core": "2.13.0",
+    "@salesforce/kit": "1.5.0",
+    "@salesforce/ts-types": "^1.4.2",
     "archiver": "4.0.1",
     "fast-xml-parser": "^3.17.4",
     "gitignore-parser": "0.0.2",
@@ -41,9 +43,9 @@
     "@commitlint/cli": "^7",
     "@commitlint/config-conventional": "^7",
     "@salesforce/ts-sinon": "^1.1.2",
-    "@salesforce/ts-types": "^1.4.2",
     "@types/archiver": "^3.1.0",
     "@types/chai": "^4",
+    "@types/deep-equal-in-any-order": "^1.0.1",
     "@types/mime": "2.0.3",
     "@types/mkdirp": "0.5.2",
     "@types/mocha": "^5",
@@ -55,6 +57,7 @@
     "chai": "^4",
     "commitizen": "^3.0.5",
     "cz-conventional-changelog": "^2.1.0",
+    "deep-equal-in-any-order": "^1.1.4",
     "deepmerge": "^4.2.2",
     "eslint": "^6.8.0",
     "eslint-config-prettier": "^6.11.0",
@@ -67,7 +70,7 @@
     "mocha-junit-reporter": "^1.23.3",
     "nyc": "^14.1.1",
     "prettier": "2.0.5",
-    "shelljs": "0.8.3",
+    "shelljs": "0.8.4",
     "shx": "^0.3.2",
     "sinon": "^7.3.1",
     "source-map-support": "^0.5.16",

src/client/metadataApiRetrieve.ts

@@ -126,9 +126,9 @@ export class MetadataApiRetrieve extends MetadataTransfer<
   protected async checkStatus(id: string): Promise<MetadataApiRetrieveStatus> {
     const connection = await this.getConnection();
     // Recasting to use the project's RetrieveResult type
-    return (connection.metadata.checkRetrieveStatus(id) as unknown) as Promise<
-      MetadataApiRetrieveStatus
-    >;
+    const status = await connection.metadata.checkRetrieveStatus(id);
+    status.fileProperties = normalizeToArray(status.fileProperties);
+    return status as MetadataApiRetrieveStatus;
   }
 
   protected async post(result: MetadataApiRetrieveStatus): Promise<RetrieveResult> {

src/collections/componentSet.ts

@@ -257,7 +257,7 @@ export class ComponentSet extends LazyCollection<MetadataComponent> {
       indentBy: new Array(indentation + 1).join(' '),
       ignoreAttributes: false,
     });
-    const toParse = this.getObject() as any;
+    const toParse = this.getObject();
     toParse.Package[XML_NS_KEY] = XML_NS_URL;
     return XML_DECL.concat(j2x.parse(toParse));
   }

src/collections/types.ts

@@ -4,7 +4,7 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-import { OptionalTreeRegistryOptions } from '../common';
+import { OptionalTreeRegistryOptions, XML_NS_KEY } from '../common';
 import { ComponentSet } from './componentSet';
 
 export interface PackageTypeMembers {
@@ -17,6 +17,7 @@ export interface PackageManifestObject {
     types: PackageTypeMembers[];
     version: string;
     fullName: string;
+    [XML_NS_KEY]?: string;
   };
 }
 

src/convert/convertContext.ts

@@ -6,11 +6,13 @@
  */
 import { WriteInfo, WriterFormat } from './types';
 import { MetadataComponent, SourceComponent } from '../resolve';
-import { join } from 'path';
+import { basename, dirname, join, resolve, sep } from 'path';
 import { JsToXml } from './streams';
-import { XML_NS_KEY, XML_NS_URL } from '../common';
-import { JsonArray, JsonMap } from '@salesforce/ts-types';
+import { META_XML_SUFFIX, XML_NS_KEY, XML_NS_URL } from '../common';
+import { getString, JsonArray, JsonMap } from '@salesforce/ts-types';
 import { ComponentSet } from '../collections';
+import { RecompositionStrategy } from '../registry/types';
+import { isEmpty } from '@salesforce/kit';
 
 abstract class ConvertTransactionFinalizer<T> {
   protected abstract _state: T;
@@ -23,7 +25,7 @@ abstract class ConvertTransactionFinalizer<T> {
     return this._state;
   }
 
-  public abstract finalize(): Promise<WriterFormat[]>;
+  public abstract finalize(defaultDirectory?: string): Promise<WriterFormat[]>;
 }
 
 export interface RecompositionState {
@@ -66,13 +68,16 @@ class RecompositionFinalizer extends ConvertTransactionFinalizer<RecompositionSt
   }
 
   private async recompose(children: ComponentSet, parent: SourceComponent): Promise<JsonMap> {
-    const parentXmlObj: JsonMap = await parent.parseXml();
+    const parentXmlObj =
+      parent.type.strategies.recomposition === RecompositionStrategy.StartEmpty
+        ? {}
+        : await parent.parseXml();
 
     for (const child of children) {
       const { directoryName: groupName } = child.type;
       const { name: parentName } = child.parent.type;
       const xmlObj = await (child as SourceComponent).parseXml();
-      const childContents = xmlObj[child.type.name];
+      const childContents = xmlObj[child.type.name] || xmlObj;
 
       if (!parentXmlObj[parentName]) {
         parentXmlObj[parentName] = { [XML_NS_KEY]: XML_NS_URL };
@@ -88,7 +93,6 @@ class RecompositionFinalizer extends ConvertTransactionFinalizer<RecompositionSt
 
       group.push(childContents);
     }
-
     return parentXmlObj;
   }
 }
@@ -125,17 +129,223 @@ class DecompositionFinalizer extends ConvertTransactionFinalizer<DecompositionSt
   }
 }
 
+export interface NonDecompositionState {
+  claimed: ChildIndex;
+  unclaimed: ChildIndex;
+}
+
+type ChildIndex = {
+  [componentKey: string]: {
+    parent: SourceComponent;
+    children: {
+      [childName: string]: JsonMap;
+    };
+  };
+};
+
+/**
+ * Merges child components that share the same parent in the conversion pipeline
+ * into a single file.
+ *
+ * Inserts unclaimed child components into the parent that belongs to the default package
+ */
+class NonDecompositionFinalizer extends ConvertTransactionFinalizer<NonDecompositionState> {
+  protected _state: NonDecompositionState = {
+    unclaimed: {},
+    claimed: {},
+  };
+
+  public async finalize(defaultDirectory: string): Promise<WriterFormat[]> {
+    await this.finalizeState(defaultDirectory);
+
+    const writerData: WriterFormat[] = [];
+    for (const { parent, children } of Object.values(this.state.claimed)) {
+      const recomposedXmlObj = await this.recompose(Object.values(children), parent);
+
+      writerData.push({
+        component: parent,
+        writeInfos: [{ source: new JsToXml(recomposedXmlObj), output: parent.xml }],
+      });
+    }
+
+    for (const { parent, children } of Object.values(this.state.unclaimed)) {
+      const recomposedXmlObj = await this.recompose(Object.values(children), parent);
+      writerData.push({
+        component: parent,
+        writeInfos: [
+          { source: new JsToXml(recomposedXmlObj), output: this.getDefaultOutput(parent) },
+        ],
+      });
+    }
+
+    return writerData;
+  }
+
+  /**
+   * This method finalizes the state by:
+   * - finding any "unprocessed components" (nondecomposed metadata types can exist in multiple locations under the same name
+   *   so we have to find all components that could potentially claim children)
+   * - removing any children from the unclaimed state that have been claimed by the unprocessed components
+   * - removing any children from the unclaimed state that have already been claimed by a prent in the claimed state
+   * - merging the remaining unclaimed children into the default parent component (either the component that matches the
+   *   defaultDirectory or the first parent component)
+   */
+  private async finalizeState(defaultDirectory: string): Promise<void> {
+    if (isEmpty(this.state.claimed)) {
+      return;
+    }
+
+    const unprocessedComponents = this.getUnprocessedComponents(defaultDirectory);
+    const parentPaths = Object.keys(this.state.claimed).concat(
+      unprocessedComponents.map((c) => c.xml)
+    );
+
+    const defaultComponentKey =
+      parentPaths.find((p) => p.startsWith(defaultDirectory)) || parentPaths[0];
+
+    const claimedChildren = [
+      ...this.getClaimedChildrenNames(),
+      ...(await this.getChildrenOfUnprocessedComponents(unprocessedComponents)),
+    ];
+
+    // merge unclaimed children into default parent component
+    for (const [key, childIndex] of Object.entries(this.state.unclaimed)) {
+      const pruned = Object.entries(childIndex.children).reduce((result, [childName, childXml]) => {
+        return !claimedChildren.includes(childName)
+          ? Object.assign(result, { [childName]: childXml })
+          : result;
+      }, {});
+      delete this.state.unclaimed[key];
+      if (this.state.claimed[defaultComponentKey]) {
+        this.state.claimed[defaultComponentKey].children = Object.assign(
+          {},
+          this.state.claimed[defaultComponentKey].children,
+          pruned
+        );
+      }
+    }
+  }
+
+  /**
+   * Returns the "unprocessed components"
+   *
+   * An unprocessed component is a component that was not resolved during component resolution.
+   * This typically only happens when a specific source path was resolved. This is problematic for
+   * nondecomposed metadata types (like CustomLabels) because we need to know the location of each
+   * child type before recomposing the final xml. So in order for each of the children to be properly
+   * claimed, we have to create new ComponentSet that will have all the parent components.
+   */
+  private getUnprocessedComponents(defaultDirectory: string): SourceComponent[] {
+    if (isEmpty(this.state.unclaimed)) {
+      return [];
+    }
+    const parents = this.getParentsOfClaimedChildren();
+    const filterSet = new ComponentSet(parents);
+
+    const { tree } = parents[0];
+    const projectDir = resolve(dirname(defaultDirectory));
+    const parentDirs = Object.keys(this.state.claimed).map((k) => {
+      const parts = k.split(sep);
+      const partIndex = parts.findIndex((p) => basename(projectDir) === p);
+      return parts[partIndex + 1];
+    });
+
+    const fsPaths = tree
+      .readDirectory(projectDir)
+      .map((p) => join(projectDir, p))
+      .filter((p) => {
+        const dirName = basename(p);
+        // Only return directories that are likely to be a project directory
+        return (
+          tree.isDirectory(p) &&
+          !dirName.startsWith('.') &&
+          dirName !== 'config' &&
+          dirName !== 'node_modules' &&
+          !parentDirs.includes(dirName)
+        );
+      });
+
+    const unprocessedComponents = ComponentSet.fromSource({ fsPaths, include: filterSet })
+      .getSourceComponents()
+      .filter((component) => !this.state.claimed[component.xml]);
+    return unprocessedComponents.toArray();
+  }
+
+  /**
+   * Returns the children of "unprocessed components"
+   */
+  private async getChildrenOfUnprocessedComponents(
+    unprocessedComponents: SourceComponent[]
+  ): Promise<string[]> {
+    const childrenOfUnprocessed = [];
+    for (const component of unprocessedComponents) {
+      for (const child of component.getChildren()) {
+        const xml = await child.parseXml();
+        const childName = getString(xml, child.type.uniqueIdElement);
+        childrenOfUnprocessed.push(childName);
+      }
+    }
+    return childrenOfUnprocessed;
+  }
+
+  private async recompose(children: JsonMap[], parent: SourceComponent): Promise<JsonMap> {
+    const parentXmlObj =
+      parent.type.strategies.recomposition === RecompositionStrategy.StartEmpty
+        ? {}
+        : await parent.parseXml();
+    const groupName = parent.type.directoryName;
+    const parentName = parent.type.name;
+    for (const child of children) {
+      if (!parentXmlObj[parentName]) {
+        parentXmlObj[parentName] = { [XML_NS_KEY]: XML_NS_URL };
+      }
+
+      const parent = parentXmlObj[parentName] as JsonMap;
+
+      if (!parent[groupName]) {
+        parent[groupName] = [];
+      }
+
+      const group = parent[groupName] as JsonArray;
+
+      group.push(child);
+    }
+
+    return parentXmlObj;
+  }
+
+  private getDefaultOutput(component: SourceComponent): string {
+    const { fullName } = component;
+    const [baseName] = fullName.split('.');
+    const output = `${baseName}.${component.type.suffix}${META_XML_SUFFIX}`;
+
+    return join(component.getPackageRelativePath('', 'source'), output);
+  }
+
+  private getClaimedChildrenNames(): string[] {
+    return Object.values(this.state.claimed).reduce(
+      (x, y) => x.concat(Object.keys(y.children)),
+      []
+    );
+  }
+
+  private getParentsOfClaimedChildren(): SourceComponent[] {
+    return Object.values(this.state.claimed).reduce((x, y) => x.concat([y.parent]), []);
+  }
+}
+
 /**
  * A state manager over the course of a single metadata conversion call.
  */
 export class ConvertContext {
   public readonly decomposition = new DecompositionFinalizer();
   public readonly recomposition = new RecompositionFinalizer();
+  public readonly nonDecomposition = new NonDecompositionFinalizer();
 
-  public async *executeFinalizers(): AsyncIterable<WriterFormat[]> {
+  public async *executeFinalizers(defaultDirectory?: string): AsyncIterable<WriterFormat[]> {
     for (const member of Object.values(this)) {
       if (member instanceof ConvertTransactionFinalizer) {
-        yield member.finalize();
+        yield member.finalize(defaultDirectory);
       }
     }
   }

src/convert/metadataConverter.ts

@@ -60,6 +60,7 @@ export class MetadataConverter {
       let writer: ComponentWriter;
       let mergeSet: ComponentSet;
       let packagePath: SourcePath;
+      let defaultDirectory: SourcePath;
 
       switch (output.type) {
         case 'directory':
@@ -68,6 +69,7 @@ export class MetadataConverter {
           }
           manifestContents = cs.getPackageXml();
           packagePath = this.getPackagePath(output);
+          defaultDirectory = packagePath;
           writer = new StandardWriter(packagePath);
           if (!isSource) {
             const manifestPath = join(packagePath, MetadataConverter.PACKAGE_XML_FILE);
@@ -80,6 +82,7 @@ export class MetadataConverter {
           }
           manifestContents = cs.getPackageXml();
           packagePath = this.getPackagePath(output);
+          defaultDirectory = packagePath;
           writer = new ZipWriter(packagePath);
           if (!isSource) {
             (writer as ZipWriter).addToZip(manifestContents, MetadataConverter.PACKAGE_XML_FILE);
@@ -89,6 +92,7 @@ export class MetadataConverter {
           if (!isSource) {
             throw new LibraryError('error_merge_metadata_target_unsupported');
           }
+          defaultDirectory = output.defaultDirectory;
           mergeSet = new ComponentSet();
           // since child components are composed in metadata format, we need to merge using the parent
           for (const component of output.mergeWith) {
@@ -100,7 +104,7 @@ export class MetadataConverter {
 
       const conversionPipeline = pipeline(
         new ComponentReader(components),
-        new ComponentConverter(targetFormat, this.registry, mergeSet),
+        new ComponentConverter(targetFormat, this.registry, mergeSet, defaultDirectory),
         writer
       );
       tasks.push(conversionPipeline);