feat: build ComponentSet using metadata and an org connection (#1182)

* feat: build ComponentSet from metadata and org connection

* fix: use minimatch for more expected matching results

* fix: address review comments

src/collections/componentSet.ts

@@ -283,7 +283,7 @@ export class ComponentSet extends LazyCollection<MetadataComponent> {
       }
     }
 
-    const connectionResolver = new ConnectionResolver(usernameOrConnection, options.registry);
+    const connectionResolver = new ConnectionResolver(usernameOrConnection, options.registry, options.metadataTypes);
     const manifest = await connectionResolver.resolve(options.componentFilter);
     const result = new ComponentSet([], options.registry);
     result.apiVersion = manifest.apiVersion;

src/collections/componentSetBuilder.ts

@@ -10,8 +10,10 @@
 import * as path from 'node:path';
 import { StateAggregator, Logger, SfError, Messages } from '@salesforce/core';
 import * as fs from 'graceful-fs';
+import * as minimatch from 'minimatch';
 import { ComponentSet } from '../collections';
 import { RegistryAccess } from '../registry';
+import { FileProperties } from '../client';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/source-deploy-retrieve', 'sdr');
@@ -57,6 +59,13 @@ export class ComponentSetBuilder {
     const logger = Logger.childFromRoot('componentSetBuilder');
     let componentSet: ComponentSet | undefined;
 
+    /**
+     * A map used when building a ComponentSet from metadata type/name pairs
+     * key = a metadata type, e.g. `ApexClass`
+     * value = an array of metadata names, e.g. `['foo_*', 'BarClass']`
+     */
+    const mdMap = new Map<string, string[]>();
+
     const { sourcepath, manifest, metadata, packagenames, apiversion, sourceapiversion, org } = options;
     try {
       if (sourcepath) {
@@ -103,31 +112,41 @@ export class ComponentSetBuilder {
 
         // Build a Set of metadata entries
         metadata.metadataEntries.forEach((rawEntry) => {
-          const splitEntry = rawEntry.split(':').map((entry) => entry.trim());
+          const [mdType, mdName] = rawEntry.split(':').map((entry) => entry.trim());
           // The registry will throw if it doesn't know what this type is.
-          registry.getTypeByName(splitEntry[0]);
+          registry.getTypeByName(mdType);
+
+          // Add metadata entries to a map for possible use with an org connection below.
+          const mdMapEntry = mdMap.get(mdType);
+          if (mdMapEntry) {
+            mdMapEntry.push(mdName);
+          } else {
+            mdMap.set(mdType, [mdName]);
+          }
+
           // this '.*' is a surprisingly valid way to specify a metadata, especially a DEB :sigh:
           // https://github.com/salesforcecli/plugin-deploy-retrieve/blob/main/test/nuts/digitalExperienceBundle/constants.ts#L140
           // because we're filtering from what we have locally, this won't allow you to retrieve new metadata (on the server only) using the partial wildcard
           // to do that, you'd need check the size of the CS created below, see if it's 0, and then query the org for the metadata that matches the regex
           // but building a CS from a metadata argument doesn't require an org, so we can't do that here
-          if (splitEntry[1]?.includes('*') && splitEntry[1]?.length > 1 && !splitEntry[1].includes('.*')) {
+          if (mdName?.includes('*') && mdName?.length > 1 && !mdName.includes('.*')) {
             // get all components of the type, and then filter by the regex of the fullName
             ComponentSet.fromSource({
               fsPaths: directoryPaths,
-              include: new ComponentSet([{ type: splitEntry[0], fullName: ComponentSet.WILDCARD }]),
+              include: new ComponentSet([{ type: mdType, fullName: ComponentSet.WILDCARD }]),
             })
               .getSourceComponents()
               .toArray()
-              .filter((cs) => Boolean(cs.fullName.match(new RegExp(splitEntry[1]))))
+              // using minimatch versus RegExp provides better (more expected) matching results
+              .filter((cs) => minimatch(cs.fullName, mdName))
               .map((match) => {
                 compSetFilter.add(match);
                 componentSet?.add(match);
               });
           } else {
             const entry = {
-              type: splitEntry[0],
-              fullName: splitEntry.length === 1 ? '*' : splitEntry[1],
+              type: mdType,
+              fullName: !mdName ? '*' : mdName,
             };
             // Add to the filtered ComponentSet for resolved source paths,
             // and the unfiltered ComponentSet to build the correct manifest.
@@ -147,15 +166,35 @@ export class ComponentSetBuilder {
       // Resolve metadata entries with an org connection
       if (org) {
         componentSet ??= new ComponentSet();
-        logger.debug(`Building ComponentSet from targetUsername: ${org.username}`);
+
+        let debugMsg = `Building ComponentSet from targetUsername: ${org.username}`;
+
+        // *** Default Filter ***
+        // exclude components based on the results of componentFilter function
+        // components with namespacePrefix where org.exclude includes manageableState (to exclude managed packages)
+        // components with namespacePrefix where manageableState equals undefined (to exclude components e.g. InstalledPackage)
+        // components where org.exclude includes manageableState (to exclude packages without namespacePrefix e.g. unlocked packages)
+        let componentFilter = (component: Partial<FileProperties>): boolean =>
+          !component?.manageableState || !org.exclude?.includes(component.manageableState);
+
+        if (metadata) {
+          debugMsg += ` filtered by metadata: ${metadata.metadataEntries.toString()}`;
+
+          componentFilter = (component: Partial<FileProperties>): boolean => {
+            if (component.type && component.fullName) {
+              const mdMapEntry = mdMap.get(component.type);
+              // using minimatch versus RegExp provides better (more expected) matching results
+              return !!mdMapEntry && mdMapEntry.some((mdName) => minimatch(component.fullName as string, mdName));
+            }
+            return false;
+          };
+        }
+
+        logger.debug(debugMsg);
         const fromConnection = await ComponentSet.fromConnection({
           usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username,
-          // exclude components based on the results of componentFilter function
-          // components with namespacePrefix where org.exclude includes manageableState (to exclude managed packages)
-          // components with namespacePrefix where manageableState equals undefined (to exclude components e.g. InstalledPackage)
-          // components where org.exclude includes manageableState (to exclude packages without namespacePrefix e.g. unlocked packages)
-          componentFilter: (component): boolean =>
-            !component?.manageableState || !org.exclude?.includes(component.manageableState),
+          componentFilter,
+          metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined,
         });
 
         for (const comp of fromConnection) {

src/collections/types.ts

@@ -88,4 +88,8 @@ export interface FromConnectionOptions extends OptionalTreeRegistryOptions {
    * filter the result components to e.g. remove managed components
    */
   componentFilter?: (component: Partial<FileProperties>) => boolean;
+  /**
+   * array of metadata type names to use for `connection.metadata.list()`
+   */
+  metadataTypes?: string[];
 }

src/resolve/connectionResolver.ts

@@ -31,10 +31,18 @@ export class ConnectionResolver {
   private connection: Connection;
   private registry: RegistryAccess;
 
-  public constructor(connection: Connection, registry = new RegistryAccess()) {
+  // Array of metadata type names to use for listMembers. By default it includes
+  // all types defined in the registry.
+  private mdTypeNames: string[];
+
+  public constructor(connection: Connection, registry = new RegistryAccess(), mdTypes?: string[]) {
     this.connection = connection;
     this.registry = registry;
     this.logger = Logger.childFromRoot(this.constructor.name);
+    this.mdTypeNames = mdTypes?.length
+      ? // ensure the types passed in are valid per the registry
+        mdTypes.filter((t) => this.registry.getTypeByName(t))
+      : Object.values(defaultRegistry.types).map((t) => t.name);
   }
 
   public async resolve(
@@ -44,10 +52,9 @@ export class ConnectionResolver {
     const childrenPromises: Array<Promise<FileProperties[]>> = [];
     const componentTypes: Set<MetadataType> = new Set();
     const lifecycle = Lifecycle.getInstance();
-    const componentPromises: Array<Promise<FileProperties[]>> = [];
-    for (const type of Object.values(defaultRegistry.types)) {
-      componentPromises.push(this.listMembers({ type: type.name }));
-    }
+
+    const componentPromises = this.mdTypeNames.map((type) => this.listMembers({ type }));
+
     (await Promise.all(componentPromises)).map(async (componentResult) => {
       for (const component of componentResult) {
         let componentType: MetadataType;

test/collections/componentSetBuilder.test.ts

@@ -397,6 +397,36 @@ describe('ComponentSetBuilder', () => {
       expect(compSet.has(apexClassComponent)).to.equal(true);
     });
 
+    it('should create ComponentSet from org connection and metadata', async () => {
+      const mdCompSet = new ComponentSet();
+      mdCompSet.add(apexClassComponent);
+
+      fromSourceStub.returns(mdCompSet);
+      const packageDir1 = path.resolve('force-app');
+
+      componentSet.add(apexClassWildcardMatch);
+      fromConnectionStub.resolves(componentSet);
+      const options = {
+        sourcepath: undefined,
+        metadata: {
+          metadataEntries: ['ApexClass:MyClas*'],
+          directoryPaths: [packageDir1],
+        },
+        manifest: undefined,
+        org: {
+          username: 'manifest-test@org.com',
+          exclude: [],
+        },
+      };
+
+      const compSet = await ComponentSetBuilder.build(options);
+      expect(fromSourceStub.calledTwice).to.equal(true);
+      expect(fromConnectionStub.calledOnce).to.equal(true);
+      expect(compSet.size).to.equal(2);
+      expect(compSet.has(apexClassComponent)).to.equal(true);
+      expect(compSet.has(apexClassWildcardMatch)).to.equal(true);
+    });
+
     it('should create ComponentSet from manifest and multiple package', async () => {
       fileExistsSyncStub.returns(true);
 

test/resolve/connectionResolver.test.ts

@@ -127,6 +127,39 @@ describe('ConnectionResolver', () => {
       ];
       expect(result.components).to.deep.equal(expected);
     });
+    it('should resolve components with specified types', async () => {
+      const metadataQueryStub = $$.SANDBOX.stub(connection.metadata, 'list');
+
+      metadataQueryStub.withArgs({ type: 'ApexClass' }).resolves([
+        {
+          ...StdFileProperty,
+          fileName: 'classes/MyApexClass1.class',
+          fullName: 'MyApexClass1',
+          type: 'ApexClass',
+        },
+        {
+          ...StdFileProperty,
+          fileName: 'classes/MyApexClass2.class',
+          fullName: 'MyApexClass2',
+          type: 'ApexClass',
+        },
+      ]);
+
+      const resolver = new ConnectionResolver(connection, undefined, ['ApexClass']);
+      const result = await resolver.resolve();
+      const expected: MetadataComponent[] = [
+        {
+          fullName: 'MyApexClass1',
+          type: registry.types.apexclass,
+        },
+        {
+          fullName: 'MyApexClass2',
+          type: registry.types.apexclass,
+        },
+      ];
+      expect(result.components).to.deep.equal(expected);
+      expect(metadataQueryStub.calledOnce).to.be.true;
+    });
     it('should resolve components with invalid type returned by metadata api', async () => {
       const metadataQueryStub = $$.SANDBOX.stub(connection.metadata, 'list');
       metadataQueryStub.withArgs({ type: 'CustomLabels' }).resolves([