diff --git a/CHANGELOG.md b/CHANGELOG.md index 6951ba5b1e..7716b8130b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## [11.1.2](https://github.com/forcedotcom/source-deploy-retrieve/compare/11.1.1...11.1.2) (2024-04-25) + + +### Bug Fixes + +* forceignore ignores output file correctly - cleanup extra type d… ([#1295](https://github.com/forcedotcom/source-deploy-retrieve/issues/1295)) ([287b13e](https://github.com/forcedotcom/source-deploy-retrieve/commit/287b13e60549fc5bc5a104a4d15a0ff549301d3b)) + + + ## [11.1.1](https://github.com/forcedotcom/source-deploy-retrieve/compare/11.1.0...11.1.1) (2024-04-23) diff --git a/METADATA_SUPPORT.md b/METADATA_SUPPORT.md index 5c592882ff..260f9a5e70 100644 --- a/METADATA_SUPPORT.md +++ b/METADATA_SUPPORT.md @@ -4,7 +4,7 @@ This list compares metadata types found in Salesforce v60 with the [metadata reg This repository is used by both the Salesforce CLIs and Salesforce's VSCode Extensions. -Currently, there are 565/597 supported metadata types. +Currently, there are 565/598 supported metadata types. For status on any existing gaps, please search or file an issue in the [Salesforce CLI issues only repo](https://github.com/forcedotcom/cli/issues). To contribute a new metadata type, please see the [Contributing Metadata Types to the Registry](./contributing/metadata.md) @@ -303,6 +303,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t | GenAiFunction | ❌ | Not supported, but support could be added | | GenAiPlanner | ❌ | Not supported, but support could be added | | GenAiPlugin | ❌ | Not supported, but support could be added | +| GenAiPluginInstructionDef | ❌ | Not supported, but support could be added | | GlobalValueSet | ✅ | | | GlobalValueSetTranslation | ✅ | | | GoogleAppsSettings | ✅ | | diff --git a/package.json b/package.json index 59ebf406b2..59e2017b24 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@salesforce/source-deploy-retrieve", - "version": "11.1.1", + "version": "11.1.2", "description": "JavaScript library to run Salesforce metadata deploys and retrieves", "main": "lib/src/index.js", "author": "Salesforce", diff --git a/src/client/metadataApiRetrieve.ts b/src/client/metadataApiRetrieve.ts index 3de873e484..9c21b038ae 100644 --- a/src/client/metadataApiRetrieve.ts +++ b/src/client/metadataApiRetrieve.ts @@ -119,7 +119,7 @@ export class MetadataApiRetrieve extends MetadataTransfer< MetadataApiRetrieveOptions > { public static DEFAULT_OPTIONS: Partial = { merge: false }; - private options: MetadataApiRetrieveOptions; + private readonly options: MetadataApiRetrieveOptions; private orgId?: string; public constructor(options: MetadataApiRetrieveOptions) { diff --git a/src/collections/componentSetBuilder.ts b/src/collections/componentSetBuilder.ts index 9dc12e185a..9d8827cc65 100644 --- a/src/collections/componentSetBuilder.ts +++ b/src/collections/componentSetBuilder.ts @@ -6,7 +6,7 @@ */ import * as path from 'node:path'; -import { StateAggregator, Logger, SfError, Messages } from '@salesforce/core'; +import { Logger, Messages, SfError, StateAggregator } from '@salesforce/core'; import fs from 'graceful-fs'; import minimatch from 'minimatch'; import { MetadataComponent } from '../resolve/types'; @@ -15,6 +15,7 @@ import { ComponentSet } from '../collections/componentSet'; import { RegistryAccess } from '../registry/registryAccess'; import type { FileProperties } from '../client/types'; import { MetadataType } from '../registry/types'; +import { MetadataResolver } from '../resolve'; import { DestructiveChangesType, FromConnectionOptions } from './types'; Messages.importMessagesDirectory(__dirname); @@ -69,6 +70,7 @@ export class ComponentSetBuilder { * @param options: options for creating a ComponentSet */ + // eslint-disable-next-line complexity public static async build(options: ComponentSetOptions): Promise { const logger = Logger.childFromRoot('componentSetBuilder'); let componentSet: ComponentSet | undefined; @@ -148,7 +150,21 @@ export class ComponentSetBuilder { include: componentSetFilter, registry: registryAccess, }); - componentSet.forceIgnoredPaths = resolvedComponents.forceIgnoredPaths; + + if (resolvedComponents.forceIgnoredPaths) { + // if useFsForceIgnore = true, then we won't be able to resolve a forceignored path, + // which we need to do to get the ignored source component + const resolver = new MetadataResolver(registryAccess, undefined, false); + + for (const ignoredPath of resolvedComponents.forceIgnoredPaths ?? []) { + resolver.getComponentsFromPath(ignoredPath).map((ignored) => { + componentSet = componentSet?.filter( + (resolved) => !(resolved.fullName === ignored.name && resolved.type === ignored.type) + ); + }); + } + } + resolvedComponents.toArray().map(addToComponentSet(componentSet)); } @@ -162,7 +178,7 @@ export class ComponentSetBuilder { }` ); - const mdMap: MetadataMap = metadata + const mdMap = metadata ? buildMapFromComponents(metadata.metadataEntries.map(entryToTypeAndName(registryAccess))) : (new Map() as MetadataMap); @@ -317,7 +333,7 @@ const typeAndNameToMetadataComponents = // TODO: use Map.groupBy when it's available const buildMapFromComponents = (components: MetadataTypeAndMetadataName[]): MetadataMap => { - const mdMap: MetadataMap = new Map(); + const mdMap: MetadataMap = new Map(); components.map((cmp) => { mdMap.set(cmp.type.name, [...(mdMap.get(cmp.type.name) ?? []), cmp.metadataName]); }); diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index 90c397962f..b542e0da07 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -183,9 +183,7 @@ function getMergeConfigOutputs( mergeSet.add(component.parent ?? component); } const writer = new StandardWriter(output.defaultDirectory); - if (output.forceIgnoredPaths) { - writer.forceIgnoredPaths = output.forceIgnoredPaths; - } + return { writer, mergeSet, diff --git a/src/convert/streams.ts b/src/convert/streams.ts index 6e5cf3b689..1a68528920 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -4,7 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { basename, dirname, isAbsolute, join } from 'node:path'; +import { isAbsolute, join } from 'node:path'; import { pipeline as cbPipeline, Readable, Stream, Transform, Writable } from 'node:stream'; import { promisify } from 'node:util'; import { Messages, SfError } from '@salesforce/core'; @@ -20,6 +20,7 @@ import { ComponentSet } from '../collections/componentSet'; import { RegistryAccess } from '../registry/registryAccess'; import { ensureFileExists } from '../utils/fileSystemHandler'; import { ComponentStatus, FileResponseSuccess } from '../client/types'; +import { ForceIgnore } from '../resolve'; import { MetadataTransformerFactory } from './transformers/metadataTransformerFactory'; import { ConvertContext } from './convertContext/convertContext'; import { SfdxFileFormat, WriteInfo, WriterFormat } from './types'; @@ -35,7 +36,6 @@ export const stream2buffer = async (stream: Stream): Promise => const buf = Array(); stream.on('data', (chunk) => buf.push(chunk)); stream.on('end', () => resolve(Buffer.concat(buf))); - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions stream.on('error', (err) => reject(`error converting stream - ${err}`)); }); @@ -113,7 +113,6 @@ export class ComponentConverter extends Transform { } export abstract class ComponentWriter extends Writable { - public forceIgnoredPaths: Set = new Set(); protected rootDestination?: SourcePath; protected logger: Logger; @@ -128,9 +127,11 @@ export class StandardWriter extends ComponentWriter { /** filepaths that converted files were written to */ public readonly converted: string[] = []; public readonly deleted: FileResponseSuccess[] = []; + public readonly forceignore: ForceIgnore; public constructor(rootDestination: SourcePath) { super(rootDestination); + this.forceignore = ForceIgnore.findAndCreate(rootDestination); } public async _write(chunk: WriterFormat, encoding: string, callback: (err?: Error) => void): Promise { @@ -144,8 +145,7 @@ export class StandardWriter extends ComponentWriter { await Promise.all( chunk.writeInfos .map(makeWriteInfoAbsolute(this.rootDestination)) - .filter(existsOrDoesntMatchIgnored(this.forceIgnoredPaths)) - .filter((info) => !this.forceIgnoredPaths.has(info.output)) + .filter(existsOrDoesntMatchIgnored(this.forceignore)) .map((info) => { if (info.shouldDelete) { this.deleted.push({ @@ -288,11 +288,6 @@ const makeWriteInfoAbsolute = }); const existsOrDoesntMatchIgnored = - (ignoredPaths: Set) => + (forceignore: ForceIgnore) => (writeInfo: WriteInfo): boolean => - existsSync(writeInfo.output) || - [...ignoredPaths].every( - (ignoredPath) => - !dirname(ignoredPath).includes(dirname(writeInfo.output)) && - !basename(ignoredPath).includes(basename(writeInfo.output)) - ); + existsSync(writeInfo.output) || forceignore.accepts(writeInfo.output); diff --git a/src/resolve/adapters/baseSourceAdapter.ts b/src/resolve/adapters/baseSourceAdapter.ts index ae49d013b3..2e232a1a9b 100644 --- a/src/resolve/adapters/baseSourceAdapter.ts +++ b/src/resolve/adapters/baseSourceAdapter.ts @@ -35,8 +35,8 @@ export abstract class BaseSourceAdapter implements SourceAdapter { public constructor( type: MetadataType, registry = new RegistryAccess(), - forceIgnore: ForceIgnore = new ForceIgnore(), - tree: TreeContainer = new NodeFSTreeContainer() + forceIgnore = new ForceIgnore(), + tree = new NodeFSTreeContainer() ) { this.type = type; this.registry = registry; diff --git a/src/resolve/forceIgnore.ts b/src/resolve/forceIgnore.ts index 3917caf402..4c1e4525aa 100644 --- a/src/resolve/forceIgnore.ts +++ b/src/resolve/forceIgnore.ts @@ -17,7 +17,7 @@ export class ForceIgnore { private readonly parser?: Ignore; private readonly forceIgnoreDirectory?: string; - private DEFAULT_IGNORE: string[] = ['**/*.dup', '**/.*', '**/package2-descriptor.json', '**/package2-manifest.json']; + private DEFAULT_IGNORE = ['**/*.dup', '**/.*', '**/package2-descriptor.json', '**/package2-manifest.json']; public constructor(forceIgnorePath = '') { try { diff --git a/src/resolve/metadataResolver.ts b/src/resolve/metadataResolver.ts index d5d0b42749..93b4511035 100644 --- a/src/resolve/metadataResolver.ts +++ b/src/resolve/metadataResolver.ts @@ -31,6 +31,7 @@ export class MetadataResolver { /** * @param registry Custom registry data * @param tree `TreeContainer` to traverse with + * @param useFsForceIgnore false = use default forceignore entries, true = search and use forceignore in project */ public constructor( private registry = new RegistryAccess(), @@ -186,7 +187,7 @@ const isProbablyPackageManifest = * If a type can be determined from a directory path, and the end part of the path isn't * the directoryName of the type itself, infer the path is part of a mixedContent component * - * @param dirPath Path to a directory + * @param registry the registry to resolve a type against */ const resolveDirectoryAsComponent = (registry: RegistryAccess) => @@ -227,8 +228,8 @@ const isMetadata = * Attempt to find similar types for types that could not be inferred * To be used after executing the resolveType() method * - * @param fsPath * @returns an array of suggestions + * @param registry a metdata registry to resolve types against */ const getSuggestionsForUnresolvedTypes = (registry: RegistryAccess) => @@ -350,7 +351,7 @@ const resolveType = /** * Any file with a registered suffix is potentially a content metadata file. * - * @param fsPath File path of a potential content metadata file + * @param registry a metadata registry to resolve types agsinst */ const parseAsContentMetadataXml = (registry: RegistryAccess) => diff --git a/src/resolve/sourceComponent.ts b/src/resolve/sourceComponent.ts index b40afa0672..d7a1cdd1a1 100644 --- a/src/resolve/sourceComponent.ts +++ b/src/resolve/sourceComponent.ts @@ -45,8 +45,8 @@ export class SourceComponent implements MetadataComponent { public parentType?: MetadataType; public content?: string; public replacements?: Record; - private treeContainer: TreeContainer; - private forceIgnore: ForceIgnore; + private readonly treeContainer: TreeContainer; + private readonly forceIgnore: ForceIgnore; private markedForDelete = false; private destructiveChangesType?: DestructiveChangesType; diff --git a/test/collections/componentSetBuilder.test.ts b/test/collections/componentSetBuilder.test.ts index 8e70506d2f..dfb97aec47 100644 --- a/test/collections/componentSetBuilder.test.ts +++ b/test/collections/componentSetBuilder.test.ts @@ -6,6 +6,7 @@ */ import * as path from 'node:path'; +import { join } from 'node:path'; import fs from 'graceful-fs'; import * as sinon from 'sinon'; import { assert, expect, config } from 'chai'; @@ -14,6 +15,7 @@ import { RegistryAccess } from '../../src/registry/registryAccess'; import { ComponentSetBuilder, entryToTypeAndName } from '../../src/collections/componentSetBuilder'; import { ComponentSet } from '../../src/collections/componentSet'; import { FromSourceOptions } from '../../src/collections/types'; +import { MetadataResolver, SourceComponent } from '../../src'; config.truncateThreshold = 0; @@ -302,6 +304,50 @@ describe('ComponentSetBuilder', () => { expect(compSet.has(customObjectComponent)).to.equal(true); expect(compSet.has({ type: 'CustomObject', fullName: '*' })).to.equal(true); }); + it('should create ComponentSet from multiple metadata (ApexClass:MyClass,CustomObject), one of which is forceignored', async () => { + const customObjSourceComponent = new SourceComponent({ + name: 'myCO', + content: join('my', 'path', 'to', 'a', 'customobject.xml'), + parentType: undefined, + type: { id: 'customobject', directoryName: 'objects', name: 'CustomObject' }, + xml: '', + }); + + componentSet.add(apexClassComponent); + componentSet.add(customObjSourceComponent); + + componentSet.add(apexClassWildcardMatch); + componentSet.forceIgnoredPaths = new Set(); + componentSet.forceIgnoredPaths.add(join('my', 'path', 'to', 'a', 'customobject.xml')); + fromSourceStub.returns(componentSet); + const packageDir1 = path.resolve('force-app'); + + sandbox.stub(MetadataResolver.prototype, 'getComponentsFromPath').returns([customObjSourceComponent]); + + const compSet = await ComponentSetBuilder.build({ + sourcepath: undefined, + manifest: undefined, + metadata: { + metadataEntries: ['ApexClass:MyClass', 'ApexClass:MyClassIsAwesome', 'CustomObject:myCO'], + directoryPaths: [packageDir1], + }, + }); + expect(fromSourceStub.callCount).to.equal(1); + const fromSourceArgs = fromSourceStub.firstCall.args[0] as FromSourceOptions; + expect(fromSourceArgs).to.have.deep.property('fsPaths', [packageDir1]); + const filter = new ComponentSet(); + filter.add({ type: 'ApexClass', fullName: 'MyClass' }); + filter.add({ type: 'ApexClass', fullName: 'MyClassIsAwesome' }); + filter.add({ type: 'CustomObject', fullName: 'myCO' }); + assert(fromSourceArgs.include instanceof ComponentSet, 'include should be a ComponentSet'); + expect(fromSourceArgs.include.getSourceComponents()).to.deep.equal(filter.getSourceComponents()); + expect(compSet.size).to.equal(3); + expect(compSet.has(apexClassComponent)).to.equal(true); + expect(compSet.has(customObjectComponent)).to.equal(false); + expect(compSet.has({ type: 'CustomObject', fullName: '*' })).to.equal(false); + expect(compSet.has({ type: 'ApexClass', fullName: 'MyClass' })).to.equal(true); + expect(compSet.has({ type: 'ApexClass', fullName: 'MyClassIsAwesome' })).to.equal(true); + }); it('should create ComponentSet from partial-match fullName (ApexClass:Prop*)', async () => { componentSet.add(apexClassComponent); diff --git a/test/mock/type-constants/staticresourceConstant.ts b/test/mock/type-constants/staticresourceConstant.ts index 2b3aec1c3e..90d41adb2f 100644 --- a/test/mock/type-constants/staticresourceConstant.ts +++ b/test/mock/type-constants/staticresourceConstant.ts @@ -22,7 +22,7 @@ export const MIXED_CONTENT_DIRECTORY_SOURCE_PATHS = [ join(MIXED_CONTENT_DIRECTORY_CONTENT_PATH, 'tests', 'test.js'), join(MIXED_CONTENT_DIRECTORY_CONTENT_PATH, 'tests', 'test2.pdf'), ]; -export const MIXED_CONTENT_DIRECTORY_COMPONENT: SourceComponent = new SourceComponent({ +export const MIXED_CONTENT_DIRECTORY_COMPONENT = new SourceComponent({ name: 'aStaticResource', type, xml: MIXED_CONTENT_DIRECTORY_XML_PATHS[0], diff --git a/test/snapshot/sampleProjects/forceignore/.forceignore b/test/snapshot/sampleProjects/forceignore/.forceignore new file mode 100644 index 0000000000..34c2648d09 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/.forceignore @@ -0,0 +1 @@ +**/*.profile diff --git a/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls b/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls new file mode 100644 index 0000000000..c0bf528883 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls @@ -0,0 +1,5 @@ +public with sharing class OneClass { + public OneClass() { + + } +} \ No newline at end of file diff --git a/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls-meta.xml b/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls-meta.xml new file mode 100644 index 0000000000..019e850990 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls-meta.xml @@ -0,0 +1,5 @@ + + + 59.0 + Active + \ No newline at end of file diff --git a/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls b/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls new file mode 100644 index 0000000000..c0bf528883 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls @@ -0,0 +1,5 @@ +public with sharing class OneClass { + public OneClass() { + + } +} \ No newline at end of file diff --git a/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls-meta.xml b/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls-meta.xml new file mode 100644 index 0000000000..019e850990 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls-meta.xml @@ -0,0 +1,5 @@ + + + 59.0 + Active + \ No newline at end of file diff --git a/test/snapshot/sampleProjects/forceignore/originalMdapi/package.xml b/test/snapshot/sampleProjects/forceignore/originalMdapi/package.xml new file mode 100644 index 0000000000..3fec33e242 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/originalMdapi/package.xml @@ -0,0 +1,12 @@ + + + + OneClass + ApexClass + + + Admin + Profile + + 60.0 + diff --git a/test/snapshot/sampleProjects/forceignore/originalMdapi/profiles/Admin.profile b/test/snapshot/sampleProjects/forceignore/originalMdapi/profiles/Admin.profile new file mode 100644 index 0000000000..00093a0e82 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/originalMdapi/profiles/Admin.profile @@ -0,0 +1,9 @@ + + + false + Salesforce + + true + AIViewInsightObjects + + diff --git a/test/snapshot/sampleProjects/forceignore/sfdx-project.json b/test/snapshot/sampleProjects/forceignore/sfdx-project.json new file mode 100644 index 0000000000..5a0bd1521c --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/sfdx-project.json @@ -0,0 +1,12 @@ +{ + "name": "forceignore", + "namespace": "", + "packageDirectories": [ + { + "default": true, + "path": "force-app" + } + ], + "sfdcLoginUrl": "https://login.salesforce.com", + "sourceApiVersion": "52.0" +} diff --git a/test/snapshot/sampleProjects/forceignore/snapshots.test.ts b/test/snapshot/sampleProjects/forceignore/snapshots.test.ts new file mode 100644 index 0000000000..70f4875bb2 --- /dev/null +++ b/test/snapshot/sampleProjects/forceignore/snapshots.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { dirsAreIdentical, dirEntsToPaths, fileSnap } from '../../helper/conversions'; +import { MetadataConverter } from '../../../../src/convert/metadataConverter'; +import { ComponentSetBuilder } from '../../../../src/collections/componentSetBuilder'; + +// we don't want failing tests outputting over each other +/* eslint-disable no-await-in-loop */ + +describe('will respect forceignore when resolving from metadata ', () => { + const testDir = path.join('test', 'snapshot', 'sampleProjects', 'forceignore'); + + // The directory containing metadata in source format to be converted + const mdapiDir = path.join(testDir, 'originalMdapi'); + + // The directory of snapshots containing expected conversion results + const snapshotsDir = path.join(testDir, '__snapshots__'); + + // The directory where metadata is converted as part of testing + const testOutput = path.join(testDir, 'testOutput'); + + /** Return only the files involved in the conversion */ + const getConvertedFilePaths = async (outputDir: string): Promise => + dirEntsToPaths( + await fs.promises.readdir(outputDir, { + recursive: true, + withFileTypes: true, + }) + ); + + it('ignores source format path', async () => { + const cs = await ComponentSetBuilder.build({ + metadata: { + metadataEntries: ['ApexClass:OneClass', 'Profile:Admin'], + directoryPaths: [mdapiDir], + }, + projectDir: testDir, + }); + + const sourceOutput = path.join(testOutput, 'source-format'); + + await new MetadataConverter().convert(cs, 'source', { + type: 'directory', + outputDirectory: sourceOutput, + genUniqueDir: false, + }); + + const convertedFiles = await getConvertedFilePaths(sourceOutput); + for (const file of convertedFiles) { + await fileSnap(file, testDir); + } + dirsAreIdentical(path.join(snapshotsDir, 'testOutput', 'source-format'), sourceOutput); + }); + + after(async () => { + await fs.promises.rm(testOutput, { recursive: true, force: true }); + }); +});