From 0f068660803c59f6f06d80e0c40e9c3686ef8254 Mon Sep 17 00:00:00 2001 From: Fotis Papadogeorgopoulos Date: Tue, 28 Jan 2025 12:11:55 +0200 Subject: [PATCH] feat(managers/npm): support pnpm catalogs (#33376) Co-authored-by: Sebastian Poxhofer --- lib/config/index.spec.ts | 2 +- lib/modules/manager/npm/extract/index.spec.ts | 30 + lib/modules/manager/npm/extract/index.ts | 32 +- lib/modules/manager/npm/extract/pnpm.spec.ts | 163 +++++ lib/modules/manager/npm/extract/pnpm.ts | 134 +++- .../npm/extract/post/locked-versions.spec.ts | 60 ++ .../npm/extract/post/locked-versions.ts | 36 +- lib/modules/manager/npm/extract/types.ts | 12 + lib/modules/manager/npm/index.ts | 2 +- lib/modules/manager/npm/post-update/index.ts | 7 +- lib/modules/manager/npm/post-update/types.ts | 1 + lib/modules/manager/npm/readme.md | 1 + lib/modules/manager/npm/schema.ts | 11 + .../manager/npm/update/dependency/common.ts | 32 + .../manager/npm/update/dependency/index.ts | 27 +- .../npm/update/dependency/pnpm.spec.ts | 574 ++++++++++++++++++ .../manager/npm/update/dependency/pnpm.ts | 157 +++++ lib/util/yaml.ts | 39 +- .../extract-fingerprint-config.spec.ts | 12 +- 19 files changed, 1285 insertions(+), 47 deletions(-) create mode 100644 lib/modules/manager/npm/update/dependency/common.ts create mode 100644 lib/modules/manager/npm/update/dependency/pnpm.spec.ts create mode 100644 lib/modules/manager/npm/update/dependency/pnpm.ts diff --git a/lib/config/index.spec.ts b/lib/config/index.spec.ts index 13fd4ac4fc995b..9cc5c378ba919c 100644 --- a/lib/config/index.spec.ts +++ b/lib/config/index.spec.ts @@ -125,7 +125,7 @@ describe('config/index', () => { const parentConfig = { ...defaultConfig }; const config = getManagerConfig(parentConfig, 'npm'); expect(config).toContainEntries([ - ['fileMatch', ['(^|/)package\\.json$']], + ['fileMatch', ['(^|/)package\\.json$', '(^|/)pnpm-workspace\\.yaml$']], ]); expect(getManagerConfig(parentConfig, 'html')).toContainEntries([ ['fileMatch', ['\\.html?$']], diff --git a/lib/modules/manager/npm/extract/index.spec.ts b/lib/modules/manager/npm/extract/index.spec.ts index ec54ac88b183d7..a73936ce273b9a 100644 --- a/lib/modules/manager/npm/extract/index.spec.ts +++ b/lib/modules/manager/npm/extract/index.spec.ts @@ -1159,6 +1159,36 @@ describe('modules/manager/npm/extract/index', () => { }, ]); }); + + it('extracts pnpm workspace yaml files', async () => { + fs.readLocalFile.mockResolvedValueOnce(codeBlock` + packages: + - pkg-a + + catalog: + is-positive: 1.0.0 + `); + const res = await extractAllPackageFiles(defaultExtractConfig, [ + 'pnpm-workspace.yaml', + ]); + expect(res).toEqual([ + { + deps: [ + { + currentValue: '1.0.0', + datasource: 'npm', + depName: 'is-positive', + depType: 'pnpm.catalog.default', + prettyDepType: 'pnpm.catalog.default', + }, + ], + managerData: { + pnpmShrinkwrap: undefined, + }, + packageFile: 'pnpm-workspace.yaml', + }, + ]); + }); }); describe('.postExtract()', () => { diff --git a/lib/modules/manager/npm/extract/index.ts b/lib/modules/manager/npm/extract/index.ts index 0b9fbba90638e0..d96f38bac16643 100644 --- a/lib/modules/manager/npm/extract/index.ts +++ b/lib/modules/manager/npm/extract/index.ts @@ -17,6 +17,7 @@ import type { import type { NpmLockFiles, NpmManagerData } from '../types'; import { getExtractedConstraints } from './common/dependency'; import { extractPackageJson } from './common/package-file'; +import { extractPnpmWorkspaceFile, tryParsePnpmWorkspaceYaml } from './pnpm'; import { postExtract } from './post'; import type { NpmPackage } from './types'; import { isZeroInstall } from './yarn'; @@ -229,12 +230,33 @@ export async function extractAllPackageFiles( const content = await readLocalFile(packageFile, 'utf8'); // istanbul ignore else if (content) { - const deps = await extractPackageFile(content, packageFile, config); - if (deps) { - npmFiles.push({ - ...deps, + // pnpm workspace files are their own package file, defined via fileMatch. + // We duck-type the content here, to allow users to rename the file itself. + const parsedPnpmWorkspaceYaml = tryParsePnpmWorkspaceYaml(content); + if (parsedPnpmWorkspaceYaml.success) { + logger.trace( + { packageFile }, + `Extracting file as a pnpm workspace YAML file`, + ); + const deps = await extractPnpmWorkspaceFile( + parsedPnpmWorkspaceYaml.data, packageFile, - }); + ); + if (deps) { + npmFiles.push({ + ...deps, + packageFile, + }); + } + } else { + logger.trace({ packageFile }, `Extracting as a package.json file`); + const deps = await extractPackageFile(content, packageFile, config); + if (deps) { + npmFiles.push({ + ...deps, + packageFile, + }); + } } } else { logger.debug({ packageFile }, `No content found`); diff --git a/lib/modules/manager/npm/extract/pnpm.spec.ts b/lib/modules/manager/npm/extract/pnpm.spec.ts index 73ad5be0f51330..9677ac719decc5 100644 --- a/lib/modules/manager/npm/extract/pnpm.spec.ts +++ b/lib/modules/manager/npm/extract/pnpm.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { Fixtures } from '../../../../../test/fixtures'; import { fs, getFixturePath, logger, partial } from '../../../../../test/util'; import { GlobalConfig } from '../../../../config/global'; @@ -7,6 +8,7 @@ import type { NpmManagerData } from '../types'; import { detectPnpmWorkspaces, extractPnpmFilters, + extractPnpmWorkspaceFile, findPnpmWorkspace, getPnpmLock, } from './pnpm'; @@ -278,10 +280,171 @@ describe('modules/manager/npm/extract/pnpm', () => { expect(Object.keys(res.lockedVersionsWithPath!)).toHaveLength(1); }); + it('extracts version from catalogs', async () => { + const lockfileContent = codeBlock` + lockfileVersion: '9.0' + + settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + + catalogs: + default: + react: + specifier: ^18 + version: 18.3.1 + + importers: + + .: + dependencies: + react: + specifier: 'catalog:' + version: 18.3.1 + + packages: + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + + react@18.3.1: + resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} + engines: {node: '>=0.10.0'} + + snapshots: + + js-tokens@4.0.0: {} + + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + + react@18.3.1: + dependencies: + loose-envify: 1.4.0 + `; + fs.readLocalFile.mockResolvedValueOnce(lockfileContent); + const res = await getPnpmLock('package.json'); + expect(Object.keys(res.lockedVersionsWithCatalog!)).toHaveLength(1); + }); + it('returns empty if no deps', async () => { fs.readLocalFile.mockResolvedValueOnce('{}'); const res = await getPnpmLock('package.json'); expect(res.lockedVersionsWithPath).toBeUndefined(); }); }); + + describe('.extractPnpmWorkspaceFile()', () => { + it('handles empty catalog entries', async () => { + expect( + await extractPnpmWorkspaceFile( + { catalog: {}, catalogs: {} }, + 'pnpm-workspace.yaml', + ), + ).toMatchObject({ + deps: [], + }); + }); + + it('parses valid pnpm-workspace.yaml file', async () => { + expect( + await extractPnpmWorkspaceFile( + { + catalog: { + react: '18.3.0', + }, + catalogs: { + react17: { + react: '17.0.2', + }, + }, + }, + 'pnpm-workspace.yaml', + ), + ).toMatchObject({ + deps: [ + { + currentValue: '18.3.0', + datasource: 'npm', + depName: 'react', + depType: 'pnpm.catalog.default', + prettyDepType: 'pnpm.catalog.default', + }, + { + currentValue: '17.0.2', + datasource: 'npm', + depName: 'react', + depType: 'pnpm.catalog.react17', + prettyDepType: 'pnpm.catalog.react17', + }, + ], + }); + }); + + it('finds relevant lockfile', async () => { + const lockfileContent = codeBlock` + lockfileVersion: '9.0' + + catalogs: + default: + react: + specifier: 18.3.1 + version: 18.3.1 + + importers: + + .: + dependencies: + react: + specifier: 'catalog:' + version: 18.3.1 + + packages: + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + + react@18.3.1: + resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} + engines: {node: '>=0.10.0'} + + snapshots: + + js-tokens@4.0.0: {} + + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + + react@18.3.1: + dependencies: + loose-envify: 1.4.0 + `; + fs.readLocalFile.mockResolvedValueOnce(lockfileContent); + fs.getSiblingFileName.mockReturnValueOnce('pnpm-lock.yaml'); + expect( + await extractPnpmWorkspaceFile( + { + catalog: { + react: '18.3.1', + }, + }, + 'pnpm-workspace.yaml', + ), + ).toMatchObject({ + managerData: { + pnpmShrinkwrap: 'pnpm-lock.yaml', + }, + }); + }); + }); }); diff --git a/lib/modules/manager/npm/extract/pnpm.ts b/lib/modules/manager/npm/extract/pnpm.ts index 4871ed0fc5baa4..3bd596e1279e6c 100644 --- a/lib/modules/manager/npm/extract/pnpm.ts +++ b/lib/modules/manager/npm/extract/pnpm.ts @@ -1,6 +1,7 @@ import is from '@sindresorhus/is'; import { findPackages } from 'find-packages'; import upath from 'upath'; +import type { z } from 'zod'; import { GlobalConfig } from '../../../../config/global'; import { logger } from '../../../../logger'; import { @@ -10,10 +11,17 @@ import { readLocalFile, } from '../../../../util/fs'; import { parseSingleYaml } from '../../../../util/yaml'; -import type { PackageFile } from '../../types'; +import type { + PackageDependency, + PackageFile, + PackageFileContent, +} from '../../types'; import type { PnpmDependencySchema, PnpmLockFile } from '../post-update/types'; +import type { PnpmCatalogsSchema } from '../schema'; +import { PnpmWorkspaceFileSchema } from '../schema'; import type { NpmManagerData } from '../types'; -import type { LockFile, PnpmWorkspaceFile } from './types'; +import { extractDependency, parseDepName } from './common/dependency'; +import type { LockFile, PnpmCatalog, PnpmWorkspaceFile } from './types'; function isPnpmLockfile(obj: any): obj is PnpmLockFile { return is.plainObject(obj) && 'lockfileVersion' in obj; @@ -87,7 +95,7 @@ export async function detectPnpmWorkspaces( for (const p of packageFiles) { const { packageFile, managerData } = p; - const { pnpmShrinkwrap } = managerData as NpmManagerData; + const pnpmShrinkwrap = managerData?.pnpmShrinkwrap; // check if pnpmShrinkwrap-file has already been provided if (pnpmShrinkwrap) { @@ -160,9 +168,11 @@ export async function getPnpmLock(filePath: string): Promise { : parseFloat(lockParsed.lockfileVersion); const lockedVersions = getLockedVersions(lockParsed); + const lockedCatalogVersions = getLockedCatalogVersions(lockParsed); return { lockedVersionsWithPath: lockedVersions, + lockedVersionsWithCatalog: lockedCatalogVersions, lockfileVersion, }; } catch (err) { @@ -171,6 +181,26 @@ export async function getPnpmLock(filePath: string): Promise { } } +function getLockedCatalogVersions( + lockParsed: PnpmLockFile, +): Record> { + const lockedVersions: Record> = {}; + + if (is.nonEmptyObject(lockParsed.catalogs)) { + for (const [catalog, dependencies] of Object.entries(lockParsed.catalogs)) { + const versions: Record = {}; + + for (const [dep, versionCarrier] of Object.entries(dependencies)) { + versions[dep] = versionCarrier.version; + } + + lockedVersions[catalog] = versions; + } + } + + return lockedVersions; +} + function getLockedVersions( lockParsed: PnpmLockFile, ): Record>> { @@ -222,3 +252,101 @@ function getLockedDependencyVersions( return res; } + +export function tryParsePnpmWorkspaceYaml(content: string): + | { + success: true; + data: PnpmWorkspaceFile; + } + | { success: false; data?: never } { + try { + const data = parseSingleYaml(content, { + customSchema: PnpmWorkspaceFileSchema, + }); + return { success: true, data }; + } catch { + return { success: false }; + } +} + +type PnpmCatalogs = z.TypeOf; + +export async function extractPnpmWorkspaceFile( + catalogs: PnpmCatalogs, + packageFile: string, +): Promise | null> { + logger.trace(`pnpm.extractPnpmWorkspaceFile(${packageFile})`); + + const pnpmCatalogs = pnpmCatalogsToArray(catalogs); + + const deps = extractPnpmCatalogDeps(pnpmCatalogs); + + let pnpmShrinkwrap; + const filePath = getSiblingFileName(packageFile, 'pnpm-lock.yaml'); + + if (await readLocalFile(filePath, 'utf8')) { + pnpmShrinkwrap = filePath; + } + + return { + deps, + managerData: { + pnpmShrinkwrap, + }, + }; +} + +/** + * In order to facilitate matching on specific catalogs, we structure the + * depType as `pnpm.catalog.default`, `pnpm.catalog.react17`, and so on. + */ +function getCatalogDepType(name: string): string { + const CATALOG_DEPENDENCY = 'pnpm.catalog'; + return `${CATALOG_DEPENDENCY}.${name}`; +} + +function extractPnpmCatalogDeps( + catalogs: PnpmCatalog[], +): PackageDependency[] { + const deps: PackageDependency[] = []; + + for (const catalog of catalogs) { + for (const [key, val] of Object.entries(catalog.dependencies)) { + const depType = getCatalogDepType(catalog.name); + const depName = parseDepName(depType, key); + const dep: PackageDependency = { + depType, + depName, + ...extractDependency(depType, depName, val!), + prettyDepType: depType, + }; + deps.push(dep); + } + } + + return deps; +} + +function pnpmCatalogsToArray({ + catalog: defaultCatalogDeps, + catalogs: namedCatalogs, +}: PnpmCatalogs): PnpmCatalog[] { + const result: PnpmCatalog[] = []; + + if (defaultCatalogDeps !== undefined) { + result.push({ name: 'default', dependencies: defaultCatalogDeps }); + } + + if (!namedCatalogs) { + return result; + } + + for (const [name, dependencies] of Object.entries(namedCatalogs)) { + result.push({ + name, + dependencies, + }); + } + + return result; +} diff --git a/lib/modules/manager/npm/extract/post/locked-versions.spec.ts b/lib/modules/manager/npm/extract/post/locked-versions.spec.ts index e141766d63f973..fffffa6d567eda 100644 --- a/lib/modules/manager/npm/extract/post/locked-versions.spec.ts +++ b/lib/modules/manager/npm/extract/post/locked-versions.spec.ts @@ -586,6 +586,66 @@ describe('modules/manager/npm/extract/post/locked-versions', () => { ]); }); + it('uses pnpm-lock for pnpm.catalog depType', async () => { + pnpm.getPnpmLock.mockResolvedValue({ + lockedVersionsWithCatalog: { + default: { + a: '1.0.0', + }, + named: { + b: '2.0.0', + }, + }, + lockfileVersion: 9.0, + }); + const packageFiles = [ + { + managerData: { + pnpmShrinkwrap: 'pnpm-lock.yaml', + }, + extractedConstraints: { + pnpm: '9.15.3', + }, + deps: [ + { + depName: 'a', + depType: 'pnpm.catalog.default', + currentValue: '1.0.0', + }, + { + depName: 'b', + depType: 'pnpm.catalog.named', + currentValue: '2.0.0', + }, + ], + packageFile: 'pnpm-workspace.yaml', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { pnpm: '9.15.3' }, + deps: [ + { + currentValue: '1.0.0', + depName: 'a', + lockedVersion: '1.0.0', + depType: 'pnpm.catalog.default', + }, + { + currentValue: '2.0.0', + depName: 'b', + lockedVersion: '2.0.0', + depType: 'pnpm.catalog.named', + }, + ], + lockFiles: ['pnpm-lock.yaml'], + managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, + packageFile: 'pnpm-workspace.yaml', + }, + ]); + }); + it('uses pnpm-lock in subfolder', async () => { pnpm.getPnpmLock.mockResolvedValue({ lockedVersionsWithPath: { diff --git a/lib/modules/manager/npm/extract/post/locked-versions.ts b/lib/modules/manager/npm/extract/post/locked-versions.ts index 0970c3faeca509..c86a4e67424542 100644 --- a/lib/modules/manager/npm/extract/post/locked-versions.ts +++ b/lib/modules/manager/npm/extract/post/locked-versions.ts @@ -8,6 +8,9 @@ import { getNpmLock } from '../npm'; import { getPnpmLock } from '../pnpm'; import type { LockFile } from '../types'; import { getYarnLock, getYarnVersionFromLock } from '../yarn'; + +const pnpmCatalogDepTypeRe = /pnpm\.catalog\.(?.*)/; + export async function getLockedVersions( packageFiles: PackageFile[], ): Promise { @@ -121,14 +124,31 @@ export async function getLockedVersions( for (const dep of packageFile.deps) { const { depName, depType } = dep; - // TODO: types (#22198) - const lockedVersion = semver.valid( - lockFileCache[pnpmShrinkwrap].lockedVersionsWithPath?.[relativeDir]?.[ - depType! - ]?.[depName!], - ); - if (is.string(lockedVersion)) { - dep.lockedVersion = lockedVersion; + + const catalogName = pnpmCatalogDepTypeRe.exec(depType!)?.groups + ?.version; + + if (catalogName) { + const lockedVersion = semver.valid( + lockFileCache[pnpmShrinkwrap].lockedVersionsWithCatalog?.[ + catalogName + ]?.[depName!], + ); + + if (is.string(lockedVersion)) { + dep.lockedVersion = lockedVersion; + } + } else { + // TODO: types (#22198) + const lockedVersion = semver.valid( + lockFileCache[pnpmShrinkwrap].lockedVersionsWithPath?.[ + relativeDir + ]?.[depType!]?.[depName!], + ); + + if (is.string(lockedVersion)) { + dep.lockedVersion = lockedVersion; + } } } } diff --git a/lib/modules/manager/npm/extract/types.ts b/lib/modules/manager/npm/extract/types.ts index a9681aec758eb1..46f7fb220fc12a 100644 --- a/lib/modules/manager/npm/extract/types.ts +++ b/lib/modules/manager/npm/extract/types.ts @@ -30,12 +30,24 @@ export interface LockFile { string, Record> >; + lockedVersionsWithCatalog?: Record>; lockfileVersion?: number; // cache version for Yarn isYarn1?: boolean; } export interface PnpmWorkspaceFile { packages: string[]; + catalog?: Record; + catalogs?: Record>; +} + +/** + * A pnpm catalog is either the default catalog (catalog:, catalogs:default), or + * a named one (catalogs:) + */ +export interface PnpmCatalog { + name: string; + dependencies: NpmPackageDependency; } export type OverrideDependency = Record; diff --git a/lib/modules/manager/npm/index.ts b/lib/modules/manager/npm/index.ts index 9385b7606c20f6..1c120457c43243 100644 --- a/lib/modules/manager/npm/index.ts +++ b/lib/modules/manager/npm/index.ts @@ -20,7 +20,7 @@ export const url = 'https://docs.npmjs.com'; export const categories: Category[] = ['js']; export const defaultConfig = { - fileMatch: ['(^|/)package\\.json$'], + fileMatch: ['(^|/)package\\.json$', '(^|/)pnpm-workspace\\.yaml$'], digest: { prBodyDefinitions: { Change: diff --git a/lib/modules/manager/npm/post-update/index.ts b/lib/modules/manager/npm/post-update/index.ts index 5b3c139175b566..6aafc2ef08f474 100644 --- a/lib/modules/manager/npm/post-update/index.ts +++ b/lib/modules/manager/npm/post-update/index.ts @@ -242,7 +242,12 @@ export async function writeUpdatedPackageFiles( await writeLocalFile(packageFile.path, packageFile.contents!); continue; } - if (!packageFile.path.endsWith('package.json')) { + if ( + !( + packageFile.path.endsWith('package.json') || + packageFile.path.endsWith('pnpm-workspace.yaml') + ) + ) { continue; } logger.debug(`Writing ${packageFile.path}`); diff --git a/lib/modules/manager/npm/post-update/types.ts b/lib/modules/manager/npm/post-update/types.ts index 35efbdd2f790e4..2ffe4d7f41bf0f 100644 --- a/lib/modules/manager/npm/post-update/types.ts +++ b/lib/modules/manager/npm/post-update/types.ts @@ -35,6 +35,7 @@ export type PnpmDependencySchema = Record; export interface PnpmLockFile { lockfileVersion: number | string; + catalogs?: Record>; importers?: Record>; dependencies: PnpmDependencySchema; devDependencies: PnpmDependencySchema; diff --git a/lib/modules/manager/npm/readme.md b/lib/modules/manager/npm/readme.md index daf5ec9582bf7d..09c5110d315474 100644 --- a/lib/modules/manager/npm/readme.md +++ b/lib/modules/manager/npm/readme.md @@ -10,6 +10,7 @@ The following `depTypes` are currently supported by the npm manager : - `overrides` - `resolutions` - `pnpm.overrides` +- `pnpm.catalog.`, such as `pnpm.catalog.default` and `pnpm.catalog.myCatalog`. [Matches any default and named pnpm catalogs](https://pnpm.io/catalogs#defining-catalogs). ### Yarn diff --git a/lib/modules/manager/npm/schema.ts b/lib/modules/manager/npm/schema.ts index 79d986fea78507..7efbf9a0f919fb 100644 --- a/lib/modules/manager/npm/schema.ts +++ b/lib/modules/manager/npm/schema.ts @@ -1,6 +1,17 @@ import { z } from 'zod'; import { Json, LooseRecord } from '../../../util/schema-utils'; +export const PnpmCatalogsSchema = z.object({ + catalog: z.optional(z.record(z.string())), + catalogs: z.optional(z.record(z.record(z.string()))), +}); + +export const PnpmWorkspaceFileSchema = z + .object({ + packages: z.array(z.string()), + }) + .and(PnpmCatalogsSchema); + export const PackageManagerSchema = z .string() .transform((val) => val.split('@')) diff --git a/lib/modules/manager/npm/update/dependency/common.ts b/lib/modules/manager/npm/update/dependency/common.ts new file mode 100644 index 00000000000000..7a84d4e21f5783 --- /dev/null +++ b/lib/modules/manager/npm/update/dependency/common.ts @@ -0,0 +1,32 @@ +import { logger } from '../../../../../logger'; +import type { Upgrade } from '../../../types'; + +export function getNewGitValue(upgrade: Upgrade): string | null { + if (!upgrade.currentRawValue) { + return null; + } + if (upgrade.currentDigest) { + logger.debug('Updating git digest'); + return upgrade.currentRawValue.replace( + upgrade.currentDigest, + // TODO #22198 + upgrade.newDigest!.substring(0, upgrade.currentDigest.length), + ); + } else { + logger.debug('Updating git version tag'); + return upgrade.currentRawValue.replace( + upgrade.currentValue, + upgrade.newValue, + ); + } +} + +export function getNewNpmAliasValue( + value: string | undefined, + upgrade: Upgrade, +): string | null { + if (!upgrade.npmPackageAlias) { + return null; + } + return `npm:${upgrade.packageName}@${value}`; +} diff --git a/lib/modules/manager/npm/update/dependency/index.ts b/lib/modules/manager/npm/update/dependency/index.ts index b6e1ca0735a0b3..0d4bb218f0205e 100644 --- a/lib/modules/manager/npm/update/dependency/index.ts +++ b/lib/modules/manager/npm/update/dependency/index.ts @@ -11,6 +11,8 @@ import type { RecursiveOverride, } from '../../extract/types'; import type { NpmDepType, NpmManagerData } from '../../types'; +import { getNewGitValue, getNewNpmAliasValue } from './common'; +import { updatePnpmCatalogDependency } from './pnpm'; function renameObjKey( oldObj: DependenciesMeta, @@ -115,29 +117,16 @@ export function updateDependency({ fileContent, upgrade, }: UpdateDependencyConfig): string | null { + if (upgrade.depType?.startsWith('pnpm.catalog')) { + return updatePnpmCatalogDependency({ fileContent, upgrade }); + } + const { depType, managerData } = upgrade; const depName: string = managerData?.key || upgrade.depName; let { newValue } = upgrade; - if (upgrade.currentRawValue) { - if (upgrade.currentDigest) { - logger.debug('Updating package.json git digest'); - newValue = upgrade.currentRawValue.replace( - upgrade.currentDigest, - // TODO #22198 - upgrade.newDigest!.substring(0, upgrade.currentDigest.length), - ); - } else { - logger.debug('Updating package.json git version tag'); - newValue = upgrade.currentRawValue.replace( - upgrade.currentValue, - upgrade.newValue, - ); - } - } - if (upgrade.npmPackageAlias) { - newValue = `npm:${upgrade.packageName}@${newValue}`; - } + newValue = getNewGitValue(upgrade) ?? newValue; + newValue = getNewNpmAliasValue(newValue, upgrade) ?? newValue; logger.debug(`npm.updateDependency(): ${depType}.${depName} = ${newValue}`); try { diff --git a/lib/modules/manager/npm/update/dependency/pnpm.spec.ts b/lib/modules/manager/npm/update/dependency/pnpm.spec.ts new file mode 100644 index 00000000000000..211371239b004d --- /dev/null +++ b/lib/modules/manager/npm/update/dependency/pnpm.spec.ts @@ -0,0 +1,574 @@ +import { codeBlock } from 'common-tags'; +import * as npmUpdater from '../..'; + +describe('modules/manager/npm/update/dependency/pnpm', () => { + it('handles implicit default catalog dependency', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: 19.0.0 + `); + }); + + it('handles explicit default catalog dependency', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalogs: + default: + react: 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalogs: + default: + react: 19.0.0 + `); + }); + + it('handles explicit named catalog dependency', () => { + const upgrade = { + depType: 'pnpm.catalog.react17', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 + + catalogs: + react17: + react: 17.0.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 + + catalogs: + react17: + react: 19.0.0 + + `); + }); + + it('does nothing if the new and old values match', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 19.0.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(pnpmWorkspaceYaml); + }); + + it('replaces package', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'config', + newName: 'abc', + newValue: '2.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + config: 1.21.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + abc: 2.0.0 + `); + }); + + it('replaces a github dependency value', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'gulp', + currentValue: 'v4.0.0-alpha.2', + currentRawValue: 'gulpjs/gulp#v4.0.0-alpha.2', + newValue: 'v4.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + gulp: gulpjs/gulp#v4.0.0-alpha.2 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + gulp: gulpjs/gulp#v4.0.0 + `); + }); + + it('replaces a npm package alias', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'hapi', + npmPackageAlias: true, + packageName: '@hapi/hapi', + currentValue: '18.3.0', + newValue: '18.3.1', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + hapi: npm:@hapi/hapi@18.3.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + hapi: npm:@hapi/hapi@18.3.1 + `); + }); + + it('replaces a github short hash', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'gulp', + currentDigest: 'abcdef7', + currentRawValue: 'gulpjs/gulp#abcdef7', + newDigest: '0000000000111111111122222222223333333333', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + gulp: gulpjs/gulp#abcdef7 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + gulp: gulpjs/gulp#0000000 + `); + }); + + it('replaces a github fully specified version', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'n', + currentValue: 'v1.0.0', + currentRawValue: 'git+https://github.com/owner/n#v1.0.0', + newValue: 'v1.1.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + n: git+https://github.com/owner/n#v1.0.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + n: git+https://github.com/owner/n#v1.1.0 + `); + }); + + it('returns null if the dependency is not present in the target catalog', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react-not', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toBeNull(); + }); + + it('returns null if catalogs are missing', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toBeNull(); + }); + + it('returns null if empty file', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const testContent = npmUpdater.updateDependency({ + fileContent: null as never, + upgrade, + }); + expect(testContent).toBeNull(); + }); + + it('preserves literal whitespace', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: 19.0.0 + `); + }); + + it('preserves single quote style', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: '18.3.1' + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: '19.0.0' + `); + }); + + it('preserves comments', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: 18.3.1 # This is a comment + # This is another comment + react-dom: 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: 19.0.0 # This is a comment + # This is another comment + react-dom: 18.3.1 + `); + }); + + it('preserves double quote style', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: "18.3.1" + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: "19.0.0" + `); + }); + + it('preserves anchors, replacing only the value', () => { + // At the time of writing, this pattern is the recommended way to sync + // dependencies in catalogs. + // @see https://github.com/pnpm/pnpm/issues/8245#issuecomment-2371335323 + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: &react 18.3.1 + react-dom: *react + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: &react 19.0.0 + react-dom: *react + `); + }); + + it('preserves whitespace with anchors', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: &react 18.3.1 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: &react 19.0.0 + `); + }); + + it('preserves quotation style with anchors', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: + react: &react "18.3.1" + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: + react: &react "19.0.0" + `); + }); + + it('preserves formatting in flow style syntax', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + const pnpmWorkspaceYaml = codeBlock` + packages: + - pkg-a + + catalog: { + # This is a comment + "react": "18.3.1" + } + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toEqual(codeBlock` + packages: + - pkg-a + + catalog: { + # This is a comment + "react": "19.0.0" + } + `); + }); + + it('does not replace aliases in the value position', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newValue: '19.0.0', + }; + // In the general case, we do not know whether we should replace the anchor + // that an alias is resolved from. We leave this up to the user, e.g. via a + // Regex custom manager. + const pnpmWorkspaceYaml = codeBlock` + __deps: + react: &react 18.3.1 + + packages: + - pkg-a + + catalog: + react: *react + react-dom: *react + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toBeNull(); + }); + + it('does not replace aliases in the key position', () => { + const upgrade = { + depType: 'pnpm.catalog.default', + depName: 'react', + newName: 'react-x', + }; + const pnpmWorkspaceYaml = codeBlock` + __vars: + &r react: "" + + packages: + - pkg-a + + catalog: + *r: 18.0.0 + `; + const testContent = npmUpdater.updateDependency({ + fileContent: pnpmWorkspaceYaml, + upgrade, + }); + expect(testContent).toBeNull(); + }); +}); diff --git a/lib/modules/manager/npm/update/dependency/pnpm.ts b/lib/modules/manager/npm/update/dependency/pnpm.ts new file mode 100644 index 00000000000000..56ecbb03a45df9 --- /dev/null +++ b/lib/modules/manager/npm/update/dependency/pnpm.ts @@ -0,0 +1,157 @@ +import is from '@sindresorhus/is'; +import type { Document } from 'yaml'; +import { CST, isCollection, isPair, isScalar, parseDocument } from 'yaml'; +import { logger } from '../../../../../logger'; +import type { UpdateDependencyConfig } from '../../../types'; +import { PnpmCatalogsSchema } from '../../schema'; +import { getNewGitValue, getNewNpmAliasValue } from './common'; + +export function updatePnpmCatalogDependency({ + fileContent, + upgrade, +}: UpdateDependencyConfig): string | null { + const { depType, managerData, depName } = upgrade; + + const catalogName = depType?.split('.').at(-1); + + // istanbul ignore if + if (!is.string(catalogName)) { + logger.error( + 'No catalogName was found; this is likely an extraction error.', + ); + return null; + } + + let { newValue } = upgrade; + + newValue = getNewGitValue(upgrade) ?? newValue; + newValue = getNewNpmAliasValue(newValue, upgrade) ?? newValue; + + logger.trace( + `npm.updatePnpmCatalogDependency(): ${depType}:${managerData?.catalogName}.${depName} = ${newValue}`, + ); + + let document; + let parsedContents; + + try { + // In order to preserve the original formatting as much as possible, we want + // manipulate the CST directly. Using the AST (the result of parseDocument) + // does not guarantee that formatting would be the same after + // stringification. However, the CST is more annoying to query for certain + // values. Thus, we use both an annotated AST and a JS representation; the + // former for manipulation, and the latter for querying/validation. + document = parseDocument(fileContent, { keepSourceTokens: true }); + parsedContents = PnpmCatalogsSchema.parse(document.toJS()); + } catch (err) { + logger.debug({ err }, 'Could not parse pnpm-workspace YAML file.'); + return null; + } + + // In pnpm-workspace.yaml, the default catalog can be either `catalog` or + // `catalog.default`, but not both (pnpm throws outright with a config error). + // Thus, we must check which entry is being used, to reference it from / set + // it in the right place. + const usesImplicitDefaultCatalog = parsedContents.catalog !== undefined; + + const oldVersion = + catalogName === 'default' && usesImplicitDefaultCatalog + ? parsedContents.catalog?.[depName!] + : parsedContents.catalogs?.[catalogName]?.[depName!]; + + if (oldVersion === newValue) { + logger.trace('Version is already updated'); + return fileContent; + } + + // Update the value + const path = getDepPath({ + depName: depName!, + catalogName, + usesImplicitDefaultCatalog, + }); + + const modifiedDocument = changeDependencyIn(document, path, { + newValue, + newName: upgrade.newName, + }); + + if (!modifiedDocument) { + // Case where we are explicitly unable to substitute the key/value, for + // example if the value was an alias. + return null; + } + + // istanbul ignore if: this should not happen in practice, but we must satisfy th etypes + if (!modifiedDocument.contents?.srcToken) { + return null; + } + + return CST.stringify(modifiedDocument.contents.srcToken); +} + +/** + * Change the scalar name and/or value of a collection item in a YAML document, + * while keeping formatting consistent. Mutates the given document. + */ +function changeDependencyIn( + document: Document, + path: string[], + { newName, newValue }: { newName?: string; newValue?: string }, +): Document | null { + const parentPath = path.slice(0, -1); + const relevantItemKey = path.at(-1); + + const parentNode = document.getIn(parentPath); + + if (!parentNode || !isCollection(parentNode)) { + return null; + } + + const relevantNode = parentNode.items.find( + (item) => + isPair(item) && isScalar(item.key) && item.key.value === relevantItemKey, + ); + + if (!relevantNode || !isPair(relevantNode)) { + return null; + } + + if (newName) { + // istanbul ignore if: the try..catch block above already throws if a key is an alias + if (!CST.isScalar(relevantNode.srcToken?.key)) { + return null; + } + CST.setScalarValue(relevantNode.srcToken.key, newName); + } + + if (newValue) { + // We only support scalar values when substituting. This explicitly avoids + // substituting aliases, since those can be resolved from a shared location, + // and replacing either the referrent anchor or the alias would be wrong in + // the general case. We leave this up to the user, e.g. via a Regex custom + // manager. + if (!CST.isScalar(relevantNode.srcToken?.value)) { + return null; + } + CST.setScalarValue(relevantNode.srcToken.value, newValue); + } + + return document; +} + +function getDepPath({ + catalogName, + depName, + usesImplicitDefaultCatalog, +}: { + usesImplicitDefaultCatalog: boolean; + catalogName: string; + depName: string; +}): string[] { + if (catalogName === 'default' && usesImplicitDefaultCatalog) { + return ['catalog', depName]; + } else { + return ['catalogs', catalogName, depName]; + } +} diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index e8d5ef50ae2eae..c0de0ede60feda 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -1,5 +1,6 @@ import type { CreateNodeOptions, + Document, DocumentOptions, ParseOptions, SchemaOptions, @@ -20,6 +21,13 @@ interface YamlOptions< removeTemplates?: boolean; } +interface YamlParseDocumentOptions + extends ParseOptions, + DocumentOptions, + SchemaOptions { + removeTemplates?: boolean; +} + interface YamlOptionsMultiple< ResT = unknown, Schema extends ZodType = ZodType, @@ -117,6 +125,29 @@ export function parseSingleYaml( content: string, options?: YamlOptions, ): ResT { + const rawDocument = parseSingleYamlDocument(content, options); + + const document = rawDocument.toJS({ maxAliasCount: 10000 }); + const schema = options?.customSchema; + if (!schema) { + return document as ResT; + } + + return schema.parse(document); +} + +/** + * Parse a YAML string into a Document representation. + * + * Only a single document is supported. + * + * @param content + * @param options + */ +export function parseSingleYamlDocument( + content: string, + options?: YamlParseDocumentOptions, +): Document { const massagedContent = massageContent(content, options); const rawDocument = parseDocument( massagedContent, @@ -127,13 +158,7 @@ export function parseSingleYaml( throw new AggregateError(rawDocument.errors, 'Failed to parse YAML file'); } - const document = rawDocument.toJS({ maxAliasCount: 10000 }); - const schema = options?.customSchema; - if (!schema) { - return document as ResT; - } - - return schema.parse(document); + return rawDocument; } export function dump(obj: any, opts?: DumpOptions): string { diff --git a/lib/workers/repository/extract/extract-fingerprint-config.spec.ts b/lib/workers/repository/extract/extract-fingerprint-config.spec.ts index 3f8fe77f905966..eca0d3a980771c 100644 --- a/lib/workers/repository/extract/extract-fingerprint-config.spec.ts +++ b/lib/workers/repository/extract/extract-fingerprint-config.spec.ts @@ -39,7 +39,11 @@ describe('workers/repository/extract/extract-fingerprint-config', () => { ).toEqual({ enabled: true, fileList: [], - fileMatch: ['(^|/)package\\.json$', 'hero.json'], + fileMatch: [ + '(^|/)package\\.json$', + '(^|/)pnpm-workspace\\.yaml$', + 'hero.json', + ], ignorePaths: ['ignore-path-2'], includePaths: ['include-path-2'], manager: 'npm', @@ -85,7 +89,11 @@ describe('workers/repository/extract/extract-fingerprint-config', () => { ).toEqual({ enabled: true, fileList: [], - fileMatch: ['(^|/)package\\.json$', 'hero.json'], + fileMatch: [ + '(^|/)package\\.json$', + '(^|/)pnpm-workspace\\.yaml$', + 'hero.json', + ], ignorePaths: ['**/node_modules/**', '**/bower_components/**'], includePaths: [], manager: 'npm',