From 97cb5ec312e151527ba2aab77ed0307917e1d845 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 27 Jul 2021 15:49:14 -0700 Subject: [PATCH] @npmcli/arborist@2.8.0 * Refactor ideal tree building to handle more complicated peerDependencies use cases. * Do not modify ideal tree while checking if a peerSet can be placed. Fix: #3377 --- .../arborist/lib/arborist/build-ideal-tree.js | 792 +++--------------- .../@npmcli/arborist/lib/arborist/reify.js | 146 +++- .../@npmcli/arborist/lib/can-place-dep.js | 405 +++++++++ .../arborist/lib/deepest-nesting-target.js | 16 + node_modules/@npmcli/arborist/lib/edge.js | 2 + node_modules/@npmcli/arborist/lib/node.js | 32 +- .../@npmcli/arborist/lib/peer-entry-sets.js | 72 ++ node_modules/@npmcli/arborist/lib/peer-set.js | 25 - .../@npmcli/arborist/lib/place-dep.js | 536 ++++++++++++ .../@npmcli/arborist/lib/printable.js | 10 + .../@npmcli/arborist/lib/shrinkwrap.js | 4 +- node_modules/@npmcli/arborist/package.json | 8 +- package-lock.json | 22 +- package.json | 2 +- 14 files changed, 1306 insertions(+), 766 deletions(-) create mode 100644 node_modules/@npmcli/arborist/lib/can-place-dep.js create mode 100644 node_modules/@npmcli/arborist/lib/deepest-nesting-target.js create mode 100644 node_modules/@npmcli/arborist/lib/peer-entry-sets.js delete mode 100644 node_modules/@npmcli/arborist/lib/peer-set.js create mode 100644 node_modules/@npmcli/arborist/lib/place-dep.js diff --git a/node_modules/@npmcli/arborist/lib/arborist/build-ideal-tree.js b/node_modules/@npmcli/arborist/lib/arborist/build-ideal-tree.js index fdb947dc5905c..7ef42289d297b 100644 --- a/node_modules/@npmcli/arborist/lib/arborist/build-ideal-tree.js +++ b/node_modules/@npmcli/arborist/lib/arborist/build-ideal-tree.js @@ -3,14 +3,20 @@ const rpj = require('read-package-json-fast') const npa = require('npm-package-arg') const pacote = require('pacote') const cacache = require('cacache') -const semver = require('semver') const promiseCallLimit = require('promise-call-limit') -const getPeerSet = require('../peer-set.js') const realpath = require('../../lib/realpath.js') const { resolve, dirname } = require('path') const { promisify } = require('util') const treeCheck = require('../tree-check.js') const readdir = promisify(require('readdir-scoped-modules')) +const { depth } = require('treeverse') + +const { + OK, + REPLACE, + CONFLICT, +} = require('../can-place-dep.js') +const PlaceDep = require('../place-dep.js') const debug = require('../debug.js') const fromPath = require('../from-path.js') @@ -19,20 +25,9 @@ const Shrinkwrap = require('../shrinkwrap.js') const Node = require('../node.js') const Link = require('../link.js') const addRmPkgDeps = require('../add-rm-pkg-deps.js') -const gatherDepSet = require('../gather-dep-set.js') const optionalSet = require('../optional-set.js') const {checkEngine, checkPlatform} = require('npm-install-checks') -// enum of return values for canPlaceDep. -// No, this is a conflict, you may not put that package here -const CONFLICT = Symbol('CONFLICT') -// Yes, this is fine, and should not be a problem -const OK = Symbol('OK') -// No need, because the package already here is fine -const KEEP = Symbol('KEEP') -// Yes, clobber the package that is already here -const REPLACE = Symbol('REPLACE') - const relpath = require('../relpath.js') // note: some of these symbols are shared so we can hit @@ -47,7 +42,6 @@ const _flagsSuspect = Symbol.for('flagsSuspect') const _workspaces = Symbol.for('workspaces') const _prune = Symbol('prune') const _preferDedupe = Symbol('preferDedupe') -const _pruneDedupable = Symbol('pruneDedupable') const _legacyBundling = Symbol('legacyBundling') const _parseSettings = Symbol('parseSettings') const _initTree = Symbol('initTree') @@ -65,10 +59,6 @@ const _loadWorkspaces = Symbol.for('loadWorkspaces') const _linkFromSpec = Symbol('linkFromSpec') const _loadPeerSet = Symbol('loadPeerSet') const _updateNames = Symbol.for('updateNames') -const _placeDep = Symbol.for('placeDep') -const _canPlaceDep = Symbol.for('canPlaceDep') -const _canPlacePeers = Symbol('canPlacePeers') -const _pruneForReplacement = Symbol('pruneForReplacement') const _fixDepFlags = Symbol('fixDepFlags') const _resolveLinks = Symbol('resolveLinks') const _rootNodeFromPackage = Symbol('rootNodeFromPackage') @@ -100,12 +90,8 @@ const _checkPlatform = Symbol('checkPlatform') const _virtualRoots = Symbol('virtualRoots') const _virtualRoot = Symbol('virtualRoot') -// used for the ERESOLVE error to show the last peer conflict encountered -const _peerConflict = Symbol('peerConflict') - const _failPeerConflict = Symbol('failPeerConflict') const _explainPeerConflict = Symbol('explainPeerConflict') -const _warnPeerConflict = Symbol('warnPeerConflict') const _edgesOverridden = Symbol('edgesOverridden') // exposed symbol for unit testing the placeDep method directly const _peerSetSource = Symbol.for('peerSetSource') @@ -163,7 +149,6 @@ module.exports = cls => class IdealTreeBuilder extends cls { this[_loadFailures] = new Set() this[_linkNodes] = new Set() this[_manifests] = new Map() - this[_peerConflict] = null this[_edgesOverridden] = new Set() this[_resolvedAdd] = [] @@ -227,17 +212,13 @@ module.exports = cls => class IdealTreeBuilder extends cls { return treeCheck(this.idealTree) } - [_checkEngineAndPlatform] () { - // engine/platform checks throw, so start the promise chain off first - return Promise.resolve() - .then(() => { - for (const node of this.idealTree.inventory.values()) { - if (!node.optional) { - this[_checkEngine](node) - this[_checkPlatform](node) - } - } - }) + async [_checkEngineAndPlatform] () { + for (const node of this.idealTree.inventory.values()) { + if (!node.optional) { + this[_checkEngine](node) + this[_checkPlatform](node) + } + } } [_checkPlatform] (node) { @@ -850,7 +831,7 @@ This is a one-time fix-up, please be patient... const tasks = [] const peerSource = this[_peerSetSource].get(node) || node for (const edge of this[_problemEdges](node)) { - if (this[_edgesOverridden].has(edge)) + if (edge.overridden) continue // peerSetSource is only relevant when we have a peerEntryEdge @@ -894,34 +875,101 @@ This is a one-time fix-up, please be patient... tasks.push({edge, dep}) } - const placed = tasks + const placeDeps = tasks .sort((a, b) => a.edge.name.localeCompare(b.edge.name, 'en')) - .map(({ edge, dep }) => this[_placeDep](dep, node, edge)) + .map(({ edge, dep }) => new PlaceDep({ + edge, + dep, + + explicitRequest: this[_explicitRequests].has(edge), + updateNames: this[_updateNames], + auditReport: this.auditReport, + force: this[_force], + preferDedupe: this[_preferDedupe], + legacyBundling: this[_legacyBundling], + strictPeerDeps: this[_strictPeerDeps], + legacyPeerDeps: this.legacyPeerDeps, + globalStyle: this[_globalStyle], + })) const promises = [] - for (const set of placed) { - for (const node of set) { - this[_mutateTree] = true - this.addTracker('idealTree', node.name, node.location) - this[_depsQueue].push(node) - - // we're certainly going to need these soon, fetch them asap - // if it fails at this point, though, dont' worry because it - // may well be an optional dep that has gone missing. it'll - // fail later anyway. - const from = fromPath(node) - promises.push(...this[_problemEdges](node).map(e => - this[_fetchManifest](npa.resolve(e.name, e.spec, from)) - .catch(er => null))) - } + for (const pd of placeDeps) { + // placing a dep is actually a tree of placing the dep itself + // and all of its peer group that aren't already met by the tree + depth({ + tree: pd, + getChildren: pd => pd.children, + visit: pd => { + const { placed, edge, canPlace: cpd } = pd + // if we didn't place anything, nothing to do here + if (!placed) + return + + // we placed something, that means we changed the tree + if (placed.errors.length) + this[_loadFailures].add(placed) + this[_mutateTree] = true + if (cpd.canPlaceSelf === OK) { + for (const edgeIn of placed.edgesIn) { + if (edgeIn === edge) + continue + const { from, valid, overridden } = edgeIn + if (!overridden && !valid && !this[_depsSeen].has(from)) { + this.addTracker('idealTree', from.name, from.location) + this[_depsQueue].push(edgeIn.from) + } + } + } else { + /* istanbul ignore else - should be only OK or REPLACE here */ + if (cpd.canPlaceSelf === REPLACE) { + // this may also create some invalid edges, for example if we're + // intentionally causing something to get nested which was + // previously placed in this location. + for (const edgeIn of placed.edgesIn) { + if (edgeIn === edge) + continue + + const { valid, overridden } = edgeIn + if (!valid && !overridden) { + // if it's already been visited, we have to re-visit + // otherwise, just enqueue normally. + this[_depsSeen].delete(edgeIn.from) + this[_depsQueue].push(edgeIn.from) + } + } + } + } + + /* istanbul ignore if - should be impossible */ + if (cpd.canPlaceSelf === CONFLICT) { + debug(() => { + const er = new Error('placed with canPlaceSelf=CONFLICT') + throw Object.assign(er, { placeDep: pd }) + }) + return + } + + // lastly, also check for the missing deps of the node we placed + this[_depsQueue].push(placed) + + // pre-fetch any problem edges, since we'll need these soon + // if it fails at this point, though, dont' worry because it + // may well be an optional dep that has gone missing. it'll + // fail later anyway. + const from = fromPath(placed) + promises.push(...this[_problemEdges](placed).map(e => + this[_fetchManifest](npa.resolve(e.name, e.spec, from)) + .catch(er => null))) + }, + }) } - await Promise.all(promises) for (const { to } of node.edgesOut.values()) { if (to && to.isLink && to.target) this[_linkNodes].add(to) } + await Promise.all(promises) return this[_buildDepStep]() } @@ -1176,8 +1224,10 @@ This is a one-time fix-up, please be patient... // allow it. either we're overriding, or it's not something // that will be installed by default anyway, and we'll fail when // we get to the point where we need to, if we need to. - if (conflictOK || !required.has(dep)) + if (conflictOK || !required.has(dep)) { + edge.overridden = true continue + } // problem this[_failPeerConflict](edge, parentEdge) @@ -1219,9 +1269,7 @@ This is a one-time fix-up, please be patient... [_explainPeerConflict] (edge, currentEdge) { const node = edge.from const curNode = node.resolve(edge.name) - const pc = this[_peerConflict] || { peer: null, current: null } - const current = curNode ? curNode.explain() : pc.current - const peerConflict = pc.peer + const current = curNode.explain() return { code: 'ERESOLVE', current, @@ -1230,640 +1278,11 @@ This is a one-time fix-up, please be patient... // the tree handling logic. currentEdge: currentEdge ? currentEdge.explain() : null, edge: edge.explain(), - peerConflict, strictPeerDeps: this[_strictPeerDeps], force: this[_force], } } - [_warnPeerConflict] (edge) { - // track that we've overridden this edge, so that we don't keep trying - // to re-resolve it in an infinite loop. - this[_edgesOverridden].add(edge) - const expl = this[_explainPeerConflict](edge) - this.log.warn('ERESOLVE', 'overriding peer dependency', expl) - } - - // starting from either node, or in the case of non-root peer deps, - // the node's parent, walk up the tree until we find the first spot - // where this dep cannot be placed, and use the one right before that. - // place dep, requested by node, to satisfy edge - // XXX split this out into a separate method or mixin? It's quite a lot - // of functionality that ought to have its own unit tests more conveniently. - [_placeDep] (dep, node, edge, peerEntryEdge = null, peerPath = []) { - if (edge.to && - !edge.error && - !this[_explicitRequests].has(edge) && - !this[_updateNames].includes(edge.name) && - !this[_isVulnerable](edge.to)) - return [] - - // top nodes should still get peer deps from their fsParent if possible, - // and only install locally if there's no other option, eg for a link - // outside of the project root, or for a conflicted dep. - const start = edge.peer && !node.isProjectRoot ? node.resolveParent || node - : node - - let target - let canPlace = null - let isSource = false - const source = this[_peerSetSource].get(dep) - for (let check = start; check; check = check.resolveParent) { - // we always give the FIRST place we possibly *can* put this a little - // extra prioritization with peer dep overrides and deduping - if (check === source) - isSource = true - - // if the current location has a peerDep on it, then we can't place here - // this is pretty rare to hit, since we always prefer deduping peers. - const checkEdge = check.edgesOut.get(edge.name) - if (!check.isTop && checkEdge && checkEdge.peer) - continue - - const cp = this[_canPlaceDep](dep, check, edge, peerEntryEdge, peerPath, isSource) - isSource = false - - // anything other than a conflict is fine to proceed with - if (cp !== CONFLICT) { - canPlace = cp - target = check - } else - break - - // nest packages like npm v1 and v2 - // very disk-inefficient - if (this[_legacyBundling]) - break - - // when installing globally, or just in global style, we never place - // deps above the first level. - const tree = this.idealTree && this.idealTree.target - if (this[_globalStyle] && check.resolveParent === tree) - break - } - - // if we can't find a target, that means that the last placed checked - // (and all the places before it) had a copy already. if we're in - // --force mode, then the user has explicitly said that they're ok - // with conflicts. This can only occur in --force mode in the case - // when a node was added to the tree with a peerOptional dep that we - // ignored, and then later, that edge became invalid, and we fail to - // resolve it. We will warn about it in a moment. - if (!target) { - if (this[_force]) { - // we know that there is a dep (not the root) which is the target - // of this edge, or else it wouldn't have been a conflict. - target = edge.to.resolveParent - canPlace = KEEP - } else - this[_failPeerConflict](edge) - } else { - // it worked, so we clearly have no peer conflicts at this point. - this[_peerConflict] = null - } - - this.log.silly( - 'placeDep', - target.location || 'ROOT', - `${dep.name}@${dep.version}`, - canPlace.description || /* istanbul ignore next */ canPlace, - `for: ${node.package._id || node.location}`, - `want: ${edge.spec || '*'}` - ) - - // Can only get KEEP here if the original edge was valid, - // and we're checking for an update but it's already up to date. - if (canPlace === KEEP) { - if (edge.peer && !target.children.get(edge.name).satisfies(edge)) { - // this is an overridden peer dep - this[_warnPeerConflict](edge) - } - - // if we get a KEEP in a update scenario, then we MAY have something - // already duplicating this unnecessarily! For example: - // ``` - // root - // +-- x (dep: y@1.x) - // | +-- y@1.0.0 - // +-- y@1.1.0 - // ``` - // Now say we do `reify({update:['y']})`, and the latest version is - // 1.1.0, which we already have in the root. We'll try to place y@1.1.0 - // first in x, then in the root, ending with KEEP, because we already - // have it. In that case, we ought to REMOVE the nm/x/nm/y node, because - // it is an unnecessary duplicate. - this[_pruneDedupable](target) - return [] - } - - // figure out which of this node's peer deps will get placed as well - const virtualRoot = dep.parent - - const newDep = new dep.constructor({ - name: dep.name, - pkg: dep.package, - resolved: dep.resolved, - integrity: dep.integrity, - legacyPeerDeps: this.legacyPeerDeps, - error: dep.errors[0], - ...(dep.isLink ? { target: dep.target, realpath: dep.target.path } : {}), - }) - if (this[_loadFailures].has(dep)) - this[_loadFailures].add(newDep) - - const placed = [newDep] - const oldChild = target.children.get(edge.name) - if (oldChild) { - // if we're replacing, we should also remove any nodes for edges that - // are now invalid, and where this (or its deps) is the only dependent, - // and also recurse on that pruning. Otherwise leaving that dep node - // around can result in spurious conflicts pushing nodes deeper into - // the tree than needed in the case of cycles that will be removed - // later anyway. - const oldDeps = [] - for (const [name, edge] of oldChild.edgesOut.entries()) { - if (!newDep.edgesOut.has(name) && edge.to) - oldDeps.push(...gatherDepSet([edge.to], e => e.to !== edge.to)) - } - newDep.replace(oldChild) - this[_pruneForReplacement](newDep, oldDeps) - // this may also create some invalid edges, for example if we're - // intentionally causing something to get nested which was previously - // placed in this location. - for (const edgeIn of newDep.edgesIn) { - if (edgeIn.invalid && edgeIn !== edge) { - this[_depsQueue].push(edgeIn.from) - this[_depsSeen].delete(edgeIn.from) - } - } - } else - newDep.parent = target - - if (edge.peer && !newDep.satisfies(edge)) { - // this is an overridden peer dep - this[_warnPeerConflict](edge) - } - - // If the edge is not an error, then we're updating something, and - // MAY end up putting a better/identical node further up the tree in - // a way that causes an unnecessary duplication. If so, remove the - // now-unnecessary node. - if (edge.valid && edge.to && edge.to !== newDep) - this[_pruneDedupable](edge.to, false) - - // visit any dependents who are upset by this change - // if it's an angry overridden peer edge, however, make sure we - // skip over it! - for (const edgeIn of newDep.edgesIn) { - if (edgeIn !== edge && !edgeIn.valid && !this[_depsSeen].has(edge.from)) { - this.addTracker('idealTree', edgeIn.from.name, edgeIn.from.location) - this[_depsQueue].push(edgeIn.from) - } - } - - // in case we just made some duplicates that can be removed, - // prune anything deeper in the tree that can be replaced by this - if (this.idealTree) { - for (const node of this.idealTree.inventory.query('name', newDep.name)) { - if (!node.isTop && node.isDescendantOf(target)) - this[_pruneDedupable](node, false) - } - } - - // also place its unmet or invalid peer deps at this location - // note that newDep has now been removed from the virtualRoot set - // by virtue of being placed in the target's node_modules. - // loop through any peer deps from the thing we just placed, and place - // those ones as well. it's safe to do this with the virtual nodes, - // because we're copying rather than moving them out of the virtual root, - // otherwise they'd be gone and the peer set would change throughout - // this loop. - for (const peerEdge of newDep.edgesOut.values()) { - const peer = virtualRoot.children.get(peerEdge.name) - - // Note: if the virtualRoot *doesn't* have the peer, then that means - // it's an optional peer dep. If it's not being properly met (ie, - // peerEdge.valid is false), that this is likely heading for an - // ERESOLVE error, unless it can walk further up the tree. - if (!peerEdge.peer || peerEdge.valid || !peer) - continue - - const peerPlaced = this[_placeDep]( - peer, newDep, peerEdge, peerEntryEdge || edge, peerPath) - placed.push(...peerPlaced) - } - - // we're done with this now, clean it up. - this[_virtualRoots].delete(virtualRoot.sourceReference) - - return placed - } - - // prune all the nodes in a branch of the tree that can be safely removed - // This is only the most basic duplication detection; it finds if there - // is another satisfying node further up the tree, and if so, dedupes. - // Even in legacyBundling mode, we do this amount of deduplication. - [_pruneDedupable] (node, descend = true) { - if (node.canDedupe(this[_preferDedupe])) { - node.root = null - return - } - if (descend) { - // sort these so that they're deterministically ordered - // otherwise, resulting tree shape is dependent on the order - // in which they happened to be resolved. - const nodeSort = (a, b) => a.location.localeCompare(b.location, 'en') - - const children = [...node.children.values()].sort(nodeSort) - const fsChildren = [...node.fsChildren].sort(nodeSort) - for (const child of children) - this[_pruneDedupable](child) - for (const topNode of fsChildren) { - const children = [...topNode.children.values()].sort(nodeSort) - for (const child of children) - this[_pruneDedupable](child) - } - } - } - - [_pruneForReplacement] (node, oldDeps) { - // gather up all the invalid edgesOut, and any now-extraneous - // deps that the new node doesn't depend on but the old one did. - const invalidDeps = new Set([...node.edgesOut.values()] - .filter(e => e.to && !e.valid).map(e => e.to)) - for (const dep of oldDeps) { - const set = gatherDepSet([dep], e => e.to !== dep && e.valid) - for (const dep of set) - invalidDeps.add(dep) - } - - // ignore dependency edges from the node being replaced, but - // otherwise filter the set down to just the set with no - // dependencies from outside the set, except the node in question. - const deps = gatherDepSet(invalidDeps, edge => - edge.from !== node && edge.to !== node && edge.valid) - - // now just delete whatever's left, because it's junk - for (const dep of deps) - dep.parent = null - } - - // check if we can place DEP in TARGET to satisfy EDGE - // Need to verify: - // - no child by that name there already - // - target does not have a peer dep on name - // - no higher-level pkg by that name and incompatible spec is depended on - // by anything lower in the tree. - // - node's peer deps and meta-peer deps are siblings in a virtual root at - // this point. make sure that the whole family can come along, so apply - // the same checks to each of them. They may land higher up in the tree, - // but we need to know that they CAN live here. - // Responses: - // - OK - Yes, because there is nothing there and no conflicts caused - // - REPLACE - Yes, and you can clobber what's there - // - KEEP - No, but what's there is fine - // - CONFLICT - You may not put that there - // - // Check peers on OK or REPLACE. KEEP and CONFLICT do not require peer - // checking, because either we're leaving it alone, or it won't work anyway. - // When we check peers, we pass along the peerEntryEdge to track the - // original edge that caused us to load the family of peer dependencies. - [_canPlaceDep] (dep, target, edge, peerEntryEdge = null, peerPath = [], isSource = false) { - /* istanbul ignore next */ - debug(() => { - if (!dep) - throw new Error('no dep??') - }) - const entryEdge = peerEntryEdge || edge - const source = this[_peerSetSource].get(dep) - - isSource = isSource || target === source - // if we're overriding the source, then we care if the *target* is - // ours, even if it wasn't actually the original source, since we - // are depending on something that has a dep that can't go in its own - // folder. for example, a -> b, b -> PEER(a). Even though a is the - // source, b has to be installed up a level, and if the root package - // depends on a, and it has a conflict, it's our problem. So, the root - // (or whatever is bringing in a) becomes the "effective source" for - // the purposes of this calculation. - const { isProjectRoot, isWorkspace } = isSource ? target : source || {} - const isMine = isProjectRoot || isWorkspace - - // Useful testing thingie right here. - // peerEntryEdge should *always* be a non-peer dependency, or a peer - // dependency from the root node. When we get spurious ERESOLVE errors, - // or *don't* get ERESOLVE errors when we should, check to see if this - // fails, because it MAY mean we got off track somehow. - /* istanbul ignore next - debug check, should be impossible */ - debug(() => { - if (peerEntryEdge && peerEntryEdge.peer && !peerEntryEdge.from.isTop) - throw new Error('lost original peerEntryEdge somehow?') - }) - - if (target.children.has(edge.name)) { - const current = target.children.get(edge.name) - - // same thing = keep, UNLESS the current doesn't satisfy and new - // one does satisfy. This can happen if it's a link to a matching target - // at a different location, which satisfies a version dep, but not a - // file: dep. If neither of them satisfy, then we can replace it, - // because presumably it's better for a peer or something. - if (dep.matches(current)) { - if (current.satisfies(edge) || !dep.satisfies(edge)) - return KEEP - } - - const { version: curVer } = current - const { version: newVer } = dep - const tryReplace = curVer && newVer && semver.gte(newVer, curVer) - if (tryReplace && dep.canReplace(current)) { - const res = this[_canPlacePeers](dep, target, edge, REPLACE, peerEntryEdge, peerPath, isSource) - /* istanbul ignore else - It's extremely rare that a replaceable - * node would be a conflict, if the current one wasn't a conflict, - * but it is theoretically possible if peer deps are pinned. In - * that case we treat it like any other conflict, and keep trying */ - if (res !== CONFLICT) - return res - } - - // ok, can't replace the current with new one, but maybe current is ok? - // no need to check if it's a peer that's valid to be here, because - // peers are always placed along with their entry source - if (edge.satisfiedBy(current)) - return KEEP - - // if we prefer deduping, then try replacing newer with older - // we always prefer to dedupe peers, because they are trying - // a bit harder to be singletons. - const preferDedupe = this[_preferDedupe] || edge.peer - if (preferDedupe && !tryReplace && dep.canReplace(current)) { - const res = this[_canPlacePeers](dep, target, edge, REPLACE, peerEntryEdge, peerPath, isSource) - /* istanbul ignore else - It's extremely rare that a replaceable - * node would be a conflict, if the current one wasn't a conflict, - * but it is theoretically possible if peer deps are pinned. In - * that case we treat it like any other conflict, and keep trying */ - if (res !== CONFLICT) - return res - } - - // check for conflict override cases. - // first: is this the only place this thing can go? If the target is - // the source, then one of these things are true. - // - // 1. the conflicted dep was deduped up to here from a lower dependency - // w -> (x,y) - // x -> (z) - // y -> PEER(p@1) - // z -> (q) - // q -> (p@2) - // - // When building, let's say that x is fully placed, with all of its - // deps, and we're _adding_ y. Since the peer on p@1 was not initially - // present, it's been deduped up to w, and now needs to be pushed out. - // Replace it, and potentially also replace its peer set (though that'll - // be accomplished by making the same determination when we call - // _canPlacePeers) - // - // 2. the dep we're TRYING to place here ought to be overridden by the - // one that's here now, because current is (a) a direct dep of the - // source, or (b) an already-placed peer in a conflicted peer set, or - // (c) an already-placed peer in a different peer set at the same level. - // If strict or ours, conflict. Otherwise, keep. - if (isSource) { - // check to see if the current module could go deeper in the tree - let canReplace = true - // only do this check when we're placing peers. when we're placing - // the original in the source, we know that the edge from the source - // is the thing we're trying to place, so its peer set will need to be - // placed here as well. the virtualRoot already has the appropriate - // overrides applied. - if (peerEntryEdge) { - const currentPeerSet = getPeerSet(current) - - // We are effectively replacing currentPeerSet with newPeerSet - // If there are any non-peer deps coming into the currentPeerSet, - // which are currently valid, and are from the target, then that - // means that we have to ensure that they're not going to be made - // invalid by putting the newPeerSet in place. - // If the edge comes from somewhere deeper than the target, then - // that's fine, because we'll create an invalid edge, detect it, - // and duplicate the node further into the tree. - // loop through the currentPeerSet checking for valid edges on - // the members of the peer set which will be made invalid. - const targetEdges = new Set() - for (const p of currentPeerSet) { - for (const edge of p.edgesIn) { - // edge from within the peerSet, ignore - if (currentPeerSet.has(edge.from)) - continue - // only care about valid edges from target. - // edges from elsewhere can dupe if offended, invalid edges - // are already being fixed or will be later. - if (edge.from !== target || !edge.valid) - continue - targetEdges.add(edge) - } - } - - for (const edge of targetEdges) { - // see if we intend to replace this one anyway - const rep = dep.parent.children.get(edge.name) - const current = edge.to - if (!rep) { - // this isn't one we're replacing. but it WAS included in the - // peerSet for some reason, so make sure that it's still - // ok with the replacements in the new peerSet - for (const curEdge of current.edgesOut.values()) { - const newRepDep = dep.parent.children.get(curEdge.name) - if (curEdge.valid && newRepDep && !newRepDep.satisfies(curEdge)) { - canReplace = false - break - } - } - continue - } - - // was this replacement already an override of some sort? - const override = [...rep.edgesIn].some(e => !e.valid) - // if we have a rep, and it's ok to put in this location, and - // it's not already part of an override in the peerSet, then - // we can continue with it. - if (rep.satisfies(edge) && !override) - continue - // Otherwise, we cannot replace. - canReplace = false - break - } - // if we're going to be replacing the peerSet, we have to remove - // and re-resolve any members of the old peerSet that are not - // present in the new one, and which will have invalid edges. - // We know that they're not depended upon by the target, or else - // they would have caused a conflict, so they'll get landed deeper - // in the tree, if possible. - if (canReplace) { - let needNesting = false - OUTER: for (const node of currentPeerSet) { - const rep = dep.parent.children.get(node.name) - // has a replacement, already addressed above - if (rep) - continue - - // ok, it has been placed here to dedupe, see if it needs to go - // back deeper within the tree. - for (const edge of node.edgesOut.values()) { - const repDep = dep.parent.children.get(edge.name) - // not in new peerSet, maybe fine. - if (!repDep) - continue - - // new thing will be fine, no worries - if (repDep.satisfies(edge)) - continue - - // uhoh, we'll have to nest them. - needNesting = true - break OUTER - } - } - - // to nest, just delete everything without a target dep - // that's in the current peerSet, and add their dependants - // to the _depsQueue for evaluation. Some of these MAY end - // up in the same location again, and that's fine. - if (needNesting) { - // avoid mutating the tree while we're examining it - const dependants = new Set() - const reresolve = new Set() - OUTER: for (const node of currentPeerSet) { - const rep = dep.parent.children.get(node.name) - if (rep) - continue - // create a separate set for each one, so we can skip any - // that might somehow have an incoming target edge - const deps = new Set() - for (const edge of node.edgesIn) { - // a target dep, skip this dep entirely, already addressed - // ignoring for coverage, because it really ought to be - // impossible, but I can't prove it yet, so this is here - // for safety. - /* istanbul ignore if - should be impossible */ - if (edge.from === target) - continue OUTER - // ignore this edge, it'll either be replaced or re-resolved - if (currentPeerSet.has(edge.from)) - continue - // ok, we care about this one. - deps.add(edge.from) - } - reresolve.add(node) - for (const d of deps) - dependants.add(d) - } - for (const dependant of dependants) { - this[_depsQueue].push(dependant) - this[_depsSeen].delete(dependant) - } - for (const node of reresolve) - node.root = null - } - } - } - - if (canReplace) { - const ret = this[_canPlacePeers](dep, target, edge, REPLACE, peerEntryEdge, peerPath, isSource) - /* istanbul ignore else - extremely rare that the peer set would - * conflict if we can replace the node in question, but theoretically - * possible, if peer deps are pinned aggressively. */ - if (ret !== CONFLICT) - return ret - } - - // so it's not a deeper dep that's been deduped. That means that the - // only way it could have ended up here is if it's a conflicted peer. - /* istanbul ignore else - would have already crashed if not forced, - * and either mine or strict, when creating the peerSet. Keeping this - * check so that we're not only relying on action at a distance. */ - if (!this[_strictPeerDeps] && !isMine || this[_force]) { - this[_warnPeerConflict](edge, dep) - return KEEP - } - } - - // no justification for overriding, and no agreement possible. - return CONFLICT - } - - // no existing node at this location! - // check to see if the target doesn't have a child by that name, - // but WANTS one, and won't be happy with this one. if this is the - // edge we're looking to resolve, then not relevant, of course. - if (target !== entryEdge.from && target.edgesOut.has(dep.name)) { - const targetEdge = target.edgesOut.get(dep.name) - // It might be that the dep would not be valid here, BUT some other - // version would. Could to try to resolve that, but that makes this no - // longer a pure synchronous function. ugh. - // This is a pretty unlikely scenario in a normal install, because we - // resolve the peer dep set against the parent dependencies, and - // presumably they all worked together SOMEWHERE to get published in the - // first place, and since we resolve shallower deps before deeper ones, - // this can only occur by a child having a peer dep that does not satisfy - // the parent. It can happen if we're doing a deep update limited by - // a specific name, however, or if a dep makes an incompatible change - // to its peer dep in a non-semver-major version bump, or if the parent - // is unbounded in its dependency list. - if (!targetEdge.satisfiedBy(dep)) - return CONFLICT - } - - // check to see what that name resolves to here, and who may depend on - // being able to reach it by crawling up past this parent. we know - // at this point that it's not the target's direct child node. if it's - // a direct dep of the target, we just make the invalid edge and - // resolve it later. - const current = target !== entryEdge.from && target.resolve(dep.name) - if (current) { - for (const edge of current.edgesIn.values()) { - if (!edge.from.isTop && edge.from.isDescendantOf(target) && edge.valid) { - if (!edge.satisfiedBy(dep)) - return CONFLICT - } - } - } - - // no objections! ok to place here - return this[_canPlacePeers](dep, target, edge, OK, peerEntryEdge, peerPath, isSource) - } - - // make sure the family of peer deps can live here alongside it. - // this doesn't guarantee that THIS solution will be the one we take, - // but it does establish that SOME solution exists at this level in - // the tree. - [_canPlacePeers] (dep, target, edge, ret, peerEntryEdge, peerPath, isSource) { - // do not go in cycles when we're resolving a peer group - if (!dep.parent || peerEntryEdge && peerPath.includes(dep)) - return ret - - const entryEdge = peerEntryEdge || edge - peerPath = [...peerPath, dep] - - for (const peerEdge of dep.edgesOut.values()) { - if (!peerEdge.peer || !peerEdge.to) - continue - const peer = peerEdge.to - const canPlacePeer = this[_canPlaceDep](peer, target, peerEdge, entryEdge, peerPath, isSource) - if (canPlacePeer !== CONFLICT) - continue - - const current = target.resolve(peer.name) - this[_peerConflict] = { - peer: peer.explain(peerEdge), - current: current && current.explain(), - } - return CONFLICT - } - return ret - } - // go through all the links in the this[_linkNodes] set // for each one: // - if outside the root, ignore it, assume it's fine, it's not our problem @@ -1945,6 +1364,7 @@ This is a one-time fix-up, please be patient... const needPrune = metaFromDisk && (mutateTree || flagsSuspect) if (this[_prune] && needPrune) this[_idealTreePrune]() + process.emit('timeEnd', 'idealTree:fixDepFlags') } diff --git a/node_modules/@npmcli/arborist/lib/arborist/reify.js b/node_modules/@npmcli/arborist/lib/arborist/reify.js index 18b5cd65262a6..1cfa6034eadb8 100644 --- a/node_modules/@npmcli/arborist/lib/arborist/reify.js +++ b/node_modules/@npmcli/arborist/lib/arborist/reify.js @@ -5,6 +5,7 @@ const pacote = require('pacote') const AuditReport = require('../audit-report.js') const {subset, intersects} = require('semver') const npa = require('npm-package-arg') +const debug = require('../debug.js') const {dirname, resolve, relative} = require('path') const {depth: dfwalk} = require('treeverse') @@ -50,6 +51,7 @@ const _createSparseTree = Symbol.for('createSparseTree') const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees') const _shrinkwrapInflated = Symbol('shrinkwrapInflated') const _bundleUnpacked = Symbol('bundleUnpacked') +const _bundleMissing = Symbol('bundleMissing') const _reifyNode = Symbol.for('reifyNode') const _extractOrLink = Symbol('extractOrLink') // defined by rebuild mixin @@ -83,8 +85,9 @@ const _omitPeer = Symbol('omitPeer') const _global = Symbol.for('global') +const _pruneBundledMetadeps = Symbol('pruneBundledMetadeps') + // defined by Ideal mixin -const _pruneBundledMetadeps = Symbol.for('pruneBundledMetadeps') const _resolvedAdd = Symbol.for('resolvedAdd') const _usePackageLock = Symbol.for('usePackageLock') const _formatPackageLock = Symbol.for('formatPackageLock') @@ -112,6 +115,10 @@ module.exports = cls => class Reifier extends cls { this[_sparseTreeDirs] = new Set() this[_sparseTreeRoots] = new Set() this[_trashList] = new Set() + // the nodes we unpack to read their bundles + this[_bundleUnpacked] = new Set() + // child nodes we'd EXPECT to be included in a bundle, but aren't + this[_bundleMissing] = new Set() } // public method @@ -334,7 +341,7 @@ module.exports = cls => class Reifier extends cls { // removed later on in the process. optionally, also mark them // as a retired paths, so that we move them out of the way and // replace them when rolling back on failure. - [_addNodeToTrashList] (node, retire) { + [_addNodeToTrashList] (node, retire = false) { const paths = [node.path, ...node.binPaths] const moves = this[_retiredPaths] this.log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths) @@ -610,10 +617,9 @@ module.exports = cls => class Reifier extends cls { [_loadBundlesAndUpdateTrees] ( depth = 0, bundlesByDepth = this[_getBundlesByDepth]() ) { - if (depth === 0) { - this[_bundleUnpacked] = new Set() + if (depth === 0) process.emit('time', 'reify:loadBundles') - } + const maxBundleDepth = bundlesByDepth.get('maxBundleDepth') if (depth > maxBundleDepth) { // if we did something, then prune the tree and update the diffs @@ -642,14 +648,30 @@ module.exports = cls => class Reifier extends cls { })) // then load their unpacked children and move into the ideal tree .then(nodes => - promiseAllRejectLate(nodes.map(node => new this.constructor({ - ...this.options, - path: node.path, - }).loadActual({ - root: node, - // don't transplant any sparse folders we created - transplantFilter: node => node.package._id, - })))) + promiseAllRejectLate(nodes.map(async node => { + const arb = new this.constructor({ + ...this.options, + path: node.path, + }) + const notTransplanted = new Set(node.children.keys()) + await arb.loadActual({ + root: node, + // don't transplant any sparse folders we created + // loadActual will set node.package to {} for empty directories + // if by chance there are some empty folders in the node_modules + // tree for some other reason, then ok, ignore those too. + transplantFilter: node => { + if (node.package._id) { + // it's actually in the bundle if it gets transplanted + notTransplanted.delete(node.name) + return true + } else + return false + }, + }) + for (const name of notTransplanted) + this[_bundleMissing].add(node.children.get(name)) + }))) // move onto the next level of bundled items .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)) } @@ -685,6 +707,27 @@ module.exports = cls => class Reifier extends cls { // https://github.com/npm/cli/issues/1597#issuecomment-667639545 [_pruneBundledMetadeps] (bundlesByDepth) { const bundleShadowed = new Set() + + // Example dep graph: + // root -> (a, c) + // a -> BUNDLE(b) + // b -> c + // c -> b + // + // package tree: + // root + // +-- a + // | +-- b(1) + // | +-- c(1) + // +-- b(2) + // +-- c(2) + // 1. mark everything that's shadowed by anything in the bundle. This + // marks b(2) and c(2). + // 2. anything with edgesIn from outside the set, mark not-extraneous, + // remove from set. This unmarks c(2). + // 3. continue until no change + // 4. remove everything in the set from the tree. b(2) is pruned + // create the list of nodes shadowed by children of bundlers for (const bundles of bundlesByDepth.values()) { // skip the 'maxBundleDepth' item @@ -700,36 +743,50 @@ module.exports = cls => class Reifier extends cls { } } } - let changed = true - while (changed) { - changed = false - for (const shadow of bundleShadowed) { - if (!shadow.extraneous) { - bundleShadowed.delete(shadow) - continue + + // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3)) + // a@1.2.3 -> (b@1.2.3) + // a@1.3.0 -> (b@2) + // b@1.2.3 -> () + // b@2 -> (c@2) + // + // root + // +-- lib + // | +-- a@1.2.3 + // | +-- b@1.2.3 + // +-- b@2 <-- shadowed, now extraneous + // +-- c@2 <-- also shadowed, because only dependent is shadowed + for (const shadow of bundleShadowed) { + for (const shadDep of shadow.edgesOut.values()) { + /* istanbul ignore else - pretty unusual situation, just being + * defensive here. Would mean that a bundled dep has a dependency + * that is unmet. which, weird, but if you bundle it, we take + * whatever you put there and assume the publisher knows best. */ + if (shadDep.to) { + bundleShadowed.add(shadDep.to) + shadDep.to.extraneous = true } + } + } + let changed + do { + changed = false + for (const shadow of bundleShadowed) { for (const edge of shadow.edgesIn) { - if (!edge.from.extraneous) { + if (!bundleShadowed.has(edge.from)) { shadow.extraneous = false bundleShadowed.delete(shadow) changed = true - } else { - for (const shadDep of shadow.edgesOut.values()) { - /* istanbul ignore else - pretty unusual situation, just being - * defensive here. Would mean that a bundled dep has a dependency - * that is unmet. which, weird, but if you bundle it, we take - * whatever you put there and assume the publisher knows best. */ - if (shadDep.to) - bundleShadowed.add(shadDep.to) - } + break } } } - } + } while (changed) + for (const shadow of bundleShadowed) { - shadow.parent = null this[_addNodeToTrashList](shadow) + shadow.root = null } } @@ -780,6 +837,7 @@ module.exports = cls => class Reifier extends cls { const node = diff.ideal const bd = this[_bundleUnpacked].has(node) const sw = this[_shrinkwrapInflated].has(node) + const bundleMissing = this[_bundleMissing].has(node) // check whether we still need to unpack this one. // test the inDepBundle last, since that's potentially a tree walk. @@ -787,7 +845,7 @@ module.exports = cls => class Reifier extends cls { !node.isRoot && // root node already exists !bd && // already unpacked to read bundle !sw && // already unpacked to read sw - !node.inDepBundle // already unpacked by another dep's bundle + (bundleMissing || !node.inDepBundle) // already unpacked by another dep's bundle if (doUnpack) unpacks.push(this[_reifyNode](node)) @@ -814,8 +872,26 @@ module.exports = cls => class Reifier extends cls { const moves = this[_retiredPaths] this[_retiredUnchanged] = {} return promiseAllRejectLate(this.diff.children.map(diff => { - const realFolder = (diff.actual || diff.ideal).path + // skip if nothing was retired + if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') + return + + const { path: realFolder } = diff.actual const retireFolder = moves[realFolder] + /* istanbul ignore next - should be impossible */ + debug(() => { + if (!retireFolder) { + const er = new Error('trying to un-retire but not retired') + throw Object.assign(er, { + realFolder, + retireFolder, + actual: diff.actual, + ideal: diff.ideal, + action: diff.action, + }) + } + }) + this[_retiredUnchanged][retireFolder] = [] return promiseAllRejectLate(diff.unchanged.map(node => { // no need to roll back links, since we'll just delete them anyway @@ -823,7 +899,7 @@ module.exports = cls => class Reifier extends cls { return mkdirp(dirname(node.path)).then(() => this[_reifyNode](node)) // will have been moved/unpacked along with bundler - if (node.inDepBundle) + if (node.inDepBundle && !this[_bundleMissing].has(node)) return this[_retiredUnchanged][retireFolder].push(node) diff --git a/node_modules/@npmcli/arborist/lib/can-place-dep.js b/node_modules/@npmcli/arborist/lib/can-place-dep.js new file mode 100644 index 0000000000000..cf6b800c44ea2 --- /dev/null +++ b/node_modules/@npmcli/arborist/lib/can-place-dep.js @@ -0,0 +1,405 @@ +// Internal methods used by buildIdealTree. +// Answer the question: "can I put this dep here?" +// +// IMPORTANT: *nothing* in this class should *ever* modify or mutate the tree +// at all. The contract here is strictly limited to read operations. We call +// this in the process of walking through the ideal tree checking many +// different potential placement targets for a given node. If a change is made +// to the tree along the way, that can cause serious problems! +// +// In order to enforce this restriction, in debug mode, canPlaceDep() will +// snapshot the tree at the start of the process, and then at the end, will +// verify that it still matches the snapshot, and throw an error if any changes +// occurred. +// +// The algorithm is roughly like this: +// - check the node itself: +// - if there is no version present, and no conflicting edges from target, +// OK, provided all peers can be placed at or above the target. +// - if the current version matches, KEEP +// - if there is an older version present, which can be replaced, then +// - if satisfying and preferDedupe? KEEP +// - else: REPLACE +// - if there is a newer version present, and preferDedupe, REPLACE +// - if the version present satisfies the edge, KEEP +// - else: CONFLICT +// - if the node is not in conflict, check each of its peers: +// - if the peer can be placed in the target, continue +// - else if the peer can be placed in a parent, and there is no other +// conflicting version shadowing it, continue +// - else CONFLICT +// - If the peers are not in conflict, return the original node's value +// +// An exception to this logic is that if the target is the deepest location +// that a node can be placed, and the conflicting node can be placed deeper, +// then we will return REPLACE rather than CONFLICT, and Arborist will queue +// the replaced node for resolution elsewhere. + +const semver = require('semver') +const debug = require('./debug.js') +const peerEntrySets = require('./peer-entry-sets.js') +const deepestNestingTarget = require('./deepest-nesting-target.js') + +const CONFLICT = Symbol('CONFLICT') +const OK = Symbol('OK') +const REPLACE = Symbol('REPLACE') +const KEEP = Symbol('KEEP') + +class CanPlaceDep { + // dep is a dep that we're trying to place. it should already live in + // a virtual tree where its peer set is loaded as children of the root. + // target is the actual place where we're trying to place this dep + // in a node_modules folder. + // edge is the edge that we're trying to satisfy with this placement. + // parent is the CanPlaceDep object of the entry node when placing a peer. + constructor (options) { + const { + dep, + target, + edge, + preferDedupe, + parent = null, + peerPath = [], + explicitRequest = false, + } = options + + debug(() => { + if (!dep) + throw new Error('no dep provided to CanPlaceDep') + + if (!target) + throw new Error('no target provided to CanPlaceDep') + + if (!edge) + throw new Error('no edge provided to CanPlaceDep') + + this._nodeSnapshot = JSON.stringify(dep) + this._treeSnapshot = JSON.stringify(target.root) + }) + + // the result of whether we can place it or not + this.canPlace = null + // if peers conflict, but this one doesn't, then that is useful info + this.canPlaceSelf = null + + this.dep = dep + this.target = target + this.edge = edge + this.explicitRequest = explicitRequest + + // preventing cycles when we check peer sets + this.peerPath = peerPath + // we always prefer to dedupe peers, because they are trying + // a bit harder to be singletons. + this.preferDedupe = !!preferDedupe || edge.peer + this.parent = parent + this.children = [] + + this.isSource = target === this.peerSetSource + this.name = edge.name + this.current = target.children.get(this.name) + this.targetEdge = target.edgesOut.get(this.name) + this.conflicts = new Map() + + // check if this dep was already subject to a peerDep override while + // building the peerSet. + this.edgeOverride = !dep.satisfies(edge) + + this.canPlace = this.checkCanPlace() + if (!this.canPlaceSelf) + this.canPlaceSelf = this.canPlace + + debug(() => { + const nodeSnapshot = JSON.stringify(dep) + const treeSnapshot = JSON.stringify(target.root) + /* istanbul ignore if */ + if (this._nodeSnapshot !== nodeSnapshot) { + throw Object.assign(new Error('dep changed in CanPlaceDep'), { + expect: this._nodeSnapshot, + actual: nodeSnapshot, + }) + } + /* istanbul ignore if */ + if (this._treeSnapshot !== treeSnapshot) { + throw Object.assign(new Error('tree changed in CanPlaceDep'), { + expect: this._treeSnapshot, + actual: treeSnapshot, + }) + } + }) + } + + checkCanPlace () { + const { target, targetEdge, current, dep } = this + + // if the dep failed to load, we're going to fail the build or + // prune it out anyway, so just move forward placing/replacing it. + if (dep.errors.length) + return current ? REPLACE : OK + + // cannot place peers inside their dependents, except for tops + if (targetEdge && targetEdge.peer && !target.isTop) + return CONFLICT + + if (targetEdge && !dep.satisfies(targetEdge) && targetEdge !== this.edge) + return CONFLICT + + return current ? this.checkCanPlaceCurrent() : this.checkCanPlaceNoCurrent() + } + + // we know that the target has a dep by this name in its node_modules + // already. Can return KEEP, REPLACE, or CONFLICT. + checkCanPlaceCurrent () { + const { preferDedupe, explicitRequest, current, target, edge, dep } = this + + if (dep.matches(current)) { + if (current.satisfies(edge) || this.edgeOverride) + return explicitRequest ? REPLACE : KEEP + } + + const { version: curVer } = current + const { version: newVer } = dep + const tryReplace = curVer && newVer && semver.gte(newVer, curVer) + if (tryReplace && dep.canReplace(current)) { + /* XXX-istanbul ignore else - It's extremely rare that a replaceable + * node would be a conflict, if the current one wasn't a conflict, + * but it is theoretically possible if peer deps are pinned. In + * that case we treat it like any other conflict, and keep trying */ + const cpp = this.canPlacePeers(REPLACE) + if (cpp !== CONFLICT) + return cpp + } + + // ok, can't replace the current with new one, but maybe current is ok? + if (current.satisfies(edge) && (!explicitRequest || preferDedupe)) + return KEEP + + // if we prefer deduping, then try replacing newer with older + if (preferDedupe && !tryReplace && dep.canReplace(current)) { + const cpp = this.canPlacePeers(REPLACE) + if (cpp !== CONFLICT) + return cpp + } + + // Check for interesting cases! + // First, is this the deepest place that this thing can go, and NOT the + // deepest place where the conflicting dep can go? If so, replace it, + // and let it re-resolve deeper in the tree. + const myDeepest = this.deepestNestingTarget + + // ok, i COULD be placed deeper, so leave the current one alone. + if (target !== myDeepest) + return CONFLICT + + // if we are not checking a peerDep, then we MUST place it here, in the + // target that has a non-peer dep on it. + if (!edge.peer && target === edge.from) + return this.canPlacePeers(REPLACE) + + // if we aren't placing a peer in a set, then we're done here. + // This is ignored because it SHOULD be redundant, as far as I can tell, + // with the deepest target and target===edge.from tests. But until we + // can prove that isn't possible, this condition is here for safety. + /* istanbul ignore if - allegedly impossible */ + if (!this.parent && !edge.peer) + return CONFLICT + + // check the deps in the peer group for each edge into that peer group + // if ALL of them can be pushed deeper, or if it's ok to replace its + // members with the contents of the new peer group, then we're good. + let canReplace = true + for (const [entryEdge, currentPeers] of peerEntrySets(current)) { + if (entryEdge === this.edge || entryEdge === this.peerEntryEdge) + continue + + // First, see if it's ok to just replace the peerSet entirely. + // we do this by walking out from the entryEdge, because in a case like + // this: + // + // v -> PEER(a@1||2) + // a@1 -> PEER(b@1) + // a@2 -> PEER(b@2) + // b@1 -> PEER(a@1) + // b@2 -> PEER(a@2) + // + // root + // +-- v + // +-- a@2 + // +-- b@2 + // + // Trying to place a peer group of (a@1, b@1) would fail to note that + // they can be replaced, if we did it by looping 1 by 1. If we are + // replacing something, we don't have to check its peer deps, because + // the peerDeps in the placed peerSet will presumably satisfy. + const entryNode = entryEdge.to + const entryRep = dep.parent.children.get(entryNode.name) + if (entryRep) { + if (entryRep.canReplace(entryNode, dep.parent.children.keys())) + continue + } + + let canClobber = !entryRep + if (!entryRep) { + const peerReplacementWalk = new Set([entryNode]) + OUTER: for (const currentPeer of peerReplacementWalk) { + for (const edge of currentPeer.edgesOut.values()) { + if (!edge.peer || !edge.valid) + continue + const rep = dep.parent.children.get(edge.name) + if (!rep) { + if (edge.to) + peerReplacementWalk.add(edge.to) + continue + } + if (!rep.satisfies(edge)) { + canClobber = false + break OUTER + } + } + } + } + if (canClobber) + continue + + // ok, we can't replace, but maybe we can nest the current set deeper? + let canNestCurrent = true + for (const currentPeer of currentPeers) { + if (!canNestCurrent) + break + + // still possible to nest this peerSet + const curDeep = deepestNestingTarget(entryEdge.from, currentPeer.name) + if (curDeep === target || target.isDescendantOf(curDeep)) { + canNestCurrent = false + canReplace = false + } + if (canNestCurrent) + continue + } + } + + // if we can nest or replace all the current peer groups, we can replace. + if (canReplace) + return this.canPlacePeers(REPLACE) + + return CONFLICT + } + + checkCanPlaceNoCurrent () { + const { target, peerEntryEdge, dep, name } = this + + // check to see what that name resolves to here, and who may depend on + // being able to reach it by crawling up past the parent. we know + // that it's not the target's direct child node, and if it was a direct + // dep of the target, we would have conflicted earlier. + const current = target !== peerEntryEdge.from && target.resolve(name) + if (current) { + for (const edge of current.edgesIn.values()) { + if (edge.from.isDescendantOf(target) && edge.valid) { + if (!dep.satisfies(edge)) + return CONFLICT + } + } + } + + // no objections, so this is fine as long as peers are ok here. + return this.canPlacePeers(OK) + } + + get deepestNestingTarget () { + const start = this.parent ? this.parent.deepestNestingTarget + : this.edge.from + return deepestNestingTarget(start, this.name) + } + + get conflictChildren () { + return this.allChildren.filter(c => c.canPlace === CONFLICT) + } + + get allChildren () { + const set = new Set(this.children) + for (const child of set) { + for (const grandchild of child.children) + set.add(grandchild) + } + return [...set] + } + + get top () { + return this.parent ? this.parent.top : this + } + + // check if peers can go here. returns state or CONFLICT + canPlacePeers (state) { + this.canPlaceSelf = state + if (this._canPlacePeers) + return this._canPlacePeers + + // TODO: represent peerPath in ERESOLVE error somehow? + const peerPath = [...this.peerPath, this.dep] + let sawConflict = false + for (const peerEdge of this.dep.edgesOut.values()) { + if (!peerEdge.peer || !peerEdge.to || peerPath.includes(peerEdge.to)) + continue + const peer = peerEdge.to + // it may be the case that the *initial* dep can be nested, but a peer + // of that dep needs to be placed shallower, because the target has + // a peer dep on the peer as well. + const target = deepestNestingTarget(this.target, peer.name) + const cpp = new CanPlaceDep({ + dep: peer, + target, + parent: this, + edge: peerEdge, + peerPath, + // always place peers in preferDedupe mode + preferDedupe: true, + }) + /* istanbul ignore next */ + debug(() => { + if (this.children.some(c => c.dep === cpp.dep)) + throw new Error('checking same dep repeatedly') + }) + this.children.push(cpp) + + if (cpp.canPlace === CONFLICT) + sawConflict = true + } + + this._canPlacePeers = sawConflict ? CONFLICT : state + return this._canPlacePeers + } + + // what is the node that is causing this peerSet to be placed? + get peerSetSource () { + return this.parent ? this.parent.peerSetSource : this.edge.from + } + + get peerEntryEdge () { + return this.top.edge + } + + static get CONFLICT () { + return CONFLICT + } + + static get OK () { + return OK + } + + static get REPLACE () { + return REPLACE + } + + static get KEEP () { + return KEEP + } + + get description () { + const { canPlace } = this + return canPlace && canPlace.description || + /* istanbul ignore next - old node affordance */ canPlace + } +} + +module.exports = CanPlaceDep diff --git a/node_modules/@npmcli/arborist/lib/deepest-nesting-target.js b/node_modules/@npmcli/arborist/lib/deepest-nesting-target.js new file mode 100644 index 0000000000000..cbaa396f3f251 --- /dev/null +++ b/node_modules/@npmcli/arborist/lib/deepest-nesting-target.js @@ -0,0 +1,16 @@ +// given a starting node, what is the *deepest* target where name could go? +// This is not on the Node class for the simple reason that we sometimes +// need to check the deepest *potential* target for a Node that is not yet +// added to the tree where we are checking. +const deepestNestingTarget = (start, name) => { + for (const target of start.ancestry()) { + // note: this will skip past the first target if edge is peer + if (target.isProjectRoot || !target.resolveParent) + return target + const targetEdge = target.edgesOut.get(name) + if (!targetEdge || !targetEdge.peer) + return target + } +} + +module.exports = deepestNestingTarget diff --git a/node_modules/@npmcli/arborist/lib/edge.js b/node_modules/@npmcli/arborist/lib/edge.js index 79510d509f283..0bd9021d56a70 100644 --- a/node_modules/@npmcli/arborist/lib/edge.js +++ b/node_modules/@npmcli/arborist/lib/edge.js @@ -37,6 +37,7 @@ const printableEdge = (edge) => { ...(edgeFrom != null ? { from: edgeFrom } : {}), ...(edgeTo ? { to: edgeTo } : {}), ...(edge.error ? { error: edge.error } : {}), + ...(edge.overridden ? { overridden: true } : {}), }) } @@ -72,6 +73,7 @@ class Edge { throw new TypeError('must provide "from" node') this[_setFrom](from) this[_error] = this[_loadError]() + this.overridden = false } satisfiedBy (node) { diff --git a/node_modules/@npmcli/arborist/lib/node.js b/node_modules/@npmcli/arborist/lib/node.js index 2ef0a64f08829..d77b18355ff31 100644 --- a/node_modules/@npmcli/arborist/lib/node.js +++ b/node_modules/@npmcli/arborist/lib/node.js @@ -481,6 +481,11 @@ class Node { return this === this.root || this === this.root.target } + * ancestry () { + for (let anc = this; anc; anc = anc.resolveParent) + yield anc + } + set root (root) { // setting to null means this is the new root // should only ever be one step @@ -878,16 +883,31 @@ class Node { // root dependency brings peer deps along with it. In that case, we // will go ahead and create the invalid state, and then try to resolve // it with more tree construction, because it's a user request. - canReplaceWith (node) { + canReplaceWith (node, ignorePeers = []) { if (node.name !== this.name) return false + if (node.packageName !== this.packageName) + return false + + ignorePeers = new Set(ignorePeers) + // gather up all the deps of this node and that are only depended // upon by deps of this node. those ones don't count, since // they'll be replaced if this node is replaced anyway. const depSet = gatherDepSet([this], e => e.to !== this && e.valid) for (const edge of this.edgesIn) { + // when replacing peer sets, we need to be able to replace the entire + // peer group, which means we ignore incoming edges from other peers + // within the replacement set. + const ignored = !this.isTop && + edge.from.parent === this.parent && + edge.peer && + ignorePeers.has(edge.from.name) + if (ignored) + continue + // only care about edges that don't originate from this node if (!depSet.has(edge.from) && !edge.satisfiedBy(node)) return false @@ -896,8 +916,8 @@ class Node { return true } - canReplace (node) { - return node.canReplaceWith(this) + canReplace (node, ignorePeers) { + return node.canReplaceWith(this, ignorePeers) } // return true if it's safe to remove this node, because anything that @@ -1210,6 +1230,12 @@ class Node { } resolve (name) { + /* istanbul ignore next - should be impossible, + * but I keep doing this mistake in tests */ + debug(() => { + if (typeof name !== 'string' || !name) + throw new Error('non-string passed to Node.resolve') + }) const mine = this.children.get(name) if (mine) return mine diff --git a/node_modules/@npmcli/arborist/lib/peer-entry-sets.js b/node_modules/@npmcli/arborist/lib/peer-entry-sets.js new file mode 100644 index 0000000000000..11f9a431607ec --- /dev/null +++ b/node_modules/@npmcli/arborist/lib/peer-entry-sets.js @@ -0,0 +1,72 @@ +// Given a node in a tree, return all of the peer dependency sets that +// it is a part of, with the entry (top or non-peer) edges into the sets +// identified. +// +// With this information, we can determine whether it is appropriate to +// replace the entire peer set with another (and remove the old one), +// push the set deeper into the tree, and so on. +// +// Returns a Map of { edge => Set(peerNodes) }, + +const peerEntrySets = node => { + // this is the union of all peer groups that the node is a part of + // later, we identify all of the entry edges, and create a set of + // 1 or more overlapping sets that this node is a part of. + const unionSet = new Set([node]) + for (const node of unionSet) { + for (const edge of node.edgesOut.values()) { + if (edge.valid && edge.peer && edge.to) + unionSet.add(edge.to) + } + for (const edge of node.edgesIn) { + if (edge.valid && edge.peer) + unionSet.add(edge.from) + } + } + const entrySets = new Map() + for (const peer of unionSet) { + for (const edge of peer.edgesIn) { + // if not valid, it doesn't matter anyway. either it's been previously + // overridden, or it's the thing we're interested in replacing. + if (!edge.valid) + continue + // this is the entry point into the peer set + if (!edge.peer || edge.from.isTop) { + // get the subset of peer brought in by this peer entry edge + const sub = new Set([peer]) + for (const peer of sub) { + for (const edge of peer.edgesOut.values()) { + if (edge.valid && edge.peer && edge.to) + sub.add(edge.to) + } + } + // if this subset does not include the node we are focused on, + // then it is not relevant for our purposes. Example: + // + // a -> (b, c, d) + // b -> PEER(d) b -> d -> e -> f <-> g + // c -> PEER(f, h) c -> (f <-> g, h -> g) + // d -> PEER(e) d -> e -> f <-> g + // e -> PEER(f) + // f -> PEER(g) + // g -> PEER(f) + // h -> PEER(g) + // + // The unionSet(e) will include c, but we don't actually care about + // it. We only expanded to the edge of the peer nodes in order to + // find the entry edges that caused the inclusion of peer sets + // including (e), so we want: + // Map{ + // Edge(a->b) => Set(b, d, e, f, g) + // Edge(a->d) => Set(d, e, f, g) + // } + if (sub.has(node)) + entrySets.set(edge, sub) + } + } + } + + return entrySets +} + +module.exports = peerEntrySets diff --git a/node_modules/@npmcli/arborist/lib/peer-set.js b/node_modules/@npmcli/arborist/lib/peer-set.js deleted file mode 100644 index 727814e1de3f0..0000000000000 --- a/node_modules/@npmcli/arborist/lib/peer-set.js +++ /dev/null @@ -1,25 +0,0 @@ -// when we have to dupe a set of peer dependencies deeper into the tree in -// order to make room for a dep that would otherwise conflict, we use -// this to get the set of all deps that have to be checked to ensure -// nothing is locking them into the current location. -// -// this is different in its semantics from an "optional set" (ie, the nodes -// that should be removed if an optional dep fails), because in this case, -// we specifically intend to include deps in the peer set that have -// dependants outside the set. -const peerSet = node => { - const set = new Set([node]) - for (const node of set) { - for (const edge of node.edgesOut.values()) { - if (edge.valid && edge.peer && edge.to) - set.add(edge.to) - } - for (const edge of node.edgesIn) { - if (edge.valid && edge.peer) - set.add(edge.from) - } - } - return set -} - -module.exports = peerSet diff --git a/node_modules/@npmcli/arborist/lib/place-dep.js b/node_modules/@npmcli/arborist/lib/place-dep.js new file mode 100644 index 0000000000000..913b2ba6c2bc7 --- /dev/null +++ b/node_modules/@npmcli/arborist/lib/place-dep.js @@ -0,0 +1,536 @@ +// Given a dep, a node that depends on it, and the edge representing that +// dependency, place the dep somewhere in the node's tree, and all of its +// peer dependencies. +// +// Handles all of the tree updating needed to place the dep, including +// removing replaced nodes, pruning now-extraneous or invalidated nodes, +// and saves a set of what was placed and what needs re-evaluation as +// a result. + +const log = require('proc-log') +const deepestNestingTarget = require('./deepest-nesting-target.js') +const CanPlaceDep = require('./can-place-dep.js') +const { + KEEP, + CONFLICT, +} = CanPlaceDep +const debug = require('./debug.js') + +const gatherDepSet = require('./gather-dep-set.js') +const peerEntrySets = require('./peer-entry-sets.js') + +class PlaceDep { + constructor (options) { + const { + dep, + edge, + parent = null, + } = options + this.name = edge.name + this.dep = dep + this.edge = edge + this.canPlace = null + + this.target = null + this.placed = null + + // inherit all these fields from the parent to ensure consistency. + const { + preferDedupe, + force, + explicitRequest, + updateNames, + auditReport, + legacyBundling, + strictPeerDeps, + legacyPeerDeps, + globalStyle, + } = parent || options + Object.assign(this, { + preferDedupe, + force, + explicitRequest, + updateNames, + auditReport, + legacyBundling, + strictPeerDeps, + legacyPeerDeps, + globalStyle, + }) + + this.children = [] + this.parent = parent + this.peerConflict = null + + this.checks = new Map() + + this.place() + } + + place () { + const { + edge, + dep, + preferDedupe, + globalStyle, + legacyBundling, + explicitRequest, + updateNames, + checks, + } = this + + // nothing to do if the edge is fine as it is + if (edge.to && + !edge.error && + !explicitRequest && + !updateNames.includes(edge.name) && + !this.isVulnerable(edge.to)) + return + + // walk up the tree until we hit either a top/root node, or a place + // where the dep is not a peer dep. + const start = this.getStartNode() + + let canPlace = null + let canPlaceSelf = null + for (const target of start.ancestry()) { + // if the current location has a peerDep on it, then we can't place here + // this is pretty rare to hit, since we always prefer deduping peers, + // and the getStartNode will start us out above any peers from the + // thing that depends on it. but we could hit it with something like: + // + // a -> (b@1, c@1) + // +-- c@1 + // +-- b -> PEEROPTIONAL(v) (c@2) + // +-- c@2 -> (v) + // + // So we check if we can place v under c@2, that's fine. + // Then we check under b, and can't, because of the optional peer dep. + // but we CAN place it under a, so the correct thing to do is keep + // walking up the tree. + const targetEdge = target.edgesOut.get(edge.name) + if (!target.isTop && targetEdge && targetEdge.peer) + continue + + const cpd = new CanPlaceDep({ + dep, + edge, + // note: this sets the parent's canPlace as the parent of this + // canPlace, but it does NOT add this canPlace to the parent's + // children. This way, we can know that it's a peer dep, and + // get the top edge easily, while still maintaining the + // tree of checks that factored into the original decision. + parent: this.parent && this.parent.canPlace, + target, + preferDedupe, + explicitRequest: this.explicitRequest, + }) + checks.set(target, cpd) + + // It's possible that a "conflict" is a conflict among the *peers* of + // a given node we're trying to place, but there actually is no current + // node. Eg, + // root -> (a, b) + // a -> PEER(c) + // b -> PEER(d) + // d -> PEER(c@2) + // We place (a), and get a peer of (c) along with it. + // then we try to place (b), and get CONFLICT in the check, because + // of the conflicting peer from (b)->(d)->(c@2). In that case, we + // should treat (b) and (d) as OK, and place them in the last place + // where they did not themselves conflict, and skip c@2 if conflict + // is ok by virtue of being forced or not ours and not strict. + if (cpd.canPlaceSelf !== CONFLICT) + canPlaceSelf = cpd + + // we found a place this can go, along with all its peer friends. + // we break when we get the first conflict + if (cpd.canPlace !== CONFLICT) + canPlace = cpd + else + break + + // if it's a load failure, just plop it in the first place attempted, + // since we're going to crash the build or prune it out anyway. + // but, this will frequently NOT be a successful canPlace, because + // it'll have no version or other information. + if (dep.errors.length) + break + + // nest packages like npm v1 and v2 + // very disk-inefficient + if (legacyBundling) + break + + // when installing globally, or just in global style, we never place + // deps above the first level. + if (globalStyle) { + const rp = target.resolveParent + if (rp && rp.isProjectRoot) + break + } + } + + Object.assign(this, { + canPlace, + canPlaceSelf, + }) + this.current = edge.to + + // if we can't find a target, that means that the last place checked, + // and all the places before it, had a conflict. + if (!canPlace) { + // if not forced, or it's our dep, or strictPeerDeps is set, then + // this is an ERESOLVE error. + if (!this.conflictOk) + return this.failPeerConflict() + + // ok! we're gonna allow the conflict, but we should still warn + // if we have a current, then we treat CONFLICT as a KEEP. + // otherwise, we just skip it. Only warn on the one that actually + // could not be placed somewhere. + if (!canPlaceSelf) { + this.warnPeerConflict() + return + } + + this.canPlace = canPlaceSelf + } + + // now we have a target, a tree of CanPlaceDep results for the peer group, + // and we are ready to go + this.placeInTree() + } + + placeInTree () { + const { + dep, + canPlace, + edge, + } = this + + /* istanbul ignore next */ + if (!canPlace) { + debug(() => { + throw new Error('canPlace not set, but trying to place in tree') + }) + return + } + + const { target } = canPlace + + log.silly( + 'placeDep', + target.location || 'ROOT', + `${dep.name}@${dep.version}`, + canPlace.description, + `for: ${this.edge.from.package._id || this.edge.from.location}`, + `want: ${edge.spec || '*'}` + ) + + const placementType = canPlace.canPlace === CONFLICT + ? canPlace.canPlaceSelf + : canPlace.canPlace + + // if we're placing in the tree with --force, we can get here even though + // it's a conflict. Treat it as a KEEP, but warn and move on. + if (placementType === KEEP) { + // this was an overridden peer dep + if (edge.peer && !edge.valid) + this.warnPeerConflict() + + // if we get a KEEP in a update scenario, then we MAY have something + // already duplicating this unnecessarily! For example: + // ``` + // root (dep: y@1) + // +-- x (dep: y@1.1) + // | +-- y@1.1.0 (replacing with 1.1.2, got KEEP at the root) + // +-- y@1.1.2 (updated already from 1.0.0) + // ``` + // Now say we do `reify({update:['y']})`, and the latest version is + // 1.1.2, which we now have in the root. We'll try to place y@1.1.2 + // first in x, then in the root, ending with KEEP, because we already + // have it. In that case, we ought to REMOVE the nm/x/nm/y node, because + // it is an unnecessary duplicate. + this.pruneDedupable(target) + return + } + + // XXX if we are replacing SOME of a peer entry group, we will need to + // remove any that are not being replaced and will now be invalid, and + // re-evaluate them deeper into the tree. + + const virtualRoot = dep.parent + this.placed = new dep.constructor({ + name: dep.name, + pkg: dep.package, + resolved: dep.resolved, + integrity: dep.integrity, + legacyPeerDeps: this.legacyPeerDeps, + error: dep.errors[0], + ...(dep.isLink ? { target: dep.target, realpath: dep.target.path } : {}), + }) + + this.oldDep = target.children.get(this.name) + if (this.oldDep) + this.replaceOldDep() + else + this.placed.parent = target + + // if it's an overridden peer dep, warn about it + if (edge.peer && !this.placed.satisfies(edge)) + this.warnPeerConflict() + + // If the edge is not an error, then we're updating something, and + // MAY end up putting a better/identical node further up the tree in + // a way that causes an unnecessary duplication. If so, remove the + // now-unnecessary node. + if (edge.valid && edge.to && edge.to !== this.placed) + this.pruneDedupable(edge.to, false) + + // in case we just made some duplicates that can be removed, + // prune anything deeper in the tree that can be replaced by this + for (const node of target.root.inventory.query('name', this.name)) { + if (node.isDescendantOf(target) && !node.isTop) { + this.pruneDedupable(node, false) + // only walk the direct children of the ones we kept + if (node.root === target.root) { + for (const kid of node.children.values()) + this.pruneDedupable(kid, false) + } + } + } + + // also place its unmet or invalid peer deps at this location + // loop through any peer deps from the thing we just placed, and place + // those ones as well. it's safe to do this with the virtual nodes, + // because we're copying rather than moving them out of the virtual root, + // otherwise they'd be gone and the peer set would change throughout + // this loop. + for (const peerEdge of this.placed.edgesOut.values()) { + if (peerEdge.valid || !peerEdge.peer || peerEdge.overridden) + continue + + const peer = virtualRoot.children.get(peerEdge.name) + + // Note: if the virtualRoot *doesn't* have the peer, then that means + // it's an optional peer dep. If it's not being properly met (ie, + // peerEdge.valid is false), then this is likely heading for an + // ERESOLVE error, unless it can walk further up the tree. + if (!peer) + continue + + // overridden peerEdge, just accept what's there already + if (!peer.satisfies(peerEdge)) + continue + + this.children.push(new PlaceDep({ + parent: this, + dep: peer, + node: this.placed, + edge: peerEdge, + })) + } + } + + replaceOldDep () { + // XXX handle replacing an entire peer group? + // what about cases where we need to push some other peer groups deeper + // into the tree? all the tree updating should be done here, and track + // all the things that we add and remove, so that we can know what + // to re-evaluate. + + // if we're replacing, we should also remove any nodes for edges that + // are now invalid, and where this (or its deps) is the only dependent, + // and also recurse on that pruning. Otherwise leaving that dep node + // around can result in spurious conflicts pushing nodes deeper into + // the tree than needed in the case of cycles that will be removed + // later anyway. + const oldDeps = [] + for (const [name, edge] of this.oldDep.edgesOut.entries()) { + if (!this.placed.edgesOut.has(name) && edge.to) + oldDeps.push(...gatherDepSet([edge.to], e => e.to !== edge.to)) + } + this.placed.replace(this.oldDep) + this.pruneForReplacement(this.placed, oldDeps) + } + + pruneForReplacement (node, oldDeps) { + // gather up all the now-invalid/extraneous edgesOut, as long as they are + // only depended upon by the old node/deps + const invalidDeps = new Set([...node.edgesOut.values()] + .filter(e => e.to && !e.valid).map(e => e.to)) + for (const dep of oldDeps) { + const set = gatherDepSet([dep], e => e.to !== dep && e.valid) + for (const dep of set) + invalidDeps.add(dep) + } + + // ignore dependency edges from the node being replaced, but + // otherwise filter the set down to just the set with no + // dependencies from outside the set, except the node in question. + const deps = gatherDepSet(invalidDeps, edge => + edge.from !== node && edge.to !== node && edge.valid) + + // now just delete whatever's left, because it's junk + for (const dep of deps) + dep.root = null + } + + // prune all the nodes in a branch of the tree that can be safely removed + // This is only the most basic duplication detection; it finds if there + // is another satisfying node further up the tree, and if so, dedupes. + // Even in legacyBundling mode, we do this amount of deduplication. + pruneDedupable (node, descend = true) { + if (node.canDedupe(this.preferDedupe)) { + // gather up all deps that have no valid edges in from outside + // the dep set, except for this node we're deduping, so that we + // also prune deps that would be made extraneous. + const deps = gatherDepSet([node], e => e.to !== node && e.valid) + for (const node of deps) + node.root = null + return + } + if (descend) { + // sort these so that they're deterministically ordered + // otherwise, resulting tree shape is dependent on the order + // in which they happened to be resolved. + const nodeSort = (a, b) => a.location.localeCompare(b.location, 'en') + + const children = [...node.children.values()].sort(nodeSort) + for (const child of children) + this.pruneDedupable(child) + const fsChildren = [...node.fsChildren].sort(nodeSort) + for (const topNode of fsChildren) { + const children = [...topNode.children.values()].sort(nodeSort) + for (const child of children) + this.pruneDedupable(child) + } + } + } + + get conflictOk () { + return this.force || (!this.isMine && !this.strictPeerDeps) + } + + get isMine () { + const { edge } = this.top + const { from: node } = edge + + if (node.isWorkspace || node.isProjectRoot) + return true + + if (!edge.peer) + return false + + // re-entry case. check if any non-peer edges come from the project, + // or any entryEdges on peer groups are from the root. + let hasPeerEdges = false + for (const edge of node.edgesIn) { + if (edge.peer) { + hasPeerEdges = true + continue + } + if (edge.from.isWorkspace || edge.from.isProjectRoot) + return true + } + if (hasPeerEdges) { + for (const edge of peerEntrySets(node).keys()) { + if (edge.from.isWorkspace || edge.from.isProjectRoot) + return true + } + } + + return false + } + + warnPeerConflict () { + this.edge.overridden = true + const expl = this.explainPeerConflict() + log.warn('ERESOLVE', 'overriding peer dependency', expl) + } + + failPeerConflict () { + const expl = this.explainPeerConflict() + throw Object.assign(new Error('could not resolve'), expl) + } + + explainPeerConflict () { + const { edge, dep } = this.top + const { from: node } = edge + const curNode = node.resolve(edge.name) + + const expl = { + code: 'ERESOLVE', + edge: edge.explain(), + dep: dep.explain(edge), + } + + if (this.parent) { + // this is the conflicted peer + expl.current = curNode && curNode.explain(edge) + expl.peerConflict = this.current && this.current.explain(this.edge) + } else { + expl.current = curNode && curNode.explain() + if (this.canPlaceSelf && this.canPlaceSelf.canPlaceSelf !== CONFLICT) { + // failed while checking for a child dep + const cps = this.canPlaceSelf + for (const peer of cps.conflictChildren) { + if (peer.current) { + expl.peerConflict = { + current: peer.current.explain(), + peer: peer.dep.explain(peer.edge), + } + break + } + } + } else { + expl.peerConflict = { + current: this.current && this.current.explain(), + peer: this.dep.explain(this.edge), + } + } + } + + const { + strictPeerDeps, + force, + isMine, + } = this + Object.assign(expl, { + strictPeerDeps, + force, + isMine, + }) + + // XXX decorate more with this.canPlace and this.canPlaceSelf, + // this.checks, this.children, walk over conflicted peers, etc. + return expl + } + + getStartNode () { + // if we are a peer, then we MUST be at least as shallow as the + // peer dependent + const from = this.parent ? this.parent.getStartNode() : this.edge.from + return deepestNestingTarget(from, this.name) + } + + get top () { + return this.parent ? this.parent.top : this + } + + isVulnerable (node) { + return this.auditReport && this.auditReport.isVulnerable(node) + } + + get allChildren () { + const set = new Set(this.children) + for (const child of set) { + for (const grandchild of child.children) + set.add(grandchild) + } + return [...set] + } +} + +module.exports = PlaceDep diff --git a/node_modules/@npmcli/arborist/lib/printable.js b/node_modules/@npmcli/arborist/lib/printable.js index ce764071dc62a..4aa2fffd104b4 100644 --- a/node_modules/@npmcli/arborist/lib/printable.js +++ b/node_modules/@npmcli/arborist/lib/printable.js @@ -31,6 +31,10 @@ class ArboristNode { this.bundled = true if (tree.inDepBundle) this.bundler = tree.getBundler().location + if (tree.isProjectRoot) + this.isProjectRoot = true + if (tree.isWorkspace) + this.isWorkspace = true const bd = tree.package && tree.package.bundleDependencies if (bd && bd.length) this.bundleDependencies = bd @@ -107,6 +111,8 @@ class Edge { this.spec = edge.spec || '*' if (edge.error) this.error = edge.error + if (edge.overridden) + this.overridden = edge.overridden } } @@ -122,6 +128,8 @@ class EdgeOut extends Edge { this.to ? ' -> ' + this.to : '' }${ this.error ? ' ' + this.error : '' + }${ + this.overridden ? ' overridden' : '' } }` } } @@ -136,6 +144,8 @@ class EdgeIn extends Edge { [util.inspect.custom] () { return `{ ${this.from || '""'} ${this.type} ${this.name}@${this.spec}${ this.error ? ' ' + this.error : '' + }${ + this.overridden ? ' overridden' : '' } }` } } diff --git a/node_modules/@npmcli/arborist/lib/shrinkwrap.js b/node_modules/@npmcli/arborist/lib/shrinkwrap.js index 3b2cf0bde1036..ebbe004de72d6 100644 --- a/node_modules/@npmcli/arborist/lib/shrinkwrap.js +++ b/node_modules/@npmcli/arborist/lib/shrinkwrap.js @@ -183,8 +183,10 @@ const assertNoNewer = async (path, data, lockTime, dir = path, seen = null) => { await assertNoNewer(path, data, lockTime, child, seen) else if (ent.isSymbolicLink()) { const target = resolve(parent, await readlink(child)) - const tstat = await stat(target).catch(() => null) + const tstat = await stat(target).catch( + /* istanbul ignore next - windows */ () => null) seen.add(relpath(path, child)) + /* istanbul ignore next - windows cannot do this */ if (tstat && tstat.isDirectory() && !seen.has(relpath(path, target))) await assertNoNewer(path, data, lockTime, target, seen) } diff --git a/node_modules/@npmcli/arborist/package.json b/node_modules/@npmcli/arborist/package.json index c45a61086ea5e..56046eaa5f357 100644 --- a/node_modules/@npmcli/arborist/package.json +++ b/node_modules/@npmcli/arborist/package.json @@ -1,6 +1,6 @@ { "name": "@npmcli/arborist", - "version": "2.7.1", + "version": "2.8.0", "description": "Manage node_modules trees", "dependencies": { "@npmcli/installed-package-contents": "^1.0.7", @@ -19,10 +19,10 @@ "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "npm-install-checks": "^4.0.0", - "npm-package-arg": "^8.1.0", + "npm-package-arg": "^8.1.5", "npm-pick-manifest": "^6.1.0", "npm-registry-fetch": "^11.0.0", - "pacote": "^11.2.6", + "pacote": "^11.3.5", "parse-conflict-json": "^1.1.1", "proc-log": "^1.0.0", "promise-all-reject-late": "^1.0.0", @@ -53,7 +53,7 @@ "test-only": "tap", "posttest": "npm run lint", "snap": "tap", - "postsnap": "npm run lint", + "postsnap": "npm run lintfix", "test-proxy": "ARBORIST_TEST_PROXY=1 tap --snapshot", "preversion": "npm test", "postversion": "npm publish", diff --git a/package-lock.json b/package-lock.json index a6cd1e55586b7..c0ea803e14656 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,7 +83,7 @@ "packages/*" ], "dependencies": { - "@npmcli/arborist": "^2.7.1", + "@npmcli/arborist": "^2.8.0", "@npmcli/ci-detect": "^1.2.0", "@npmcli/config": "^2.2.0", "@npmcli/package-json": "^1.0.1", @@ -755,9 +755,9 @@ } }, "node_modules/@npmcli/arborist": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@npmcli/arborist/-/arborist-2.7.1.tgz", - "integrity": "sha512-EGDHJs6dna/52BrStr/6aaRcMLrYxGbSjT4V3JzvoTBY9/w5i2+1KNepmsG80CAsGADdo6nuNnFwb7sDRm8ZAw==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@npmcli/arborist/-/arborist-2.8.0.tgz", + "integrity": "sha512-R9rTyak1rGdmVTyiU14dgBb+qMllY3B6I8hp7FB4xXsU9dJDrYZJR8I+191CMo5Y1941jTDCtNcXXW9TldPEFQ==", "inBundle": true, "dependencies": { "@npmcli/installed-package-contents": "^1.0.7", @@ -776,10 +776,10 @@ "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "npm-install-checks": "^4.0.0", - "npm-package-arg": "^8.1.0", + "npm-package-arg": "^8.1.5", "npm-pick-manifest": "^6.1.0", "npm-registry-fetch": "^11.0.0", - "pacote": "^11.2.6", + "pacote": "^11.3.5", "parse-conflict-json": "^1.1.1", "proc-log": "^1.0.0", "promise-all-reject-late": "^1.0.0", @@ -11002,9 +11002,9 @@ "dev": true }, "@npmcli/arborist": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@npmcli/arborist/-/arborist-2.7.1.tgz", - "integrity": "sha512-EGDHJs6dna/52BrStr/6aaRcMLrYxGbSjT4V3JzvoTBY9/w5i2+1KNepmsG80CAsGADdo6nuNnFwb7sDRm8ZAw==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@npmcli/arborist/-/arborist-2.8.0.tgz", + "integrity": "sha512-R9rTyak1rGdmVTyiU14dgBb+qMllY3B6I8hp7FB4xXsU9dJDrYZJR8I+191CMo5Y1941jTDCtNcXXW9TldPEFQ==", "requires": { "@npmcli/installed-package-contents": "^1.0.7", "@npmcli/map-workspaces": "^1.0.2", @@ -11022,10 +11022,10 @@ "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "npm-install-checks": "^4.0.0", - "npm-package-arg": "^8.1.0", + "npm-package-arg": "^8.1.5", "npm-pick-manifest": "^6.1.0", "npm-registry-fetch": "^11.0.0", - "pacote": "^11.2.6", + "pacote": "^11.3.5", "parse-conflict-json": "^1.1.1", "proc-log": "^1.0.0", "promise-all-reject-late": "^1.0.0", diff --git a/package.json b/package.json index d417e19677e6a..706b9b59b7c83 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "./package.json": "./package.json" }, "dependencies": { - "@npmcli/arborist": "^2.7.1", + "@npmcli/arborist": "^2.8.0", "@npmcli/ci-detect": "^1.2.0", "@npmcli/config": "^2.2.0", "@npmcli/package-json": "^1.0.1",