Accept vendored packages and subpackages under their host's identity, recording those reached

doc/file-formats.md

@@ -135,9 +135,27 @@ attested.
   import out of `node_modules` is `npm`. Workspace/top-level buckets (`sources`)
   are first-party and omit `ecosystem`. Artifacts predating this field lack it and
   still load.
-- A record carries no `repo`: a dependency's repository is metadata, which a
-  bundle carries (see `modules` under `stasis.code.br`) and a lockfile neither
-  writes nor reads.
+- A record carries no `repo`, `vendored` or `subpackages`: a dependency's
+  repository, the packages copied into it and its subpath entry points are
+  metadata, which a bundle carries (see `modules` under `stasis.code.br`) and a
+  lockfile neither writes nor reads.
+- A file under a dependency is that dependency's, at the directory its
+  `package.json` is in, and a `package.json` nested inside it, one nothing
+  expects, must not name another package or version: where it came from is the
+  question the check asks. Each answer accepted instead is recorded in the
+  bundle:
+  - A package that keeps copies of others in a directory, each beside a
+    `package.json` naming it, listed in `@exodus/stasis-core`'s
+    `bundle-util.js` (Next.js, `dist/compiled`): a file there is still the
+    host's, and a bundle lists the copy it is in (`vendored`).
+  - A subpackage, a subpath entry point with a `package.json` of its own named
+    in the package's namespace for the very directory it is in
+    (`@hookform/resolvers/zod` at `zod/`, as microbundle lays them out): its
+    files are the package's, its `version` (a placeholder, `1.0.0`) its own,
+    and a bundle lists it (`subpackages`). One named in the namespace for
+    another directory is still held to the package's version.
+  - A stale copy known upstream, `@redis/client`'s `dist/package.json`, is
+    let through, listed in `state.js`.
 - `imports` records observed resolutions (conditions → parent file → specifier →
   resolved project-relative path). Under `lock = frozen`, disk resolutions are
   checked: a divergence from the recorded target is fatal (catching a specifier
@@ -294,6 +312,60 @@ SIGINT shutdown, a CLI reporting failures) still persists what it cleanly captur
   audit --repo-advisories` asks it rather than looking one up, and for a record
   with none, the one its bundled `package.json` names by the same rules (see the
   `stasis` README).
+- A dependency record may carry `vendored`, after `repo`: the packages copied
+  into the dependency (Next.js compiles ~140 into `dist/compiled`) whose files
+  the bundle carries, by the directory each one's own `package.json` is in, with
+  that `package.json`'s `name` and, where it gives one, `version`:
+
+  ```json
+  "node_modules/next": {
+    "name": "next",
+    "version": "16.4.0",
+    "ecosystem": "npm",
+    "vendored": {
+      "dist/compiled/react-is": { "name": "react-is", "version": "19.3.0-canary-278794d7-20261002" },
+      "dist/compiled/ua-parser-js": { "name": "ua-parser-js" }
+    },
+    "files": { "dist/compiled/react-is/index.js": "…", "dist/compiled/ua-parser-js/ua-parser.js": "…" }
+  }
+  ```
+
+  A file is the copy's whose directory is the deepest one listed above it. Only
+  a copy a bundled file is in is listed: of Next.js's ~140, the ones the run or
+  scan reached, and each bundle (a split layout's code half and resources half
+  apart) lists those of the files it carries itself. So every directory listed
+  holds one of the record's files: parse rejects one that holds none, as it does
+  one through a `node_modules`, a `vendored` without a `name`, one with another
+  key, and one on first-party code. The directory names what the host calls the
+  copy, its `package.json` what the copy calls itself (Next.js's `loader-utils2`
+  is `loader-utils`, its React `react-builtin`), and neither is checked against
+  a registry. Like `repo`, it is metadata: never written to the lockfile, never
+  attested, and held to nothing; a run fills in a directory a record lists none
+  for, and a merge takes the union, the existing side's where both list one.
+  `stasis sbom` lists each copy as a package contained in its host (see
+  `doc/sbom.md`).
+- A dependency record may carry `subpackages`, after `vendored`: its subpath
+  entry points with a `package.json` of their own whose files the bundle
+  carries, by directory, each with that `package.json`'s `name`, which must be
+  the package's name for the directory (`@hookform/resolvers/zod` at `zod/`),
+  and, where it gives one, `version`:
+
+  ```json
+  "node_modules/@hookform/resolvers": {
+    "name": "@hookform/resolvers",
+    "version": "5.9.1",
+    "ecosystem": "npm",
+    "subpackages": { "zod": { "name": "@hookform/resolvers/zod", "version": "1.0.0" } },
+    "files": { "zod/dist/zod.js": "…" }
+  }
+  ```
+
+  A file is in a subpackage when that subpackage's `package.json` is the
+  nearest one with a name above it, past `{"type":"module"}` markers. The rest is as for `vendored`: only those a bundled
+  file is in are listed, each half of a split layout its own, parse rejects an
+  entry holding none of the record's files (or misnamed, or on first-party
+  code), and it is metadata, held to nothing and merged by union. They are the
+  package's own code, so `stasis sbom` lists none.
 - `formats`: project-relative path → format, same vocabulary as the lockfile's
   `formats`. May be missing per file for code whose format Node infers. TypeScript
   sources are stored verbatim (types intact); Node strips types at load time.

doc/sbom.md

@@ -36,6 +36,21 @@ Packages are deduplicated by ecosystem + name + version across all inputs and
 sorted by name then version. Inventory is at the **package** level, not the file
 level. Legacy `version: 0` bundles record no `name`/`version`, so they contribute nothing.
 
+- **Vendored packages**, the copies a dependency keeps of others that a bundle
+  lists in its record's `vendored` (Next.js's `dist/compiled/<dir>`, see
+  `doc/file-formats.md`), are contained in it: nested in its `components` in
+  CycloneDX, each `bom-ref` the host's with the directory as a purl subpath and
+  the directory as its `evidence.occurrences` location; in SPDX, packages after
+  the rest, `SPDXRef-Package-<n>-Vendored-<i>`, which the host `CONTAINS`, the
+  directory as `packageFileName`. Only those a bundled file is in are listed,
+  each under the name its own `package.json` gives, with the `version` it gives
+  if any, and a purl only beside a version: Next.js's copies mostly record
+  none, and a name without one names no release, nor always a registry package
+  (Next.js calls its React `react-builtin`). Across inputs, a host lists the
+  union of its copies. A lockfile records none. A dependency's `subpackages`
+  (its own subpath entry points, `@hookform/resolvers/zod`) are its own code,
+  not packages: none is listed.
+
 ## Package URLs
 
 Each component carries a [purl](https://github.com/package-url/purl-spec), built

stasis-core/src/add.js

@@ -8,7 +8,7 @@ import { Lockfile } from './lockfile.js'
 import { brotliOptions } from './brotli.js'
 import { detectRepo, findPackageMetadata, normalizeEntries, packageType, readJson } from './bundle-util.js'
 import { canonicalizePath, sha512integrity } from './state-util.js'
-import { assertRealPathWithinBase, classifyFormat, hasNodeModulesSegment, isAutoExcludedDir, isAutoExcludedFile, isBinaryPlist, isBrotliQuality, isExecutableMode, moduleFileKey, moduleInfo, parseResourcesOption, pathExt, sortPaths, splitNodeModulesPath, toPosix } from './util.js'
+import { assertRealPathWithinBase, classifyFormat, hasNodeModulesSegment, isAutoExcludedDir, isAutoExcludedFile, isBinaryPlist, isBrotliQuality, isExecutableMode, moduleFileKey, moduleInfo, parseResourcesOption, pathExt, sortPaths, splitNodeModulesPath, toPosix, withMetadataOf } from './util.js'
 
 const CONFIG_FILE = 'stasis.config.json'
 const LOCK_FILE = 'stasis.lock.json'
@@ -61,9 +61,11 @@ function assembleBundle(baseDir, files, workspaceName, workspaceVersion, repo) {
   const modules = new Map()
   const formats = new Map()
   const executable = new Set()
-  // `identity`'s name, version, ecosystem and repo (moduleInfo takes those alone).
+  // `identity`'s name, version, ecosystem, repo and vendored (moduleInfo takes those alone); a later
+  // file's adds the vendored package it is in.
   const bucketFiles = (dir, identity) => {
-    if (!modules.has(dir)) modules.set(dir, moduleInfo({ ...identity, files: Object.create(null) }))
+    const bucket = modules.get(dir)
+    modules.set(dir, bucket === undefined ? moduleInfo({ ...identity, files: Object.create(null) }) : withMetadataOf(bucket, identity))
     return modules.get(dir).files
   }
   // findPackageMetadata's result depends only on the file's directory.

stasis-core/src/artifact-util.js

@@ -110,16 +110,110 @@ const normalizeModuleRepo = (dir, { ecosystem, repo }, what) => {
   return normalized
 }
 
-// A module bucket record in canonical key order; `ecosystem` and `repo` are omitted (not undefined) when absent.
-export const moduleInfo = ({ name, version, ecosystem, repo, files }) =>
-  ({ name, version, ...(ecosystem === undefined ? {} : { ecosystem }), ...(repo === undefined ? {} : { repo }), files })
+// A package name or version: characters some ecosystem uses there (npm's legacy `~'!()*` too), so not space or `"#$%&,:;<=>?[\]^`{|}`.
+export const isPackageString = (v) => typeof v === 'string' && /^[\w.+@/~'!()*-]+$/u.test(v)
+export const PACKAGE_BLOCK = { name: isPackageString, version: isPackageString }
+
+// A directory key of a record's `vendored` or `subpackages`: an in-bucket path in canonical form, through
+// no `node_modules` (a package installed there is a bucket of its own).
+const isBucketDir = (dir) => typeof dir === 'string' && dir !== '' && !dir.includes('\\') &&
+  dir.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..' && segment !== 'node_modules')
+
+// Whether a package.json naming `manifestName`, at `dir` (package-relative, POSIX) inside the package named
+// `hostName`, is one of its subpackages: a subpath entry point with a package.json of its own, named in
+// the package's namespace for the very directory it is in (`@hookform/resolvers/zod` at zod/), as
+// microbundle lays them out. It is the package's own code, and its `version`, a placeholder (1.0.0), its
+// own: held to nothing.
+export const isSubpackage = (manifestName, hostName, dir) =>
+  typeof manifestName === 'string' && isBucketDir(dir) && manifestName === `${hostName}/${dir}`
+
+// A record's per-directory metadata, each entry the `name` (required) and `version` of the package.json in
+// that directory: `vendored`, the packages copied into a dependency (any name), and `subpackages`, its own
+// subpath entry points (isSubpackage).
+const DIR_METADATA = {
+  vendored: () => true,
+  subpackages: (entry, name, sub) => isSubpackage(entry.name, name, sub),
+}
+
+// The directory of `entries` a bucket file `rel` is in: the deepest one listed that holds it.
+const entryDirOf = (entries, rel) => {
+  for (let slash = rel.lastIndexOf('/'); slash > 0; slash = rel.lastIndexOf('/', slash - 1)) {
+    if (Object.hasOwn(entries, rel.slice(0, slash))) return rel.slice(0, slash)
+  }
+  return undefined
+}
+
+// The entries of a `vendored` or `subpackages` that hold one of `files`, each file counted for the deepest
+// entry holding it; path-sorted and frozen, undefined if none. An artifact lists those it carries a file of.
+function carriedEntries(entries, files) {
+  if (entries === undefined) return undefined
+  const carried = new Set()
+  for (const rel of Object.keys(files)) {
+    const dir = entryDirOf(entries, rel)
+    if (dir !== undefined) carried.add(dir)
+  }
+  return carried.size === 0 ? undefined : Object.freeze(fromEntries([...carried].toSorted(sortPaths).map((dir) => [dir, entries[dir]])))
+}
+
+// A dependency's `field` of DIR_METADATA, validated: each entry holds one of the record's `files`. First-party
+// code carries none.
+const normalizeDirMetadata = (field, dir, { name, ecosystem, files, [field]: entries }, what) => {
+  if (entries === undefined) return undefined
+  if (dependencyEcosystem(dir, ecosystem) === undefined) assert(false, `${what}: '${dir}' is no dependency's bucket, and carries no ${field}`)
+  if (!isPlainObject(entries)) assert(false, `${what} module '${dir}' ${field} must be an object`)
+  const normalized = fromEntries(Object.entries(entries).map(([sub, block]) => {
+    if (!isBucketDir(sub)) assert(false, `${what} module '${dir}' ${field}: invalid directory ${JSON.stringify(sub)}`)
+    const entry = normalizeBlock(block, PACKAGE_BLOCK, `${what} module '${dir}' ${field} '${sub}'`)
+    if (entry?.name === undefined) assert(false, `${what} module '${dir}' ${field} '${sub}' has no name`)
+    if (!DIR_METADATA[field](entry, name, sub)) assert(false, `${what} module '${dir}' ${field} '${sub}' is named '${entry.name}', not '${name}/${sub}'`)
+    return [sub, entry]
+  }))
+  const carried = carriedEntries(normalized, files)
+  for (const sub of Object.keys(normalized)) {
+    if (!Object.hasOwn(carried ?? {}, sub)) assert(false, `${what} module '${dir}' ${field} '${sub}' holds none of its files`)
+  }
+  return carried
+}
+
+// A module bucket record in canonical key order; `ecosystem`, `repo`, `vendored` and `subpackages` are omitted (not undefined) when absent.
+export const moduleInfo = ({ name, version, ecosystem, repo, vendored, subpackages, files }) => ({
+  name, version,
+  ...(ecosystem === undefined ? {} : { ecosystem }),
+  ...(repo === undefined ? {} : { repo }),
+  ...(vendored === undefined ? {} : { vendored }),
+  ...(subpackages === undefined ? {} : { subpackages }),
+  files,
+})
+
+// `record` with the metadata `other` (of the same bucket) has and it lacks: `repo` where it records none,
+// the `vendored` and `subpackages` entries of directories it lists none for. Metadata is held to nothing,
+// so where both record one, `record`'s stands. `record` itself when that adds nothing.
+export function withMetadataOf(record, other) {
+  const repo = record.repo ?? other.repo
+  const fields = Object.keys(DIR_METADATA).filter((field) =>
+    other[field] !== undefined && Object.keys(other[field]).some((sub) => !Object.hasOwn(record[field] ?? {}, sub)))
+  if (repo === record.repo && fields.length === 0) return record
+  const merged = fromEntries(fields.map((field) => [field, Object.freeze(fromEntries([...Object.entries(other[field]), ...Object.entries(record[field] ?? {})]))]))
+  return moduleInfo({ ...record, repo, ...merged })
+}
+
+// A record's `vendored` and `subpackages` as `what` writes them: the entries holding one of `files`, validated.
+const carriedDirMetadata = (dir, info, files, what) => fromEntries(Object.keys(DIR_METADATA).map((field) =>
+  [field, normalizeDirMetadata(field, dir, { ...info, files, [field]: carriedEntries(info[field], files) }, what)]))
 
 // A parsed bucket `dir` of a `what` artifact: an absent version has one spelling (a literal null folds
-// into undefined so identity comparisons and JSON round-trips can't split on it), `repo` is validated,
-// and `files` is null-prototype.
-export function normalizeModule({ name, version, ecosystem, repo, files }, dir, what) {
+// into undefined so identity comparisons and JSON round-trips can't split on it), `repo`, `vendored` and
+// `subpackages` are validated, and `files` is null-prototype.
+export function normalizeModule({ name, version, ecosystem, repo, vendored, subpackages, files }, dir, what) {
   assert(ecosystem === undefined || typeof ecosystem === 'string')
-  return moduleInfo({ name, version: version ?? undefined, ecosystem, repo: normalizeModuleRepo(dir, { ecosystem, repo }, what), files: fromEntries(Object.entries(files)) })
+  files = fromEntries(Object.entries(files))
+  return moduleInfo({
+    name, version: version ?? undefined, ecosystem,
+    repo: normalizeModuleRepo(dir, { ecosystem, repo }, what),
+    vendored: normalizeDirMetadata('vendored', dir, { name, ecosystem, files, vendored }, what),
+    subpackages: normalizeDirMetadata('subpackages', dir, { name, ecosystem, files, subpackages }, what),
+    files,
+  })
 }
 
 // The keys a module map records -- the set an artifact's `executable` must be a subset of. `scope` MUST
@@ -360,16 +454,22 @@ export function serializeImports(imports) {
 }
 
 // A module map as the serialized `modules` (node_modules buckets) and `sources` (the rest) objects
-// of a `what` artifact, buckets and files path-sorted so the bytes are canonical. `repo: false`
-// leaves each record's `repo` out: metadata, which a lockfile never carries.
-export function groupModules(modules, { skipEmpty = false, repo = true, what } = {}) {
+// of a `what` artifact, buckets and files path-sorted so the bytes are canonical. Each record's
+// `vendored` and `subpackages` keep the entries holding one of the files it carries. `metadata: false`
+// leaves each record's `repo`, `vendored` and `subpackages` out: metadata, which a lockfile never carries.
+export function groupModules(modules, { skipEmpty = false, metadata = true, what } = {}) {
   const grouped = { modules: [], sources: [] }
   for (const [dir, info] of modules) {
     if (skipEmpty && Object.keys(info.files).length === 0) continue
     const inNodeModules = hasNodeModulesSegment(dir)
     if (inNodeModules) assert(info.name && info.version && info.files)
     const files = fromEntries(Object.entries(info.files).toSorted(byPath))
-    grouped[inNodeModules ? 'modules' : 'sources'].push([dir, moduleInfo({ ...info, repo: repo ? normalizeModuleRepo(dir, info, what) : undefined, files })])
+    grouped[inNodeModules ? 'modules' : 'sources'].push([dir, moduleInfo({
+      ...info,
+      repo: metadata ? normalizeModuleRepo(dir, info, what) : undefined,
+      ...(metadata ? carriedDirMetadata(dir, info, files, what) : { vendored: undefined, subpackages: undefined }),
+      files,
+    })])
   }
   return { modules: fromEntries(grouped.modules.toSorted(byPath)), sources: fromEntries(grouped.sources.toSorted(byPath)) }
 }
@@ -498,8 +598,9 @@ export function mergeModuleMaps(a, b, label) {
           `package without one; regenerate it (bundle=replace / lock=replace)`))
       assert(existing.ecosystem === info.ecosystem,
         `${label}: module '${dir}' ecosystem mismatch ('${existing.ecosystem ?? '(none)'}' vs '${info.ecosystem ?? '(none)'}')`)
-      // `repo` is metadata, held to nothing: either side's, where only one records it, else `a`'s.
-      if (existing.repo === undefined && info.repo !== undefined) out.set(dir, moduleInfo({ ...existing, repo: info.repo }))
+      // `repo`, `vendored` and `subpackages` are metadata, held to nothing: either side's, where only one records it, else `a`'s.
+      const merged = withMetadataOf(existing, info)
+      if (merged !== existing) out.set(dir, merged)
       for (const [rel, value] of Object.entries(info.files)) {
         if (Object.hasOwn(existing.files, rel)) {
           assert(existing.files[rel] === value, `${label}: content mismatch for '${moduleFileKey(dir, rel)}'`)