[WIP] bundler: tree-shaking

tools/src/main/js/bundler/bundle-schemas.js

@@ -23,6 +23,13 @@ function isAbsoluteUri(ref) {
     return /^[a-z][a-z0-9+.-]*:/i.test(ref);
 }
 
+/**
+ * Unescape a single JSON Pointer (RFC6901) reference token.
+ */
+function unescapeJsonPointerToken(token) {
+    return token.replace(/~1/g, '/').replace(/~0/g, '~');
+}
+
 /**
  * Resolve a JSON Pointer (RFC6901) against an object. Returns { ok: boolean, value?: any, error?: string }
  */
@@ -36,7 +43,7 @@ function resolveJsonPointer(root, pointer) {
     if (!p.startsWith('/')) {
         return { ok: false, error: `Pointer must start with '/': ${pointer}` };
     }
-    const parts = p.split('/').slice(1).map(seg => seg.replace(/~1/g, '/').replace(/~0/g, '~'));
+    const parts = p.split('/').slice(1).map(unescapeJsonPointerToken);
     let current = root;
     for (const key of parts) {
         if (!isObject(current) && !Array.isArray(current)) {
@@ -74,7 +81,7 @@ function collectRefKeywords(obj, keys, predicate, pathStack = []) {
     return result;
 }
 
-const FILE_REF_RE = /^(.+\.schema\.json)(#.*)?$/;
+const FILE_REF_RE = Object.freeze(/^(.+\.schema\.json)(#.*)?$/);
 
 /**
  * make schema name from schema file
@@ -191,6 +198,136 @@ function stripTopLevelKeys(obj, keysToRemove = []) {
     return clone;
 }
 
+const REF_KEYWORDS = Object.freeze(['$ref', '$dynamicRef', '$recursiveRef']);
+const DYNAMIC_ANCHOR_KEYWORDS = Object.freeze(['$dynamicAnchor', '$recursiveAnchor']);
+
+/**
+ * Whether a sub-tree contains a dynamic anchor. Such schemas may be targeted by
+ * $dynamicRef/$recursiveRef via anchor name instead of JSON Pointer, so they
+ * cannot be shaken away safely.
+ */
+function containsDynamicAnchor(obj) {
+    if (!isObject(obj)) return false;
+    if (Array.isArray(obj)) return obj.some(containsDynamicAnchor);
+    return Object.entries(obj).some(([k, v]) =>
+        DYNAMIC_ANCHOR_KEYWORDS.includes(k) || containsDynamicAnchor(v));
+}
+
+/**
+ * Parse a bundle-internal pointer
+ *   `#/<defsKeyword>/<schemaName>[/<defsKeyword>/<subName>][/...]`
+ * @return {null|{schemaName: string, subName: null|string}}
+ */
+function parseBundleRef(ref, defsKeyword) {
+    if (typeof ref !== 'string' || !ref.startsWith(`#/${defsKeyword}/`)) return null;
+    const segs = ref.slice(2).split('/').map(unescapeJsonPointerToken);
+    const schemaName = segs[1];
+    if (!schemaName) return null;
+    const subName = (segs[2] === defsKeyword && segs[3]) ? segs[3] : null;
+    return { schemaName, subName };
+}
+
+/**
+ * Remove, in place, every embedded definition of `schema[defsKeyword]` that is
+ * not transitively reachable from the schema root via $ref / $dynamicRef / $recursiveRef.
+ *
+ * Granularity:
+ *  - a pointer to `#/<defs>/X` (or into X outside of its defs container) keeps X whole;
+ *  - a pointer to `#/<defs>/X/<defs>/Y` keeps only X.<defs>.Y — the rest of X is replaced
+ *    by `not: {}`, so the hollowed-out container never validates anything by accident;
+ *  - X not reached at all is dropped.
+ *
+ * @param {object} schema - the bundled schema; mutated
+ * @param {string} defsKeyword
+ */
+function treeShakeBundle(schema, defsKeyword) {
+    const defs = schema[defsKeyword];
+    if (!isObject(defs)) return;
+
+    /** Marker: an embedded definition is used as a whole, not just some of its sub-definitions. */
+    const usedWhole = Symbol('used-whole');
+    /** @type {Map<string, typeof usedWhole|Set<string>>} schemaName -> usedWhole | set of used sub-def names */
+    const used = new Map();
+    const queue = [];
+
+    const enqueue = (node) => {
+        collectRefKeywords(node, REF_KEYWORDS).forEach(({ref}) => { queue.push(ref) });
+    };
+
+    const markWhole = (schemaName) => {
+        if (used.get(schemaName) === usedWhole) return;
+        used.set(schemaName, usedWhole);
+        if (schemaName in defs) enqueue(defs[schemaName]);
+    };
+
+    const markSub = (schemaName, subName) => {
+        const current = used.get(schemaName);
+        if (current === usedWhole) return;
+        const set = current || new Set();
+        if (set.has(subName)) return;
+        set.add(subName);
+        used.set(schemaName, set);
+        const target = defs[schemaName]?.[defsKeyword]?.[subName];
+        if (target === undefined) {
+            // Dangling pointer: keep schema whole and let the post-check report it.
+            markWhole(schemaName);
+        } else {
+            enqueue(target);
+        }
+    };
+
+    // Seeds: the root document (without its defs) ...
+    enqueue(stripTopLevelKeys(schema, [defsKeyword]));
+    // ... and every schema that can be reached dynamically by anchor name.
+    for (const [schemaName, defSchema] of Object.entries(defs)) {
+        if (containsDynamicAnchor(defSchema)) markWhole(schemaName);
+    }
+
+    // Worklist fixpoint
+    while (queue.length > 0) {
+        const parsed = parseBundleRef(queue.pop(), defsKeyword);
+        if (!parsed) continue; // external files, URLs, plain anchors: nothing to keep here
+        if (parsed.subName) {
+            markSub(parsed.schemaName, parsed.subName);
+        } else {
+            markWhole(parsed.schemaName);
+        }
+    }
+
+    // Prune in place
+    const before = Object.keys(defs).length;
+    let removedSubDefs = 0;
+    for (const [schemaName, defSchema] of Object.entries(defs)) {
+        const usage = used.get(schemaName);
+        if (!usage) {
+            console.log(`  Removed unused ${defsKeyword} entry '${schemaName}'`);
+            delete defs[schemaName];
+            continue;
+        }
+        if (usage === usedWhole) continue;
+        // Only sub-definitions are referenced: drop the schema body entirely,
+        // keep nothing but the referenced sub-definitions, and mark the container as not usable at all.
+        const keptSubDefs = {};
+        for (const [name, body] of Object.entries(defSchema[defsKeyword] || {})) {
+            if (usage.has(name)) {
+                keptSubDefs[name] = body;
+            } else {
+                console.log(`  Removed unused sub-definition '${schemaName}/${defsKeyword}/${name}'`);
+                removedSubDefs++;
+            }
+        }
+        defs[schemaName] = {
+            title: defSchema.title,
+            description: defSchema.description,
+            [defsKeyword]: keptSubDefs,
+            // `not: {}` rejects every instance, so the hollowed-out container is unusable on its own;
+            // unlike `false` it can still hold `$defs`, and unlike `not: true` it is valid in draft-04.
+            not: {}
+        };
+    }
+    console.log(`  ${defsKeyword}: ${before} -> ${Object.keys(defs).length} entries, ${removedSubDefs} sub-definitions removed`);
+}
+
 async function bundleSchemas(modelsDirectory, rootSchemaPath, options = {}) {
     try {
         const absoluteModelsDir = path.resolve(modelsDirectory);
@@ -316,38 +453,25 @@ async function bundleSchemas(modelsDirectory, rootSchemaPath, options = {}) {
             cleanedDefinitions[makeSchemaName(schemaPath)] = stripTopLevelKeys(defSchema, keysToStripFromDefs);
         }
 
-        // The root schema's own definition entry is emitted only when
-        // something references it: its content already forms the top level of
-        // the bundle, so an unreferenced copy is dead weight that downstream
-        // tools (e.g. schema documentation generators) render as a duplicate
-        // of the document root.
-        const rootDefName = makeSchemaName(absoluteRootPath);
-        const rootDefEntry = cleanedDefinitions[rootDefName];
-        delete cleanedDefinitions[rootDefName];
-
         // Build the final schema with root schema properties at the top level
+        // The root schema's own entry in `cleanedDefinitions` is handled by the
+        // tree-shaker below like any other module: dropped when unreferenced,
+        // hollowed out to its referenced sub-definitions otherwise.
         const finalSchema = {
             ...rootSchemaRewritten,
             "$schema": schemaVersion,
             [defsKeyword]: cleanedDefinitions
         };
 
-        const rootDefPointer = `#/${defsKeyword}/${rootDefName}`;
-        const rootDefRefs = collectRefKeywords(
-            finalSchema,
-            ['$ref', '$dynamicRef', '$recursiveRef'],
-            (v) => v === rootDefPointer || v.startsWith(`${rootDefPointer}/`)
-        );
-        if (rootDefRefs.length > 0) {
-            console.log(`Keeping ${defsKeyword} entry '${rootDefName}' (referenced ${rootDefRefs.length}x)`);
-            finalSchema[defsKeyword][rootDefName] = rootDefEntry;
-        }
+        // Tree-shake: drop every definition not reachable from the root
+        console.log('Tree-shaking unused definitions...');
+        treeShakeBundle(finalSchema, defsKeyword);
 
         // Post-check: ensure all internal JSON Pointer refs resolve in the final bundle
         console.log('Validating internal ref pointers ($ref, $dynamicRef, $recursiveRef)...');
         const internalRefs = collectRefKeywords(
             finalSchema,
-            ['$ref', '$dynamicRef', '$recursiveRef'],
+            REF_KEYWORDS,
             (v) => typeof v === 'string' && v.startsWith('#')
         );
         for (const { ref, key, path: refPath } of internalRefs) {

tools/src/main/js/bundler/test/snapshot/main-bundled.min.schema.json

@@ -1 +1 @@
-{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"http://example.com/test/main-bundled.min.schema.json","$comment":"FIXTURE: root schema; entry point of the bundle. This $comment is the only one that survives minification. The root title/description must survive in both outputs.","title":"Main","description":"Root document of the bundler test fixtures.","type":"object","properties":{"whole":{"title":"Whole","description":"A module referenced in its entirety.","$ref":"#/$defs/used-whole"},"sub":{"title":"Sub","description":"A single definition of another module.","$ref":"#/$defs/used-subdefs-only/$defs/kept"},"local":{"title":"Local","description":"A definition of this very schema.","$ref":"#/$defs/main/$defs/mainLocal"}},"$defs":{"dynamic-anchor":{"$dynamicAnchor":"node","title":"Dynamic anchor","description":"A recursive tree node, addressable via a dynamic anchor.","type":"object","properties":{"children":{"title":"Children","description":"The child nodes of this node.","type":"array","items":{"title":"Child","description":"A nested node.","$ref":"#/$defs/dynamic-anchor"}}}},"unused":{"title":"Unused","description":"A module that nothing reachable refers to.","type":"object","properties":{"pullsNothingIn":{"title":"Pulls nothing in","description":"A flag defined by another module.","$ref":"#/$defs/used-whole/$defs/unusedButKept"}}},"used-subdefs-only":{"title":"Used sub-defs only","description":"A module of which only individual definitions are referenced.","type":"object","properties":{"body":{"title":"Body","description":"The body of this module.","$ref":"#/$defs/unused"}},"$defs":{"kept":{"title":"Kept","description":"A definition that is reached directly from the root schema.","type":"object","properties":{"shared":{"title":"Shared","description":"A value defined by another module.","$ref":"#/$defs/used-whole/$defs/sharedByOther"},"fromMain":{"title":"From main","description":"A value defined by the root schema.","$ref":"#/$defs/main/$defs/mainLocal"},"chain":{"title":"Chain","description":"A value defined further down in this module.","$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"}}},"keptTransitively":{"title":"Kept transitively","description":"A string value.","type":"string"},"dropped":{"title":"Dropped","description":"A definition nothing reachable refers to.","type":"null"}}},"used-whole":{"title":"Used whole","description":"A module that is referenced in its entirety.","type":"object","properties":{"name":{"title":"Name","description":"A display name.","type":"string"},"inner":{"title":"Inner","description":"Sibling annotations next to a $ref: title/description are kept, while the sibling $comment is removed in the minified output.","$ref":"#/$defs/used-whole/$defs/inner"},"license":{"title":"License","description":"The license of this item.","$ref":"externals/not-bundled.schema.json#/$defs/licenseId"},"remote":{"title":"Remote","description":"A thing defined by a remote schema.","$ref":"https://example.com/remote.schema.json#/$defs/thing"},"inArray":{"title":"In array","description":"A composition of several referenced schemas.","allOf":[{"$ref":"#/$defs/used-whole/$defs/inner"},{"$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"},{"$ref":"externals/not-bundled.schema.json#/$defs/licenseId"}]}},"$defs":{"inner":{"title":"Inner","description":"An integer value.","type":"integer"},"unusedButKept":{"title":"Unused but kept","description":"A flag.","type":"boolean"},"sharedByOther":{"title":"Shared by other","description":"A numeric value shared across modules.","type":"number"}}},"main":{"title":"Main","description":"Root document of the bundler test fixtures.","type":"object","properties":{"whole":{"title":"Whole","description":"A module referenced in its entirety.","$ref":"#/$defs/used-whole"},"sub":{"title":"Sub","description":"A single definition of another module.","$ref":"#/$defs/used-subdefs-only/$defs/kept"},"local":{"title":"Local","description":"A definition of this very schema.","$ref":"#/$defs/main/$defs/mainLocal"}},"$defs":{"mainLocal":{"title":"Main local","description":"A local string definition of the root schema.","type":"string"},"mainUnused":{"title":"Main unused","description":"A definition nothing refers to.","type":"null"}}}}}
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"http://example.com/test/main-bundled.min.schema.json","$comment":"FIXTURE: root schema; entry point of the bundle. This $comment is the only one that survives minification. The root title/description must survive in both outputs.","title":"Main","description":"Root document of the bundler test fixtures.","type":"object","properties":{"whole":{"title":"Whole","description":"A module referenced in its entirety.","$ref":"#/$defs/used-whole"},"sub":{"title":"Sub","description":"A single definition of another module.","$ref":"#/$defs/used-subdefs-only/$defs/kept"},"local":{"title":"Local","description":"A definition of this very schema.","$ref":"#/$defs/main/$defs/mainLocal"}},"$defs":{"dynamic-anchor":{"$dynamicAnchor":"node","title":"Dynamic anchor","description":"A recursive tree node, addressable via a dynamic anchor.","type":"object","properties":{"children":{"title":"Children","description":"The child nodes of this node.","type":"array","items":{"title":"Child","description":"A nested node.","$ref":"#/$defs/dynamic-anchor"}}}},"used-subdefs-only":{"title":"Used sub-defs only","description":"A module of which only individual definitions are referenced.","$defs":{"kept":{"title":"Kept","description":"A definition that is reached directly from the root schema.","type":"object","properties":{"shared":{"title":"Shared","description":"A value defined by another module.","$ref":"#/$defs/used-whole/$defs/sharedByOther"},"fromMain":{"title":"From main","description":"A value defined by the root schema.","$ref":"#/$defs/main/$defs/mainLocal"},"chain":{"title":"Chain","description":"A value defined further down in this module.","$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"}}},"keptTransitively":{"title":"Kept transitively","description":"A string value.","type":"string"}},"not":{}},"used-whole":{"title":"Used whole","description":"A module that is referenced in its entirety.","type":"object","properties":{"name":{"title":"Name","description":"A display name.","type":"string"},"inner":{"title":"Inner","description":"Sibling annotations next to a $ref: title/description are kept, while the sibling $comment is removed in the minified output.","$ref":"#/$defs/used-whole/$defs/inner"},"license":{"title":"License","description":"The license of this item.","$ref":"externals/not-bundled.schema.json#/$defs/licenseId"},"remote":{"title":"Remote","description":"A thing defined by a remote schema.","$ref":"https://example.com/remote.schema.json#/$defs/thing"},"inArray":{"title":"In array","description":"A composition of several referenced schemas.","allOf":[{"$ref":"#/$defs/used-whole/$defs/inner"},{"$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"},{"$ref":"externals/not-bundled.schema.json#/$defs/licenseId"}]}},"$defs":{"inner":{"title":"Inner","description":"An integer value.","type":"integer"},"unusedButKept":{"title":"Unused but kept","description":"A flag.","type":"boolean"},"sharedByOther":{"title":"Shared by other","description":"A numeric value shared across modules.","type":"number"}}},"main":{"title":"Main","description":"Root document of the bundler test fixtures.","$defs":{"mainLocal":{"title":"Main local","description":"A local string definition of the root schema.","type":"string"}},"not":{}}}}

tools/src/main/js/bundler/test/snapshot/main-bundled.schema.json

@@ -45,31 +45,9 @@
         }
       }
     },
-    "unused": {
-      "title": "Unused",
-      "description": "A module that nothing reachable refers to.",
-      "type": "object",
-      "properties": {
-        "pullsNothingIn": {
-          "$comment": "Even though this refs a module, it is itself unreachable and must not influence shaking.",
-          "title": "Pulls nothing in",
-          "description": "A flag defined by another module.",
-          "$ref": "#/$defs/used-whole/$defs/unusedButKept"
-        }
-      }
-    },
     "used-subdefs-only": {
       "title": "Used sub-defs only",
       "description": "A module of which only individual definitions are referenced.",
-      "type": "object",
-      "properties": {
-        "body": {
-          "$comment": "Lives in the dropped body, so this ref must NOT make unused.schema.json reachable.",
-          "title": "Body",
-          "description": "The body of this module.",
-          "$ref": "#/$defs/unused"
-        }
-      },
       "$defs": {
         "kept": {
           "$comment": "Reached from main. Exercises three ref flavours.",
@@ -102,14 +80,9 @@
           "title": "Kept transitively",
           "description": "A string value.",
           "type": "string"
-        },
-        "dropped": {
-          "$comment": "Not reachable -> shaken away.",
-          "title": "Dropped",
-          "description": "A definition nothing reachable refers to.",
-          "type": "null"
         }
-      }
+      },
+      "not": {}
     },
     "used-whole": {
       "title": "Used whole",
@@ -179,41 +152,15 @@
     "main": {
       "title": "Main",
       "description": "Root document of the bundler test fixtures.",
-      "type": "object",
-      "properties": {
-        "whole": {
-          "$comment": "External ref without fragment -> module is kept as a whole.",
-          "title": "Whole",
-          "description": "A module referenced in its entirety.",
-          "$ref": "#/$defs/used-whole"
-        },
-        "sub": {
-          "$comment": "External ref into a sub-def -> only that sub-def (and what it reaches) is kept.",
-          "title": "Sub",
-          "description": "A single definition of another module.",
-          "$ref": "#/$defs/used-subdefs-only/$defs/kept"
-        },
-        "local": {
-          "$comment": "Internal ref -> rewritten to #/$defs/main/$defs/mainLocal, so the root's own def entry must be emitted.",
-          "title": "Local",
-          "description": "A definition of this very schema.",
-          "$ref": "#/$defs/main/$defs/mainLocal"
-        }
-      },
       "$defs": {
         "mainLocal": {
           "$comment": "Referenced from main itself and from used-subdefs-only.",
           "title": "Main local",
           "description": "A local string definition of the root schema.",
           "type": "string"
-        },
-        "mainUnused": {
-          "$comment": "Never referenced -> shaken away from #/$defs/main/$defs.",
-          "title": "Main unused",
-          "description": "A definition nothing refers to.",
-          "type": "null"
         }
-      }
+      },
+      "not": {}
     }
   }
 }