esm: add initialize hook

Refs: nodejs/loaders#147

doc/api/esm.md

@@ -685,6 +685,12 @@ of Node.js applications.
 <!-- YAML
 added: v8.8.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/48842
+    description: Added `initialize` hook and deprecated `getGlobalPreload`.
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/48559
+    description: Allow loaders to use `import()` and unflag `Module.register`.
   - version:
     - v18.6.0
     - v16.17.0
@@ -938,19 +944,81 @@ export async function load(url, context, nextLoad) {
 In a more advanced scenario, this can also be used to transform an unsupported
 source to a supported one (see [Examples](#examples) below).
 
+#### `initialize()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> The loaders API is being redesigned. This hook may disappear or its
+> signature may change. Do not rely on the API described below.
+
+> In a previous version of this API, this was split across 3 separate, now
+> deprecated, hooks (`getFormat`, `getSource`, and `transformSource`).
+
+* `data` {any} The data provided via `Module.register(loader, parentUrl, data)`
+* Returns: {any} The data returned to the caller of `Module.register`
+
+The `initialize` hook provides a way to define a custom method of that runs
+in the loader's thread when the loader is initialized. This hook can send
+and receive data from a `Module.register` invocation, including ports and
+other transferrable objects.
+
+Loader code:
+
+```js
+// In the below example this file is referenced as
+// '/path-to-my-loader.js'
+
+export function initialize({ number, port }) {
+  port.postMessage(`increment: ${number + 1}`);
+  return 'ok';
+}
+```
+
+Caller code:
+
+```js
+import assert from 'node:assert';
+import { register } from 'node:module';
+import { MessageChannel } from 'node:worker_threads';
+
+// In this example '/path-to-my-loader.js' is replaced with the
+// path to the file containing the loader contents above.
+
+// This example showcases how a message channel can be used to
+// communicate to the loader, by sending `port2` to the loader.
+const { port1, port2 } = new MessageChannel();
+
+port1.on('message', (msg) => {
+  assert(msg === 'increment: 2');
+});
+
+const result = register('/path-to-my-loader.js', import.meta.url, {
+  data: { number: 1, port: port2 },
+  transferList: [port2],
+});
+
+assert(result === 'ok');
+```
+
 #### `globalPreload()`
 
 <!-- YAML
+deprecated: REPLACEME
 changes:
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/48842
+    description: Deprecated in favor of `initialize` hook.
   - version:
     - v18.6.0
     - v16.17.0
     pr-url: https://github.com/nodejs/node/pull/42623
     description: Add support for chaining globalPreload hooks.
 -->
 
-> The loaders API is being redesigned. This hook may disappear or its
-> signature may change. Do not rely on the API described below.
+> Deprecated: Use `initialize` instead.
 
 > In a previous version of this API, this hook was named
 > `getGlobalPreloadCode`.

doc/api/module.md

@@ -175,6 +175,27 @@ globalPreload: http-to-https
 globalPreload: unpkg
 ```
 
+This function can also be used to pass data to the loader's `initialize`
+hook include transferrable objects like ports.
+
+```mjs
+import { register } from 'node:module';
+import { MessageChannel } from 'node:worker_threads';
+
+// This example showcases how a message channel can be used to
+// communicate to the loader, by sending `port2` to the loader.
+const { port1, port2 } = new MessageChannel();
+
+port1.on('message', (msg) => {
+  console.log(msg);
+});
+
+register('./my-programmatic-loader.mjs', import.meta.url, {
+  data: { number: 1, port: port2 },
+  transferList: [port2],
+});
+```
+
 ### `module.syncBuiltinESMExports()`
 
 <!-- YAML

lib/internal/modules/esm/hooks.js

@@ -2,6 +2,7 @@
 
 const {
   ArrayPrototypePush,
+  ArrayPrototypePushApply,
   FunctionPrototypeCall,
   Int32Array,
   ObjectAssign,
@@ -127,28 +128,33 @@ class Hooks {
    * Import and register custom/user-defined module loader hook(s).
    * @param {string} urlOrSpecifier
    * @param {string} parentURL
+   * @param {any} [data] Arbitrary data to be passed from the custom
+   * loader (user-land) to the worker.
    */
-  async register(urlOrSpecifier, parentURL) {
+  async register(urlOrSpecifier, parentURL, data) {
     const moduleLoader = require('internal/process/esm_loader').esmLoader;
     const keyedExports = await moduleLoader.import(
       urlOrSpecifier,
       parentURL,
       kEmptyObject,
     );
-    this.addCustomLoader(urlOrSpecifier, keyedExports);
+    return this.addCustomLoader(urlOrSpecifier, keyedExports, data);
   }
 
   /**
    * Collect custom/user-defined module loader hook(s).
    * After all hooks have been collected, the global preload hook(s) must be initialized.
    * @param {string} url Custom loader specifier
    * @param {Record<string, unknown>} exports
+   * @param {any} [data] Arbitrary data to be passed from the custom loader (user-land)
+   * to the worker.
    */
-  addCustomLoader(url, exports) {
+  addCustomLoader(url, exports, data) {
     const {
       globalPreload,
       resolve,
       load,
+      initialize,
     } = pluckHooks(exports);
 
     if (globalPreload) {
@@ -162,6 +168,7 @@ class Hooks {
       const next = this.#chains.load[this.#chains.load.length - 1];
       ArrayPrototypePush(this.#chains.load, { __proto__: null, fn: load, url, next });
     }
+    return initialize?.(data);
   }
 
   /**
@@ -553,15 +560,26 @@ class HooksProxy {
     }
   }
 
-  async makeAsyncRequest(method, ...args) {
+  /**
+   * Invoke a remote method asynchronously.
+   * @param {any[]} transferList Objects in `args` to be transferred
+   * @param {string} method Method to invoke
+   * @param  {any[]} args Arguments to pass to `method`
+   * @returns {Promise<any>}
+   */
+  async makeAsyncRequest(transferList, method, ...args) {
     this.waitForWorker();
 
     MessageChannel ??= require('internal/worker/io').MessageChannel;
     const asyncCommChannel = new MessageChannel();
 
     // Pass work to the worker.
-    debug('post async message to worker', { method, args });
-    this.#worker.postMessage({ method, args, port: asyncCommChannel.port2 }, [asyncCommChannel.port2]);
+    debug('post async message to worker', { method, args, transferList });
+    const finalTransferList = [asyncCommChannel.port2];
+    if (transferList) {
+      ArrayPrototypePushApply(finalTransferList, transferList);
+    }
+    this.#worker.postMessage({ __proto__: null, method, args, port: asyncCommChannel.port2 }, finalTransferList);
 
     if (this.#numberOfPendingAsyncResponses++ === 0) {
       // On the next lines, the main thread will await a response from the worker thread that might
@@ -593,12 +611,19 @@ class HooksProxy {
     return body;
   }
 
-  makeSyncRequest(method, ...args) {
+  /**
+   * Invoke a remote method synchronously.
+   * @param {any[]} transferList Objects in `args` to be transferred
+   * @param {string} method Method to invoke
+   * @param  {any[]} args Arguments to pass to `method`
+   * @returns {any}
+   */
+  makeSyncRequest(transferList, method, ...args) {
     this.waitForWorker();
 
     // Pass work to the worker.
-    debug('post sync message to worker', { method, args });
-    this.#worker.postMessage({ method, args });
+    debug('post sync message to worker', { method, args, transferList });
+    this.#worker.postMessage({ __proto__: null, method, args }, transferList);
 
     let response;
     do {
@@ -710,6 +735,7 @@ function pluckHooks({
   globalPreload,
   resolve,
   load,
+  initialize,
 }) {
   const acceptedHooks = { __proto__: null };
 
@@ -723,6 +749,10 @@ function pluckHooks({
     acceptedHooks.load = load;
   }
 
+  if (initialize) {
+    acceptedHooks.initialize = initialize;
+  }
+
   return acceptedHooks;
 }
 

lib/internal/modules/esm/loader.js

@@ -307,15 +307,15 @@ class ModuleLoader {
     return module.getNamespace();
   }
 
-  register(specifier, parentUrl) {
+  register(specifier, parentUrl, data, transferList) {
     if (!this.#customizations) {
       // `CustomizedModuleLoader` is defined at the bottom of this file and
       // available well before this line is ever invoked. This is here in
       // order to preserve the git diff instead of moving the class.
       // eslint-disable-next-line no-use-before-define
       this.setCustomizations(new CustomizedModuleLoader());
     }
-    return this.#customizations.register(specifier, parentUrl);
+    return this.#customizations.register(specifier, parentUrl, data, transferList);
   }
 
   /**
@@ -426,10 +426,13 @@ class CustomizedModuleLoader {
    *                                   be registered.
    * @param {string} parentURL The parent URL from where the loader will be
    *                           registered if using it package name as specifier
+   * @param {any} data Arbitrary data to be passed from the custom loader
+   * (user-land) to the worker.
+   * @param {any[]} transferList Objects in `data` that are changing ownership
    * @returns {{ format: string, url: URL['href'] }}
    */
-  register(originalSpecifier, parentURL) {
-    return hooksProxy.makeSyncRequest('register', originalSpecifier, parentURL);
+  register(originalSpecifier, parentURL, data, transferList) {
+    return hooksProxy.makeSyncRequest(transferList, 'register', originalSpecifier, parentURL, data);
   }
 
   /**
@@ -442,12 +445,12 @@ class CustomizedModuleLoader {
    * @returns {{ format: string, url: URL['href'] }}
    */
   resolve(originalSpecifier, parentURL, importAssertions) {
-    return hooksProxy.makeAsyncRequest('resolve', originalSpecifier, parentURL, importAssertions);
+    return hooksProxy.makeAsyncRequest(undefined, 'resolve', originalSpecifier, parentURL, importAssertions);
   }
 
   resolveSync(originalSpecifier, parentURL, importAssertions) {
     // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec.
-    return hooksProxy.makeSyncRequest('resolve', originalSpecifier, parentURL, importAssertions);
+    return hooksProxy.makeSyncRequest(undefined, 'resolve', originalSpecifier, parentURL, importAssertions);
   }
 
   /**
@@ -457,7 +460,7 @@ class CustomizedModuleLoader {
    * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>}
    */
   load(url, context) {
-    return hooksProxy.makeAsyncRequest('load', url, context);
+    return hooksProxy.makeAsyncRequest(undefined, 'load', url, context);
   }
 
   importMetaInitialize(meta, context, loader) {
@@ -514,18 +517,22 @@ function getHooksProxy() {
  * Register a single loader programmatically.
  * @param {string} specifier
  * @param {string} [parentURL]
- * @returns {void}
+ * @param {any} [data] Arbitrary data passed to loader's `initialize` hook
+ * @param {any[]} transferList Objects in `data` that are changing ownership
+ * @returns {any}
  * @example
  * ```js
  * register('./myLoader.js');
  * register('ts-node/esm', import.meta.url);
  * register('./myLoader.js', import.meta.url);
  * register(new URL('./myLoader.js', import.meta.url));
+ * register('./myLoader.js', import.meta.url, {banana: 'tasty'});
+ * register('./myLoader.js', import.meta.url, someArrayBuffer, [someArrayBuffer]);
  * ```
  */
-function register(specifier, parentURL = 'data:') {
+function register(specifier, parentURL = 'data:', data, transferList) {
   const moduleLoader = require('internal/process/esm_loader').esmLoader;
-  moduleLoader.register(`${specifier}`, parentURL);
+  return moduleLoader.register(`${specifier}`, parentURL, data, transferList);
 }
 
 module.exports = {

lib/internal/modules/esm/utils.js

@@ -123,6 +123,16 @@ async function initializeHooks() {
   const hooks = new Hooks();
   esmLoader.setCustomizations(hooks);
 
+  // We need the loader customizations to be set _before_ we start invoking
+  // `--require`, otherwise loops can happen because a `--require` script
+  // might call `register(...)` before we've installed ourselves. These
+  // global values are magically set in `setupUserModules` just for us and
+  // we call them in the correct order.
+  // N.B.  This block appears here specifically in order to ensure that
+  // `--require` calls occur before `--loader` ones do.
+  globalThis.loadPreloadModules();
+  globalThis.initializeFrozenIntrinsics();
+
   const parentURL = pathToFileURL(cwd).href;
   for (let i = 0; i < customLoaderURLs.length; i++) {
     await hooks.register(

lib/internal/process/pre_execution.js

@@ -154,9 +154,16 @@ function setupUserModules(isLoaderWorker = false) {
   initializeESMLoader(isLoaderWorker);
   const CJSLoader = require('internal/modules/cjs/loader');
   assert(!CJSLoader.hasLoadedAnyUserCJSModule);
-  loadPreloadModules();
-  // Need to be done after --require setup.
-  initializeFrozenIntrinsics();
+  if (isLoaderWorker) {
+    // Loader workers are responsible for doing this themselves.
+    globalThis.loadPreloadModules = loadPreloadModules;
+    globalThis.initializeFrozenIntrinsics = initializeFrozenIntrinsics;
+  } else {
+    loadPreloadModules();
+    // Need to be done after --require setup.
+    initializeFrozenIntrinsics();
+  }
+
 }
 
 function refreshRuntimeOptions() {