module: improve external format support

doc/api/errors.md

@@ -2921,7 +2921,19 @@ extension.
 
 > Stability: 1 - Experimental
 
-An attempt was made to load a module with an unknown or unsupported format.
+An attempt was made to load a module with an unknown format.
+
+<a id="ERR_UNSUPPORTED_MODULE_FORMAT"></a>
+
+### `ERR_UNSUPPORTED_MODULE_FORMAT`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+An attempt was made to load a module with an unsupported format.
 
 <a id="ERR_UNKNOWN_SIGNAL"></a>
 

doc/api/module.md

@@ -881,6 +881,67 @@ console.log('some module!');
 Running `node --import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register(pathToFileURL("./import-map-hooks.js"));' main.js`
 should print `some module!`.
 
+### Extending support
+
+> Stability: 1.0 - Early development
+
+A `"register"` [entry point][`"exports"`] for a package loaded via [`--import`][] is automatically
+run at startup. A package extending Node.js support for typescript would look something like this:
+
+```json
+{
+  "name": "example-nodejs-extension",
+  "keywords": [
+    "nodejs-extension",
+    "typescript"
+  ],
+  "exports": {
+    "register": "./registration.mjs",
+    "typescript": "./hooks/typescript.mjs"
+  }
+}
+```
+
+Setting the keyword `nodejs-extension` is important for users to find the package (which may be
+automated). It should also contain the official name(s) for the support it provides, such as
+`typescript`; it should avoid red-herrings such as including `typescript` when the library does not
+extend support for typescript but is merely itself written in typescript.
+
+```mjs
+import { register } from 'node:module';
+
+register('example-nodejs-extension/typescript');
+```
+
+`typescript.mjs` would contain [customization hooks][hooks]:
+
+* A [`resolve` hook][resolve hook] that sets `format` for applicable modules to the format it
+  handles, such as `'typescript'`.
+* A [`load` hook][load hook] that transpiles the external format (as signalled by its resolve hook)
+  to something Node.js understands.
+* Optionally, an [`initialize` hook][`initialize`].
+
+#### External formats
+
+> Stability: 1.0 - Early development
+
+Node.js natively understands a handful of formats (see the table in [load hook][]).
+Non-native or external formats require transpilation to something Node.js understands. When
+attempting to run a module with an external format, such as `node main.ts`, Node.js will look at the
+file extension to attempt to identify the type. If the type is recognized, Node.js will print a
+message with instructions (that lead here).
+
+##### Setting up an extension
+
+These steps configure Node.js to support a format it doesn't understand:
+
+1. Find and install an extension via your preferred package manager. Some package managers provide a
+   CLI utility, such as `npm search nodejs-extension typescript`, as well as a website.
+1. Once installed, in order to get Node.js to automatically use it, create a [`.env`][`--env-file`]
+   file containing an [`--import`][] for your chosen extension, like
+   `NODE_OPTIONS="--import=example-nodejs-extension"`.
+1. Include the env file flag on subsequent runs, like `node --env-file=.env main.ts`.
+
 ## Source map v3 support
 
 <!-- YAML
@@ -1033,6 +1094,8 @@ returned object contains the following keys:
 [Source map v3 format]: https://sourcemaps.info/spec.html#h.mofvlxcwqzej
 [`"exports"`]: packages.md#exports
 [`--enable-source-maps`]: cli.md#--enable-source-maps
+[`--env-file`]: cli.md#--env-file
+[`--import`]: cli.md#--import
 [`ArrayBuffer`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer
 [`NODE_V8_COVERAGE=dir`]: cli.md#node_v8_coveragedir
 [`SharedArrayBuffer`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer
@@ -1048,5 +1111,6 @@ returned object contains the following keys:
 [load hook]: #loadurl-context-nextload
 [module wrapper]: modules.md#the-module-wrapper
 [realm]: https://tc39.es/ecma262/#realm
+[resolve hook]: #resolvespecifier-context-nextresolve
 [source map include directives]: https://sourcemaps.info/spec.html#h.lmz475t4mvbx
 [transferrable objects]: worker_threads.md#portpostmessagevalue-transferlist

lib/internal/errors.js

@@ -1792,8 +1792,7 @@ E('ERR_UNKNOWN_BUILTIN_MODULE', 'No such built-in module: %s', Error);
 E('ERR_UNKNOWN_CREDENTIAL', '%s identifier does not exist: %s', Error);
 E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s', TypeError);
 E('ERR_UNKNOWN_FILE_EXTENSION', 'Unknown file extension "%s" for %s', TypeError);
-E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s for URL %s',
-  RangeError);
+E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: "%s" for URL %s', RangeError);
 E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s', TypeError);
 E('ERR_UNSUPPORTED_DIR_IMPORT', function(path, base, exactUrl) {
   lazyInternalUtil().setOwnProperty(this, 'url', exactUrl);
@@ -1809,6 +1808,10 @@ E('ERR_UNSUPPORTED_ESM_URL_SCHEME', (url, supported) => {
   msg += `. Received protocol '${url.protocol}'`;
   return msg;
 }, Error);
+E('ERR_UNSUPPORTED_MODULE_FORMAT',
+  'Module format "%s" for URL %s has no installed translator. Install one and re-run. ' +
+  'See https://nodejs.org/api/module.html#extending-support for more information.',
+  TypeError);
 E('ERR_USE_AFTER_CLOSE', '%s was closed', Error);
 
 // This should probably be a `TypeError`.

lib/internal/modules/esm/formats.js

@@ -1,6 +1,11 @@
 'use strict';
 
-const { RegExpPrototypeExec } = primordials;
+const {
+  ObjectAssign,
+  ObjectValues,
+  RegExpPrototypeExec,
+  SafeSet,
+} = primordials;
 const { getOptionValue } = require('internal/options');
 const { getValidatedPath } = require('internal/fs/utils');
 const pathModule = require('path');
@@ -9,17 +14,40 @@ const { fs: fsConstants } = internalBinding('constants');
 
 const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
 
-const extensionFormatMap = {
+/**
+ * @typedef {string} FileExtension The extension of a file, including the leading dot.
+ * @typedef {string} FormatName A tag-friendly name for a module format. This is used to search for
+ * a module translator.
+ */
+/**
+ * @type {Record<FileExtension, FormatName>}
+ */
+const externalFormatDict = {
   '__proto__': null,
-  '.cjs': 'commonjs',
-  '.js': 'module',
-  '.json': 'json',
-  '.mjs': 'module',
+  '.ts': 'typescript',
+  '.tsx': 'typescript',
+  '.cts': 'typescript',
+  '.mts': 'typescript',
 };
-
-if (experimentalWasmModules) {
-  extensionFormatMap['.wasm'] = 'wasm';
-}
+/**
+ * @type {Set<FormatName>} A de-duplicated set of all external format names against which to check
+ * after all internal formats have failed.
+ */
+const externalFormats = new SafeSet(ObjectValues(externalFormatDict));
+/**
+ * @type {Record<FileExtension, FormatName>}
+ */
+const extensionFormatDict = ObjectAssign(
+  {
+    '__proto__': null,
+    '.cjs': 'commonjs',
+    '.js': 'module',
+    '.json': 'json',
+    '.mjs': 'module',
+    ...(experimentalWasmModules && { '.wasm': 'wasm' }),
+  },
+  externalFormatDict,
+);
 
 /**
  * @param {string} mime
@@ -57,7 +85,8 @@ function getFormatOfExtensionlessFile(url) {
 }
 
 module.exports = {
-  extensionFormatMap,
+  extensionFormatDict,
+  externalFormats,
   getFormatOfExtensionlessFile,
   mimeToFormat,
 };

lib/internal/modules/esm/get_format.js

@@ -11,7 +11,7 @@ const {
 } = primordials;
 const { getOptionValue } = require('internal/options');
 const {
-  extensionFormatMap,
+  extensionFormatDict,
   getFormatOfExtensionlessFile,
   mimeToFormat,
 } = require('internal/modules/esm/formats');
@@ -145,7 +145,7 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE
     }
   }
 
-  const format = extensionFormatMap[ext];
+  const format = extensionFormatDict[ext];
   if (format) { return format; }
 
   // Explicit undefined return indicates load hook should rerun format check
@@ -200,6 +200,6 @@ function defaultGetFormat(url, context) {
 module.exports = {
   defaultGetFormat,
   defaultGetFormatWithoutErrors,
-  extensionFormatMap,
+  extensionFormatDict,
   extname,
 };

lib/internal/modules/esm/loader.js

@@ -19,6 +19,7 @@ const {
 const {
   ERR_REQUIRE_ESM,
   ERR_UNKNOWN_MODULE_FORMAT,
+  ERR_UNSUPPORTED_MODULE_FORMAT,
 } = require('internal/errors').codes;
 const { getOptionValue } = require('internal/options');
 const { pathToFileURL, isURL } = require('internal/url');
@@ -279,6 +280,14 @@ class ModuleLoader {
       const translator = getTranslators().get(finalFormat);
 
       if (!translator) {
+        const { externalFormats } = require('internal/modules/esm/formats');
+        // `defaultGetFormat()` will have resolved a recognised external file extension to a format.
+        if (externalFormats.has(finalFormat)) {
+          throw new ERR_UNSUPPORTED_MODULE_FORMAT(
+            finalFormat,
+            responseURL,
+          );
+        }
         throw new ERR_UNKNOWN_MODULE_FORMAT(finalFormat, responseURL);
       }
 

lib/repl.js

@@ -186,7 +186,7 @@ const {
 
 const history = require('internal/repl/history');
 const {
-  extensionFormatMap,
+  extensionFormatDict,
 } = require('internal/modules/esm/formats');
 const {
   makeContextifyScript,
@@ -1405,7 +1405,7 @@ function complete(line, callback) {
     if (this.allowBlockingCompletions) {
       const subdir = match[2] || '';
       // File extensions that can be imported:
-      const extensions = ObjectKeys(extensionFormatMap);
+      const extensions = ObjectKeys(extensionFormatDict);
 
       // Only used when loading bare module specifiers from `node_modules`:
       const indexes = ArrayPrototypeMap(extensions, (ext) => `index${ext}`);

test/es-module/test-esm-invalid-data-urls.js

@@ -6,17 +6,17 @@ const assert = require('assert');
   await assert.rejects(import('data:text/plain,export default0'), {
     code: 'ERR_UNKNOWN_MODULE_FORMAT',
     message:
-      'Unknown module format: text/plain for URL data:text/plain,' +
+      'Unknown module format: "text/plain" for URL data:text/plain,' +
       'export default0',
   });
   await assert.rejects(import('data:text/plain;base64,'), {
     code: 'ERR_UNKNOWN_MODULE_FORMAT',
     message:
-      'Unknown module format: text/plain for URL data:text/plain;base64,',
+      'Unknown module format: "text/plain" for URL data:text/plain;base64,',
   });
   await assert.rejects(import('data:text/css,.error { color: red; }'), {
     code: 'ERR_UNKNOWN_MODULE_FORMAT',
-    message: 'Unknown module format: text/css for URL data:text/css,.error { color: red; }',
+    message: 'Unknown module format: "text/css" for URL data:text/css,.error { color: red; }',
   });
   await assert.rejects(import('data:WRONGtext/javascriptFORMAT,console.log("hello!");'), {
     code: 'ERR_UNKNOWN_MODULE_FORMAT',

test/es-module/test-esm-loader-invalid-format.mjs

@@ -5,6 +5,6 @@ import assert from 'assert';
 import('../fixtures/es-modules/test-esm-ok.mjs')
 .then(assert.fail, expectsError({
   code: 'ERR_UNKNOWN_MODULE_FORMAT',
-  message: /Unknown module format: esm/
+  message: /Unknown module format: "esm"/
 }))
 .then(mustCall());

test/es-module/test-esm-non-js.mjs

@@ -1,20 +1,78 @@
 import { spawnPromisified } from '../common/index.mjs';
-import { fileURL } from '../common/fixtures.mjs';
-import { match, strictEqual } from 'node:assert';
+import * as fixtures from '../common/fixtures.mjs';
+import tmpdir from '../common/tmpdir.js';
+
+import {
+  deepStrictEqual,
+  match,
+  strictEqual,
+} from 'node:assert';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 import { execPath } from 'node:process';
 import { describe, it } from 'node:test';
 
 
-describe('ESM: non-js extensions fail', { concurrency: true }, () => {
-  it(async () => {
-    const { code, stderr, signal } = await spawnPromisified(execPath, [
-      '--input-type=module',
-      '--eval',
-      `import ${JSON.stringify(fileURL('es-modules', 'file.unknown'))}`,
-    ]);
+describe('ESM: non-js extensions', { concurrency: true }, () => {
+  describe('when no handler is installed', { concurrency: true }, () => {
+    it('should throw on unknown file extension', async () => {
+      const { code, stderr, signal } = await spawnPromisified(execPath, [
+        '--input-type=module',
+        '--eval',
+        `import ${JSON.stringify(fixtures.fileURL('es-modules', 'file.unknown'))}`,
+      ]);
+
+      match(stderr, /ERR_UNKNOWN_FILE_EXTENSION/);
+      strictEqual(code, 1);
+      strictEqual(signal, null);
+    });
+
+    it('should identify format and handle .ts', async () => {
+      const { code, stderr, signal } = await spawnPromisified(execPath, [
+        '--input-type=module',
+        '--eval',
+        `import ${JSON.stringify(fixtures.fileURL('empty.ts'))}`,
+      ]);
+
+      match(stderr, /ERR_UNSUPPORTED_MODULE_FORMAT/);
+      match(stderr, /typescript/);
+      match(stderr, /install/);
+      match(stderr, /nodejs\.org/);
+      strictEqual(code, 1);
+      strictEqual(signal, null);
+    });
+  });
+
+  it('when handler is installed', { concurrency: true }, async (t) => {
+    const cwd = tmpdir.resolve('ts-test' + Math.random());
+
+    try {
+      tmpdir.refresh();
+      const nodeModules = path.join(cwd, 'node_modules');
+      await fs.mkdir(nodeModules, { recursive: true });
+      await fs.cp(
+        fixtures.fileURL('external-modules', 'web-loader'),
+        path.join(nodeModules, 'web-loader'),
+        { recursive: true },
+      );
+      await fs.writeFile(path.join(cwd, 'main.ts'), 'const foo: number = 1; console.log(foo);\n');
+      await fs.writeFile(path.join(cwd, '.env'), 'NODE_OPTIONS=--import=web-loader\n');
+
+      await t.test('should identify format and handle .ts', async () => {
+        const output = await spawnPromisified(execPath, [
+          '--env-file=.env',
+          'main.ts',
+        ], { cwd });
 
-    match(stderr, /ERR_UNKNOWN_FILE_EXTENSION/);
-    strictEqual(code, 1);
-    strictEqual(signal, null);
+        deepStrictEqual(output, {
+          code: 0,
+          signal: null,
+          stderr: '',
+          stdout: '1\n',
+        });
+      });
+    } finally {
+      await fs.rm(cwd, { recursive: true, force: true });
+    }
   });
 });

test/es-module/test-esm-resolve-type.mjs

@@ -185,7 +185,7 @@ try {
     [ 'qmod', 'index.js', 'imp.js', 'commonjs', 'module', 'module', '?k=v'],
     [ 'hmod', 'index.js', 'imp.js', 'commonjs', 'module', 'module', '#Key'],
     [ 'qhmod', 'index.js', 'imp.js', 'commonjs', 'module', 'module', '?k=v#h'],
-    [ 'ts-mod-com', 'index.js', 'imp.ts', 'module', 'commonjs', undefined],
+    [ 'ts-mod-com', 'index.js', 'imp.ts', 'module', 'commonjs', 'typescript'],
   ].forEach((testVariant) => {
     const [
       moduleName,

test/fixtures/external-modules/web-loader/index.mjs

@@ -0,0 +1,4 @@
+import { register } from 'node:module';
+
+register('./ts.mjs', import.meta.url);
+register('./txt.mjs', import.meta.url);