[esm-integration] Add pthread support

Under ESM integration all dependencies must be satisfied at import time,
but pthreads requires that we have supply the memory via postMessage,
so the memory is, by definition, not available at import time.

On order to work around this issue we create a smaller pthread
stub/loader file that delays the import of the main program until the
initial `postMessage` has been received.  Once the memory is received we
load main program using a dynamic `import` statement.

Author: sbc100

eslint.config.mjs

@@ -35,6 +35,7 @@ export default [{
     'src/postamble*.js',
     'src/closure-externs/',
     'src/embind/',
+    'src/pthread_esm_startup.mjs',
     'src/emrun_postjs.js',
     'src/wasm_worker.js',
     'src/audio_worklet.js',

package.json

@@ -26,7 +26,7 @@
   },
   "scripts": {
     "lint": "eslint .",
-    "fmt": "prettier --write src/*.mjs tools/*.mjs",
-    "check": "prettier --check src/*.mjs tools/*.mjs"
+    "fmt": "prettier --write src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs",
+    "check": "prettier --check src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs"
   }
 }

site/source/docs/compiling/Modularized-Output.rst

@@ -163,7 +163,7 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to
 
 Some additional limitations are:
 
-* ``-pthread`` / :ref:`wasm_workers` are not yet supported.
+* :ref:`wasm_workers` is not yet supported.
 
 * :ref:`abort_on_wasm_exceptions` is not supported (requires wrapping wasm
   exports).

src/lib/libpthread.js

@@ -29,6 +29,13 @@ const MAX_PTR = Number((2n ** 64n) - 1n);
 #else
 const MAX_PTR = (2 ** 32) - 1
 #endif
+
+#if WASM_ESM_INTEGRATION
+const pthreadWorkerScript = TARGET_BASENAME + '.pthread.mjs';
+#else
+const pthreadWorkerScript = TARGET_JS_NAME;
+#endif
+
 // Use a macro to avoid duplicating pthread worker options.
 // We cannot use a normal JS variable since the vite bundler requires that worker
 // options be inline.
@@ -295,7 +302,9 @@ var LibraryPThread = {
 
 #if ASSERTIONS
       assert(wasmMemory instanceof WebAssembly.Memory, 'WebAssembly memory should have been loaded by now!');
+#if !WASM_ESM_INTEGRATION
       assert(wasmModule instanceof WebAssembly.Module, 'WebAssembly Module should have been loaded by now!');
+#endif
 #endif
 
       // When running on a pthread, none of the incoming parameters on the module
@@ -333,7 +342,9 @@ var LibraryPThread = {
 #else // WASM2JS
         wasmMemory,
 #endif // WASM2JS
+#if !WASM_ESM_INTEGRATION
         wasmModule,
+#endif
 #if LOAD_SOURCE_MAP
         wasmSourceMap,
 #endif
@@ -391,7 +402,7 @@ var LibraryPThread = {
 #if TRUSTED_TYPES
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
-        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => import.meta.url });
+        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => new URL('{{{ pthreadWorkerScript }}}', import.meta.url) });
         worker = new Worker(p.createScriptURL('ignored'), {{{ pthreadWorkerOptions }}});
       } else
 #endif
@@ -409,7 +420,7 @@ var LibraryPThread = {
       // the first case in their bundling step. The latter ends up producing an invalid
       // URL to import from the server (e.g., for webpack the file:// path).
       // See https://github.com/webpack/webpack/issues/12638
-      worker = new Worker(new URL('{{{ TARGET_JS_NAME }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
+      worker = new Worker(new URL('{{{ pthreadWorkerScript }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
 #else // EXPORT_ES6
       var pthreadMainJs = _scriptName;
 #if expectToReceiveOnModule('mainScriptUrlOrBlob')

src/modularize.js

@@ -8,7 +8,7 @@
 // JS program code (INNER_JS_CODE) and wrapping it in a factory function.
 
 #if SOURCE_PHASE_IMPORTS
-import source wasmModule from './{settings.WASM_BINARY_FILE}';
+import source wasmModule from './{{{ WASM_BINARY_FILE }}}';
 #endif
 
 #if ENVIRONMENT_MAY_BE_WEB && !EXPORT_ES6 && !(MINIMAL_RUNTIME && !PTHREADS)

src/postamble.js

@@ -306,6 +306,9 @@ export default async function init(moduleArg = {}) {
   Object.assign(Module, moduleArg);
   processModuleArgs();
 #if WASM_ESM_INTEGRATION
+#if PTHREADS
+  registerTLSInit(__emscripten_tls_init);
+#endif
   updateMemoryViews();
 #if DYNCALLS && '$dynCalls' in addedLibraryItems
 
@@ -318,7 +321,7 @@ export default async function init(moduleArg = {}) {
   run();
 }
 
-#if PTHREADS || WASM_WORKERS
+#if (WASM_WORKERS || PTHREADS) && !WASM_ESM_INTEGRATION
 // When run as a worker thread run `init` immediately.
 if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) await init()
 #endif

src/preamble.js

@@ -990,7 +990,7 @@ function getWasmImports() {
 #endif // WASM_ASYNC_COMPILATION
 #endif // SOURCE_PHASE_IMPORTS
 }
-#endif
+#endif // WASM_ESM_INTEGRATION
 
 #if !WASM_BIGINT
 // Globals used by JS i64 conversions (see makeSetValue)

src/pthread_esm_startup.mjs

@@ -0,0 +1,53 @@
+/**
+ * @license
+ * Copyright 2025 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+// This file is used as the initial script loaded into pthread workers when
+// running in WASM_ESM_INTEGRATION mode.
+// Tyhe point of this file is to delay the loading of the main program module
+// until the wasm memory has been received via postMessage.
+
+#if RUNTIME_DEBUG
+console.log("Running pthread_esm_startup");
+#endif
+
+#if ENVIRONMENT_MAY_BE_NODE
+// Create as web-worker-like an environment as we can.
+var worker_threads = await import('worker_threads');
+global.Worker = worker_threads.Worker;
+var parentPort = worker_threads['parentPort'];
+parentPort.on('message', (msg) => global.onmessage?.({ data: msg }));
+Object.assign(globalThis, {
+  self: global,
+  postMessage: (msg) => parentPort['postMessage'](msg),
+});
+#endif
+
+self.onmessage = async (msg) => {
+#if RUNTIME_DEBUG
+  console.log('pthread_esm_startup', msg.data.cmd);
+#endif
+  if (msg.data.cmd == 'load') {
+    // Until we initialize the runtime, queue up any further incoming messages
+    // that can arrive while the async import (await import below) is happening.
+    // For examples the `run` message often arrives right away before the import
+    // is complete.
+    let messageQueue = [msg];
+    self.onmessage = (e) => messageQueue.push(e);
+
+    // Now that we have the wasmMemory we can import the main program
+    globalThis.wasmMemory = msg.data.wasmMemory;
+    const prog = await import('./{{{ TARGET_JS_NAME }}}');
+
+    // Now that the import is completed the main program will have installed
+    // its own `onmessage` handler and replaced our handler.
+    // Now we can dispatch any queued messages to this new handler.
+    for (let msg of messageQueue) {
+      await self.onmessage(msg);
+    }
+
+    await prog.default()
+  }
+};

src/runtime_common.js

@@ -29,7 +29,7 @@ var readyPromiseResolve, readyPromiseReject;
 var wasmModuleReceived;
 #endif
 
-#if ENVIRONMENT_MAY_BE_NODE
+#if ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
   // Create as web-worker-like an environment as we can.
   var parentPort = worker_threads['parentPort'];
@@ -39,7 +39,7 @@ if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
     postMessage: (msg) => parentPort['postMessage'](msg),
   });
 }
-#endif // ENVIRONMENT_MAY_BE_NODE
+#endif // ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 #endif
 
 #if PTHREADS

src/runtime_init_memory.js

@@ -12,6 +12,14 @@
 // check for full engine support (use string 'subarray' to avoid closure compiler confusion)
 
 function initMemory() {
+#if WASM_ESM_INTEGRATION && PTHREADS
+  if (ENVIRONMENT_IS_PTHREAD) {
+    wasmMemory = globalThis.wasmMemory;
+    assert(wasmMemory);
+    updateMemoryViews();
+  }
+#endif
+
   {{{ runIfWorkerThread('return') }}}
 
 #if expectToReceiveOnModule('wasmMemory')

src/runtime_pthread.js

@@ -104,8 +104,10 @@ if (ENVIRONMENT_IS_PTHREAD) {
 #endif
         }
 
+#if !WASM_ESM_INTEGRATION
         wasmMemory = msgData.wasmMemory;
         updateMemoryViews();
+#endif
 
 #if LOAD_SOURCE_MAP
         wasmSourceMap = resetPrototype(WasmSourceMap, msgData.wasmSourceMap);
@@ -114,13 +116,15 @@ if (ENVIRONMENT_IS_PTHREAD) {
         wasmOffsetConverter = resetPrototype(WasmOffsetConverter, msgData.wasmOffsetConverter);
 #endif
 
+#if !WASM_ESM_INTEGRATION
 #if MINIMAL_RUNTIME
         // Pass the shared Wasm module in the Module object for MINIMAL_RUNTIME.
         Module['wasm'] = msgData.wasmModule;
         loadModule();
 #else
         wasmModuleReceived(msgData.wasmModule);
 #endif // MINIMAL_RUNTIME
+#endif
       } else if (cmd === 'run') {
 #if ASSERTIONS
         assert(msgData.pthread_ptr);

test/common.py

@@ -683,6 +683,8 @@ def also_with_modularize(f):
   @wraps(f)
   def metafunc(self, modularize, *args, **kwargs):
     if modularize:
+      if '-sWASM_ESM_INTEGRATION':
+        self.skipTest('also_with_modularize is not compatible with WASM_ESM_INTEGRATION')
       self.emcc_args += ['--extern-post-js', test_file('modularize_post_js.js'), '-sMODULARIZE']
     f(self, *args, **kwargs)
 
@@ -1177,8 +1179,6 @@ def setup_node_pthreads(self):
     self.emcc_args += ['-Wno-pthreads-mem-growth', '-pthread']
     if self.get_setting('MINIMAL_RUNTIME'):
       self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME')
-    if self.get_setting('WASM_ESM_INTEGRATION'):
-      self.skipTest('pthreads not yet supported with WASM_ESM_INTEGRATION')
     nodejs = self.get_nodejs()
     self.js_engines = [nodejs]
     self.node_args += shared.node_pthread_flags(nodejs)

test/test_core.py

@@ -7761,6 +7761,7 @@ def test_webidl(self, mode, allow_memory_growth):
   # PTHREAD_POOL_DELAY_LOAD=1 adds a pthreadPoolReady promise that users
   # can wait on for pthread initialization.
   @node_pthreads
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_ASYNC_COMPILATION')
   def test_embind_sync_if_pthread_delayed(self):
     self.set_setting('WASM_ASYNC_COMPILATION', 0)
     self.set_setting('PTHREAD_POOL_DELAY_LOAD', 1)
@@ -9283,6 +9284,7 @@ def test_pthread_unhandledrejection(self):
   @node_pthreads
   @no_wasm2js('wasm2js does not support PROXY_TO_PTHREAD (custom section support)')
   @also_with_modularize
+  @no_esm_integration('USE_OFFSET_CONVERTER')
   def test_pthread_offset_converter(self):
     self.set_setting('PROXY_TO_PTHREAD')
     self.set_setting('EXIT_RUNTIME')
@@ -9703,6 +9705,7 @@ def test_emscripten_async_load_script(self):
   @no_sanitize('sanitizers do not support WASM_WORKERS')
   @also_with_minimal_runtime
   @also_with_modularize
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
   def test_wasm_worker_hello(self):
     if self.is_wasm2js() and '-sMODULARIZE' in self.emcc_args:
       self.skipTest('WASM2JS + MODULARIZE + WASM_WORKERS is not supported')
@@ -9711,11 +9714,13 @@ def test_wasm_worker_hello(self):
 
   @node_pthreads
   @no_sanitize('sanitizers do not support WASM_WORKERS')
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
   def test_wasm_worker_malloc(self):
     self.do_run_in_out_file_test('wasm_worker/malloc_wasm_worker.c', emcc_args=['-sWASM_WORKERS'])
 
   @node_pthreads
   @no_sanitize('sanitizers do not support WASM_WORKERS')
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
   def test_wasm_worker_wait_async(self):
     self.do_runf('atomic/test_wait_async.c', emcc_args=['-sWASM_WORKERS'])
 

tools/link.py

@@ -810,8 +810,8 @@ def phase_linker_setup(options, linker_args):  # noqa: C901, PLR0912, PLR0915
       exit_with_error('WASM_ESM_INTEGRATION is not compatible with dynamic linking')
     if settings.ASYNCIFY:
       exit_with_error('WASM_ESM_INTEGRATION is not compatible with -sASYNCIFY')
-    if settings.WASM_WORKERS or settings.PTHREADS:
-      exit_with_error('WASM_ESM_INTEGRATION is not compatible with multi-threading')
+    if settings.WASM_WORKERS:
+      exit_with_error('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
     if settings.USE_OFFSET_CONVERTER:
       exit_with_error('WASM_ESM_INTEGRATION is not compatible with USE_OFFSET_CONVERTER')
     if not settings.WASM_ASYNC_COMPILATION:
@@ -2233,6 +2233,11 @@ def phase_final_emitting(options, target, js_target, wasm_target):
     support_target = unsuffixed(js_target) + '.support.mjs'
     move_file(final_js, support_target)
     create_esm_wrapper(js_target, support_target, wasm_target)
+    if settings.PTHREADS:
+      support_target = unsuffixed(js_target) + '.pthread.mjs'
+      pthread_code = building.read_and_preprocess(utils.path_from_root('src/pthread_esm_startup.mjs'), expand_macros=True)
+      write_file(support_target, pthread_code)
+      fix_es6_import_statements(support_target)
   else:
     move_file(final_js, js_target)