Add support for MODULARIZE with USE_PTHREADS

Author: juj

emcc.py

@@ -1081,6 +1081,15 @@ def check(input_file):
     if shared.Settings.MODULARIZE_INSTANCE:
       shared.Settings.MODULARIZE = 1
 
+    if shared.Settings.MODULARIZE:
+      assert not options.proxy_to_worker, '-s MODULARIZE=1 and -s MODULARIZE_INSTANCE=1 are not compatible with --proxy-to-worker (if you want to run in a worker with -s MODULARIZE=1, you likely want to do the worker side setup manually)'
+
+    if not shared.Settings.EXPORT_NAME:
+      # If we are building directly to .html with -s MODULARIZE=1 (but not -s MODULARIZE_INSTANCE=1), default to exporting
+      # Emscripten code under function EmscriptenCode() so that it does not collide with the Module object. (When not modularizing,
+      # or if building with MODULARIZE_INSTANCE, or if building to .js manually, there is no conflict, so default to 'Module' in those cases)
+      shared.Settings.EXPORT_NAME = 'EmscriptenCode' if final_suffix == '.html' and shared.Settings.MODULARIZE and not shared.Settings.MODULARIZE_INSTANCE else 'Module'
+
     if shared.Settings.EMULATE_FUNCTION_POINTER_CASTS:
       shared.Settings.ALIASING_FUNCTION_POINTERS = 0
 
@@ -1186,11 +1195,6 @@ def check(input_file):
         exit_with_error('USE_PTHREADS=2 is not longer supported')
       if shared.Settings.ALLOW_MEMORY_GROWTH:
         exit_with_error('Memory growth is not yet supported with pthreads')
-      if shared.Settings.MODULARIZE:
-        # currently worker.js uses the global namespace, so it's setting of
-        # ENVIRONMENT_IS_PTHREAD is not picked up, in addition to all the other
-        # modifications it performs.
-        exit_with_error('MODULARIZE is not yet supported with pthreads')
       # UTF8Decoder.decode doesn't work with a view of a SharedArrayBuffer
       shared.Settings.TEXTDECODER = 0
       options.js_libraries.append(shared.path_from_root('src', 'library_pthread.js'))
@@ -1225,6 +1229,21 @@ def check(input_file):
       ]
 
     if shared.Settings.USE_PTHREADS:
+      if shared.Settings.MODULARIZE:
+        # MODULARIZE+USE_PTHREADS mode requires extra exports out to Module so that worker.js
+        # can access them:
+
+        # general threading variables:
+        shared.Settings.EXPORTED_RUNTIME_METHODS += ['PThread', 'ExitStatus']
+
+        # pthread stack setup:
+        shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$establishStackSpaceInJsModule']
+        shared.Settings.EXPORTED_FUNCTIONS += ['establishStackSpaceInJsModule']
+
+        # stack check:
+        if shared.Settings.STACK_OVERFLOW_CHECK:
+          shared.Settings.EXPORTED_RUNTIME_METHODS += ['writeStackCookie', 'checkStackCookie']
+
       if shared.Settings.LINKABLE:
         exit_with_error('-s LINKABLE=1 is not supported with -s USE_PTHREADS>0!')
       if shared.Settings.SIDE_MODULE:
@@ -1960,8 +1979,9 @@ def repl(m):
 
       if shared.Settings.USE_PTHREADS:
         target_dir = os.path.dirname(os.path.abspath(target))
-        shutil.copyfile(shared.path_from_root('src', 'worker.js'),
-                        os.path.join(target_dir, shared.Settings.PTHREAD_WORKER_FILE))
+        worker_output = os.path.join(target_dir, shared.Settings.PTHREAD_WORKER_FILE)
+        with open(worker_output, 'w') as f:
+          f.write(shared.read_and_preprocess(shared.path_from_root('src', 'worker.js'), expand_macros=True))
 
       # Generate the fetch-worker.js script for multithreaded emscripten_fetch() support if targeting pthreads.
       if shared.Settings.FETCH and shared.Settings.USE_PTHREADS:
@@ -2942,18 +2962,28 @@ def un_src(self):
     """Use this if you want to modify the script and need it to be inline."""
     if self.src is None:
       return
+    run_module = ''
+    if shared.Settings.MODULARIZE and not shared.Settings.MODULARIZE_INSTANCE:
+      run_module = 'script.onload = function() { %s(Module); };' % shared.Settings.EXPORT_NAME
+
     self.inline = '''
           var script = document.createElement('script');
           script.src = "%s";
+          %s
           document.body.appendChild(script);
-''' % self.src
+''' % (self.src, run_module)
     self.src = None
 
   def replacement(self):
+    onload = ''
+    if shared.Settings.MODULARIZE and not shared.Settings.MODULARIZE_INSTANCE:
+      assert shared.Settings.EXPORT_NAME != 'Module', 'EXPORT_NAME cannot be called "Module"!'
+      onload = 'onload="%s(Module)"' % shared.Settings.EXPORT_NAME
+
     """Returns the script tag to replace the {{{ SCRIPT }}} tag in the target"""
     assert (self.src or self.inline) and not (self.src and self.inline)
     if self.src:
-      return '<script async type="text/javascript" src="%s"></script>' % quote(self.src)
+      return '<script async type="text/javascript" src="%s" %s></script>' % (quote(self.src), onload)
     else:
       return '<script>\n%s\n</script>' % self.inline
 

src/library_pthread.js

@@ -356,12 +356,13 @@ var LibraryPThread = {
           // it could load up the same file. In that case, developer must either deliver the Blob
           // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can
           // independently load up the same main application file.
-          urlOrBlob: Module['mainScriptUrlOrBlob'] || currentScriptUrl,
+          urlOrBlob: Module['mainScriptUrlOrBlob'] || _scriptDir,
 #if WASM
           wasmMemory: wasmMemory,
           wasmModule: wasmModule,
 #else
           buffer: HEAPU8.buffer,
+          asmJsUrlOrBlob: Module["asmJsUrlOrBlob"],
 #endif
           tempDoublePtr: tempDoublePtr,
           TOTAL_MEMORY: TOTAL_MEMORY,
@@ -1181,6 +1182,13 @@ var LibraryPThread = {
 #endif
     return func.apply(null, callArgs);
   },
+
+#if MODULARIZE
+  $establishStackSpaceInJsModule: function(stackBase, stackMax) {
+    STACK_BASE = STACKTOP = stackBase;
+    STACK_MAX = stackMax;
+  },
+#endif
 };
 
 autoAddDeps(LibraryPThread, '$PThread');

src/modules.js

@@ -426,6 +426,18 @@ function exportRuntime() {
     'getTempRet0',
     'setTempRet0',
   ];
+  if (MODULARIZE) {
+    // In MODULARIZE=1 mode, the following functions need to be exported out to Module for worker.js to access.
+    if (STACK_OVERFLOW_CHECK) {
+      runtimeElements.push('writeStackCookie');
+      runtimeElements.push('checkStackCookie');
+      runtimeElements.push('abortStackOverflow');
+    }
+    if (USE_PTHREADS) {
+      runtimeElements.push('PThread');
+      runtimeElements.push('ExitStatus');
+    }
+  }
   if (SUPPORT_BASE64_EMBEDDING) {
     runtimeElements.push('intArrayFromBase64');
     runtimeElements.push('tryParseAsDataURI');

src/parseTools.js

@@ -1477,6 +1477,34 @@ function makeStaticString(string) {
   return '(stringToUTF8("' + string + '", ' + ptr + ', ' + len + '), ' + ptr + ')';
 }
 
+// Generates access to module exports variable in pthreads worker.js
+function makeAsmExportAccessInPthread(variable) {
+  if (MODULARIZE) {
+    return "Module['" + variable + "']" // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case.
+  } else {
+    return EXPORT_NAME + "['" + variable + "']"
+  }
+}
+
+// Generates access to a global scope variable in pthreads worker.js
+function makeAsmGlobalAccessInPthread(variable) {
+  if (MODULARIZE) {
+    return "Module['" + variable + "']" // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case.
+  } else {
+    return variable
+  }
+}
+
+// Generates access to both global scope variable and exported Module variable, e.g. "Module['foo'] = foo" or just plain "foo" depending on if we are MODULARIZEing.
+// Used the be able to initialize both variables at the same time.
+function makeAsmExportAndGlobalAccessInPthread(variable) {
+  if (MODULARIZE) {
+    return "Module['" + variable + "'] = " + variable // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case.
+  } else {
+    return variable
+  }
+}
+
 // Some things, like the dynamic and stack bases, will be computed later and
 // applied. Return them as {{{ STR }}} for that replacing later.
 

src/preamble.js

@@ -389,14 +389,6 @@ assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned');
 }
 #endif
 
-#if USE_PTHREADS
-if (ENVIRONMENT_IS_PTHREAD) {
-#if SEPARATE_ASM != 0
-  importScripts('{{{ SEPARATE_ASM }}}'); // load the separated-out asm.js
-#endif
-}
-#endif
-
 #if EMTERPRETIFY
 function abortStackOverflowEmterpreter() {
   abort("Emterpreter stack overflow! Decrease the recursion level or increase EMT_STACK_MAX in tools/emterpretify.py (current value " + EMT_STACK_MAX + ").");
@@ -1059,7 +1051,13 @@ function createWasm(env) {
 
 Module['asm'] = function(global, env, providedBuffer) {
   // memory was already allocated (so js could use the buffer)
-  env['memory'] = wasmMemory;
+  env['memory'] = wasmMemory
+#if MODULARIZE && USE_PTHREADS
+  // Pthreads assign wasmMemory in their worker startup. In MODULARIZE mode, they cannot assign inside the
+  // Module scope, so lookup via Module as well.
+  || Module['wasmMemory']
+#endif
+  ;
   // import table
   env['table'] = wasmTable = new WebAssembly.Table({
     'initial': {{{ getQuoted('WASM_TABLE_SIZE') }}},

src/settings.js

@@ -936,7 +936,10 @@ var EXPLICIT_ZEXT = 0;
 
 // Global variable to export the module as for environments without a
 // standardized module loading system (e.g. the browser and SM shell).
-var EXPORT_NAME = 'Module';
+// Default EXPORT_NAME is 'Module', but if building directly to .html
+// with -s MODULARIZE=1 but not with -s MODULARIZE_INSTANCE=1, then default EXPORT_NAME
+// is 'EmscriptenCode' to not conflict with the default provided Module object.
+var EXPORT_NAME = '';
 
 // When set to 0, we do not emit eval() and new Function(), which disables some functionality
 // (causing runtime errors if attempted to be used), but allows the emitted code to be

src/shell.js

@@ -81,11 +81,34 @@ if (Module['ENVIRONMENT']) {
 // 2) We could be the application main() thread proxied to worker. (with Emscripten -s PROXY_TO_WORKER=1) (ENVIRONMENT_IS_WORKER == true, ENVIRONMENT_IS_PTHREAD == false)
 // 3) We could be an application pthread running in a worker. (ENVIRONMENT_IS_WORKER == true and ENVIRONMENT_IS_PTHREAD == true)
 #if USE_PTHREADS
-var ENVIRONMENT_IS_PTHREAD;
-if (!ENVIRONMENT_IS_PTHREAD) ENVIRONMENT_IS_PTHREAD = false; // ENVIRONMENT_IS_PTHREAD=true will have been preset in worker.js. Make it false in the main runtime thread.
-var PthreadWorkerInit; // Collects together variables that are needed at initialization time for the web workers that host pthreads.
-if (!ENVIRONMENT_IS_PTHREAD) PthreadWorkerInit = {};
-var currentScriptUrl = (typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined;
+
+if (typeof ENVIRONMENT_IS_PTHREAD === 'undefined') {
+  // ENVIRONMENT_IS_PTHREAD=true will have been preset in worker.js. Make it false in the main runtime thread.
+  // N.B. this line needs to appear without 'var' keyword to avoid 'var hoisting' from occurring. (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/var)
+  ENVIRONMENT_IS_PTHREAD = false;
+  var PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads.
+}
+#if MODULARIZE
+else {
+  // Grab imports from the pthread to local scope.
+  var buffer = {{{EXPORT_NAME}}}.buffer;
+  var tempDoublePtr = {{{EXPORT_NAME}}}.tempDoublePtr;
+  var TOTAL_MEMORY = {{{EXPORT_NAME}}}.TOTAL_MEMORY;
+  var STATICTOP = {{{EXPORT_NAME}}}.STATICTOP;
+  var DYNAMIC_BASE = {{{EXPORT_NAME}}}.DYNAMIC_BASE;
+  var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}.DYNAMICTOP_PTR;
+  var PthreadWorkerInit = {{{EXPORT_NAME}}}.PthreadWorkerInit;
+  // Note that not all runtime fields are imported above. Values for STACK_BASE, STACKTOP and STACK_MAX are not yet known at worker.js load time.
+  // These will be filled in at pthread startup time (the 'run' message for a pthread - pthread start establishes the stack frame)
+}
+#endif
+
+#if !MODULARIZE
+// In MODULARIZE mode _scriptDir needs to be captured already at the very top of the page immediately when the page is parsed, so it is generated there
+// before the page load. In non-MODULARIZE modes generate it here.
+var _scriptDir = (typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined;
+#endif
+
 #endif // USE_PTHREADS
 
 // `/` should be present at the end if `scriptDirectory` is not empty