[wasm2js] Remove all handling for external memory file

We have a lot of support code and complexity in emscripten for handling
of external memory files.  However, its only ever used in wasm2js mode
which is a legacy mode.  The only reason we continue to support it here
IIUC is because its slightly more space efficient than embedding the
data as base64.

For small programs like hello_world this is an over codesize win.  For
larger programs there is a regression in overall size in proportion to
the amount of static data in the program.

Author: sbc100

ChangeLog.md

@@ -21,6 +21,9 @@ See docs/process.md for more on how version tagging works.
 3.1.55 (in development)
 -----------------------
 - Update sdl2-mixer port from 2.6.0 to 2.8.0
+- Emscripten no longer supported extracting static data and serving it as a
+  separate `.mem` data file.  The feature was already only available under
+  wasm2js (`-sWASM=0`) so this change will only effect users of this setting.
 
 3.1.54 - 02/15/24
 -----------------
@@ -58,7 +61,6 @@ See docs/process.md for more on how version tagging works.
 - Allow comments in response files. Any line starting with `#` is now ignored.
   This is useful when listing exported symbols. (#21330)
 
-
 3.1.53 - 01/29/24
 -----------------
 - The llvm version that emscripten uses was updated to 19.0.0 trunk. (#21165)

emcc.py

@@ -145,7 +145,6 @@ def __init__(self):
     self.emrun = False
     self.cpu_profiler = False
     self.memory_profiler = False
-    self.memory_init_file = None
     self.use_preload_cache = False
     self.use_preload_plugins = False
     self.valid_abspaths = []
@@ -1316,7 +1315,7 @@ def consume_arg_file():
       ports.show_ports()
       should_exit = True
     elif check_arg('--memory-init-file'):
-      options.memory_init_file = int(consume_arg())
+      exit_with_error('--memory-init-file is no longer supported')
     elif check_flag('--proxy-to-worker'):
       settings_changes.append('PROXY_TO_WORKER=1')
     elif check_arg('--valid-abspath'):

src/postamble_minimal.js

@@ -221,13 +221,6 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
   updateMemoryViews();
 #endif
 
-#if !MEM_INIT_IN_WASM && !SINGLE_FILE
-#if ASSERTIONS
-  if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
-#endif
-  HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
-#endif
-
   initRuntime(wasmExports);
 #if PTHREADS
   // Export Wasm module for pthread creation to access.

src/preamble.js

@@ -990,9 +990,6 @@ function createWasm() {
 #endif
     updateMemoryViews();
 #endif
-#if !MEM_INIT_IN_WASM
-    runMemoryInitializer();
-#endif
 
 #if '$wasmTable' in addedLibraryItems && !RELOCATABLE
     wasmTable = wasmExports['__indirect_function_table'];
@@ -1140,88 +1137,6 @@ function getCompilerSetting(name) {
 }
 #endif // RETAIN_COMPILER_SETTINGS
 
-#if !MEM_INIT_IN_WASM
-var memoryInitializer = <<< MEM_INITIALIZER >>>;
-
-function runMemoryInitializer() {
-#if PTHREADS
-  if (ENVIRONMENT_IS_PTHREAD) return;
-#endif
-  if (!isDataURI(memoryInitializer)) {
-    memoryInitializer = locateFile(memoryInitializer);
-  }
-  if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
-    var data = readBinary(memoryInitializer);
-    HEAPU8.set(data, {{{ GLOBAL_BASE }}});
-  } else {
-    addRunDependency('memory initializer');
-    var applyMemoryInitializer = (data) => {
-      if (data.byteLength) data = new Uint8Array(data);
-#if ASSERTIONS
-      for (var i = 0; i < data.length; i++) {
-        assert(HEAPU8[{{{ GLOBAL_BASE }}} + i] === 0, "area for memory initializer should not have been touched before it's loaded");
-      }
-#endif
-      HEAPU8.set(data, {{{ GLOBAL_BASE }}});
-      // Delete the typed array that contains the large blob of the memory initializer request response so that
-      // we won't keep unnecessary memory lying around. However, keep the XHR object itself alive so that e.g.
-      // its .status field can still be accessed later.
-      if (Module['memoryInitializerRequest']) delete Module['memoryInitializerRequest'].response;
-      removeRunDependency('memory initializer');
-    };
-    var doBrowserLoad = () => {
-      readAsync(memoryInitializer, applyMemoryInitializer, () => {
-        var e = new Error('could not load memory initializer ' + memoryInitializer);
-#if MODULARIZE
-          readyPromiseReject(e);
-#else
-          throw e;
-#endif
-      });
-    };
-#if SUPPORT_BASE64_EMBEDDING
-    var memoryInitializerBytes = tryParseAsDataURI(memoryInitializer);
-    if (memoryInitializerBytes) {
-      applyMemoryInitializer(memoryInitializerBytes.buffer);
-    } else
-#endif
-    if (Module['memoryInitializerRequest']) {
-      // a network request has already been created, just use that
-      var useRequest = () => {
-        var request = Module['memoryInitializerRequest'];
-        var response = request.response;
-        if (request.status !== 200 && request.status !== 0) {
-#if SUPPORT_BASE64_EMBEDDING
-          var data = tryParseAsDataURI(Module['memoryInitializerRequestURL']);
-          if (data) {
-            response = data.buffer;
-          } else {
-#endif
-            // If you see this warning, the issue may be that you are using locateFile and defining it in JS. That
-            // means that the HTML file doesn't know about it, and when it tries to create the mem init request early, does it to the wrong place.
-            // Look in your browser's devtools network console to see what's going on.
-            console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer);
-            doBrowserLoad();
-            return;
-#if SUPPORT_BASE64_EMBEDDING
-          }
-#endif
-        }
-        applyMemoryInitializer(response);
-      };
-      if (Module['memoryInitializerRequest'].response) {
-        setTimeout(useRequest, 0); // it's already here; but, apply it asynchronously
-      } else {
-        Module['memoryInitializerRequest'].addEventListener('load', useRequest); // wait for it
-      }
-    } else {
-      // fetch it from the network ourselves
-      doBrowserLoad();
-    }
-  }
-}
-#endif // MEM_INIT_IN_WASM == 0
-
 #if MAIN_MODULE && ASYNCIFY
 // With MAIN_MODULE + ASYNCIFY the normal method of placing stub functions in
 // wasmImports for as-yet-undefined symbols doesn't work since ASYNCIFY then

src/settings_internal.js

@@ -143,11 +143,6 @@ var AUDIO_WORKLET_FILE = '';
 // Base URL the source mapfile, if relevant
 var SOURCE_MAP_BASE = '';
 
-// When this is false we use an external memory init file
-// See --memory-init-file.  When not using wasm2js this flag is ignored, and
-// this setting will always be true.
-var MEM_INIT_IN_WASM = true;
-
 // If set to 1, src/base64Utils.js will be included in the bundle.
 // This is set internally when needed (SINGLE_FILE)
 var SUPPORT_BASE64_EMBEDDING = false;

src/shell_minimal.js

@@ -82,7 +82,7 @@ if (ENVIRONMENT_IS_NODE && ENVIRONMENT_IS_SHELL) {
 #endif
 
 #if !SINGLE_FILE
-#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
+#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && !WASM2JS) || WASM == 2)
 // Wasm or Wasm2JS loading:
 
 if (ENVIRONMENT_IS_NODE) {
@@ -95,13 +95,10 @@ if (ENVIRONMENT_IS_NODE) {
   Module['wasm'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.wasm');
 #endif
 #endif
-#if !MEM_INIT_IN_WASM
-  Module['mem'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.mem');
-#endif
 }
 #endif
 
-#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
+#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && !WASM2JS) || WASM == 2)
 if (ENVIRONMENT_IS_SHELL) {
 #if WASM == 2
   if (typeof WebAssembly != 'undefined') Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
@@ -111,9 +108,6 @@ if (ENVIRONMENT_IS_SHELL) {
   Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
 #endif
 #endif
-#if !MEM_INIT_IN_WASM
-  Module['mem'] = read('{{{ TARGET_BASENAME }}}.mem', 'binary');
-#endif
 }
 #endif
 

test/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,8 @@
 {
-  "a.html": 567,
-  "a.html.gz": 379,
-  "a.js": 17800,
-  "a.js.gz": 7981,
-  "a.mem": 3123,
-  "a.mem.gz": 2693,
-  "total": 21490,
-  "total_gz": 11053
+  "a.html": 354,
+  "a.html.gz": 266,
+  "a.js": 22323,
+  "a.js.gz": 11632,
+  "total": 22677,
+  "total_gz": 11898
 }

test/code_size/hello_webgl_wasm2js.json

@@ -1,10 +1,8 @@
 {
-  "a.html": 567,
-  "a.html.gz": 379,
-  "a.js": 17272,
-  "a.js.gz": 7814,
-  "a.mem": 3123,
-  "a.mem.gz": 2693,
-  "total": 20962,
-  "total_gz": 10886
+  "a.html": 354,
+  "a.html.gz": 266,
+  "a.js": 21794,
+  "a.js.gz": 11450,
+  "total": 22148,
+  "total_gz": 11716
 }

test/code_size/hello_world_wasm2js.js

@@ -1,19 +1,30 @@
-var c = Module, g, h, k = new TextDecoder("utf8"), l;
+var d = Module, g, h, k = new TextDecoder("utf8"), l;
 
-function d(b) {
-    this.exports = function(f) {
-        function m(e) {
-            e.set = function(a, n) {
-                this[a] = n;
+function e(b) {
+    this.exports = function(r) {
+        function u(c) {
+            c.set = function(a, f) {
+                this[a] = f;
             };
-            e.get = function(a) {
+            c.get = function(a) {
                 return this[a];
             };
-            return e;
+            return c;
         }
-        return function(e) {
-            var a = new ArrayBuffer(16777216), n = e.a.a;
-            e = m([]);
+        function x(c, a, f) {
+            for (var v, p = 0, t = a, w = f.length, y = a + (3 * w >> 2) - ("=" == f[w - 2]) - ("=" == f[w - 1]); p < w; p += 4) a = m[f.charCodeAt(p + 1)], 
+            v = m[f.charCodeAt(p + 2)], c[t++] = m[f.charCodeAt(p)] << 2 | a >> 4, t < y && (c[t++] = a << 4 | v >> 2), 
+            t < y && (c[t++] = v << 6 | m[f.charCodeAt(p + 3)]);
+        }
+        for (var q, m = new Uint8Array(123), n = 25; 0 <= n; --n) m[48 + n] = 52 + n, m[65 + n] = n, 
+        m[97 + n] = 26 + n;
+        m[43] = 62;
+        m[47] = 63;
+        return function(c) {
+            var a = new ArrayBuffer(16777216), f = new Uint8Array(a), v = c.a.a;
+            q = f;
+            x(q, 1024, "aGVsbG8h");
+            c = u([]);
             return {
                 b: Object.create(Object.prototype, {
                     grow: {},
@@ -24,41 +35,40 @@ function d(b) {
                     }
                 }),
                 c: function() {},
-                d: function(p, q) {
-                    n(1024);
+                d: function(p, t) {
+                    v(1024);
                     return 0;
                 },
-                e: e
+                e: c
             };
-        }(f);
+        }(r);
     }(b);
 }
 
-(function(b, f) {
+(function(b, r) {
     return {
-        then: function(m) {
-            m({
-                instance: new d(f)
+        then: function(u) {
+            u({
+                instance: new e(r)
             });
         }
     };
-})(c.wasm, {
+})(d.wasm, {
     a: {
         a: b => {
-            var f = console, m = f.log;
+            var r = console, u = r.log;
             if (b) {
-                for (var e = b + void 0, a = b; !(a >= e) && g[a]; ) ++a;
-                b = k.decode(g.subarray(b, a));
+                for (var x = b + void 0, q = b; !(q >= x) && g[q]; ) ++q;
+                b = k.decode(g.subarray(b, q));
             } else b = "";
-            m.call(f, b);
+            u.call(r, b);
         }
     }
 }).then((b => {
     b = b.instance.exports;
     l = b.d;
     h = b.b;
     g = new Uint8Array(h.buffer);
-    g.set(new Uint8Array(c.mem), 1024);
     b.c();
     l();
 }));

test/code_size/hello_world_wasm2js.json

@@ -1,10 +1,8 @@
 {
-  "a.html": 671,
-  "a.html.gz": 430,
-  "a.js": 708,
-  "a.js.gz": 444,
-  "a.mem": 6,
-  "a.mem.gz": 32,
-  "total": 1385,
-  "total_gz": 906
+  "a.html": 323,
+  "a.html.gz": 253,
+  "a.js": 1060,
+  "a.js.gz": 636,
+  "total": 1383,
+  "total_gz": 889
 }

test/common.py

@@ -847,17 +847,6 @@ def setup_node_pthreads(self):
     self.js_engines = [nodejs]
     self.node_args += shared.node_pthread_flags(nodejs)
 
-  def uses_memory_init_file(self):
-    if self.get_setting('SIDE_MODULE') or self.is_wasm():
-      return False
-
-    if '--memory-init-file' in self.emcc_args:
-      return int(self.emcc_args[self.emcc_args.index('--memory-init-file') + 1])
-
-    # side modules handle memory differently; binaryen puts the memory in the wasm module
-    opt_supports = any(opt in self.emcc_args for opt in ('-O2', '-O3', '-Os', '-Oz'))
-    return opt_supports
-
   def set_temp_dir(self, temp_dir):
     self.temp_dir = temp_dir
     self.canonical_temp_dir = get_canonical_temp_dir(self.temp_dir)
@@ -1104,11 +1093,6 @@ def build(self, filename, libraries=None, includes=None, force_c=False, js_outfi
     self.run_process(cmd, stderr=self.stderr_redirect if not DEBUG else None)
     self.assertExists(output)
 
-    if js_outfile and self.uses_memory_init_file():
-      src = read_file(output)
-      # side memory init file, or an empty one in the js
-      assert ('/* memory initializer */' not in src) or ('/* memory initializer */ allocate([]' in src)
-
     return output
 
   def get_func(self, src, name):