expand loader docs

README.md

@@ -58,8 +58,10 @@ main()
         - [Async module loader](#async-module-loader)
         - [Async on host, sync in QuickJS](#async-on-host-sync-in-quickjs)
     - [Testing your code](#testing-your-code)
-    - [Using in the browser without a build step](#using-in-the-browser-without-a-build-step)
-    - [quickjs-emscripten-core, variants, and advanced packaging](#quickjs-emscripten-core-variants-and-advanced-packaging)
+    - [Packaging](#packaging)
+      - [Reducing package size](#reducing-package-size)
+      - [WebAssembly loading](#webassembly-loading)
+      - [Using in the browser without a build step](#using-in-the-browser-without-a-build-step)
     - [Debugging](#debugging)
     - [Supported Platforms](#supported-platforms)
     - [More Documentation](#more-documentation)
@@ -515,7 +517,94 @@ For more testing examples, please explore the typescript source of [quickjs-emsc
 [debug_sync]: https://github.com/justjake/quickjs-emscripten/blob/main/doc/quickjs-emscripten/exports.md#debug_sync
 [testquickjswasmmodule]: https://github.com/justjake/quickjs-emscripten/blob/main/doc/quickjs-emscripten/classes/TestQuickJSWASMModule.md
 
-### Using in the browser without a build step
+### Packaging
+
+The main `quickjs-emscripten` package includes several build variants of the WebAssembly module:
+
+- `RELEASE...` build variants should be used in production. They offer better performance and smaller file size compared to `DEBUG...` build variants.
+  - `RELEASE_SYNC`: This is the default variant used when you don't explicitly provide one. It offers the fastest performance and smallest file size.
+  - `RELEASE_ASYNC`: The default variant if you need [asyncify][] magic, which comes at a performance cost. See the asyncify docs for details.
+- `DEBUG...` build variants can be helpful during development and testing. They include source maps and assertions for catching bugs in your code. We recommend running your tests with _both_ a debug build variant and the release build variant you'll use in production.
+  - `DEBUG_SYNC`: Instrumented to detect memory leaks, in addition to assertions and source maps.
+  - `DEBUG_ASYNC`: An [asyncify][] variant with source maps.
+
+To use a variant, call `newQuickJSWASMModule` or `newQuickJSAsyncWASMModule` with the variant object. These functions return a promise that resolves to a [QuickJSWASMModule](./doc/quickjs-emscripten/classes/QuickJSWASMModule.md), the same as `getQuickJS`.
+
+```typescript
+import {
+  newQuickJSWASMModule, newQuickJSAsyncWASMModule,
+  RELEASE_SYNC, DEBUG_SYNC,
+  RELEASE_ASYNC, DEBUG_ASYNC,
+} from 'quickjs-emscripten'
+
+const QuickJSReleaseSync = await newQuickJSWASMModule(RELEASE_SYNC)
+const QuickJSDebugSync = await newQuickJSWASMModule(DEBUG_SYNC)
+const QuickJSReleaseAsync = await newQuickJSAsyncWASMModule(RELEASE_ASYNC)
+const QuickJSDebugAsync = await newQuickJSAsyncWASMModule(DEBUG_ASYNC)
+
+for (const quickjs of [QuickJSReleaseSync, QuickJSDebugSync, QuickJSReleaseAsync, QuickJSDebugAsync]) {
+  const vm = quickjs.newContext()
+  const result = vm.unwrapResult(vm.evalCode("1 + 1")).consume(vm.getNumber)
+  console.log(result)
+  vm.dispose()
+  quickjs.dispose()
+}
+```
+
+#### Reducing package size
+
+Including 4 different copies of the WebAssembly module in the main package gives it an install size of [about 9.04mb](https://packagephobia.com/result?p=quickjs-emscripten). If you're building a CLI package or library of your own, or otherwise don't need to include 4 different variants in your `node_modules`, you can switch to the `quickjs-emscripten-core` package, which contains only the Javascript code for this library, and install one (or more) variants a-la-carte as separate packages.
+
+The most minimal setup would be to install `quickjs-emscripten-core` and `@jitl/quickjs-wasmfile-release-sync`:
+
+```bash
+yarn add quickjs-emscripten-core @jitl/quickjs-wasmfile-release-sync
+du -h node_modules
+# -> 
+```
+
+Then, you can use `newQuickJSWASMModuleFromVariant` to create a QuickJS module from the variant (see [the minimal example][minimal]):
+
+```typescript
+// src/quickjs.mjs
+import { newQuickJSWASMModuleFromVariant } from 'quickjs-emscripten-core'
+import RELEASE_SYNC from '@jitl/quickjs-wasmfile-release-sync'
+export const QuickJS = await newQuickJSWASMModuleFromVariant(RELEASE_SYNC)
+
+// src/app.mjs
+import { QuickJS } from './quickjs.mjs'
+console.log(QuickJS.evalCode("1 + 1"))
+```
+
+See the [documentation of quickjs-emscripten-core][core] for more details and the list of variant packages.
+
+[core]: https://github.com/justjake/quickjs-emscripten/blob/main/doc/quickjs-emscripten-core/README.md
+
+#### WebAssembly loading
+
+To run QuickJS, we need to load a WebAssembly module into the host Javascript runtime's memory (usually as an ArrayBuffer or TypedArray) and [compile it](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/instantiate_static) to a [WebAssembly.Module](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Module). This means we need to find the file path or URI of the WebAssembly module, and then read it using an API like `fetch` (browser) or `fs.readFile` (NodeJS). `quickjs-emscripten` tries to handle this automatically using patterns like `new URL('./local-path', import.meta.url)` that work in the browser or are handled automatically by bundlers, or `__dirname` in NodeJS, but you may need to configure this manually if these don't work in your environment, or you want more control about how the WebAssembly module is loaded.
+
+To customize the loading of an existing variant, create a new variant with your loading settings using `newVariant`, passing [CustomizeVariantOptions][newVariant]. For example, you need to customize loading in Cloudflare Workers (see [the full example][cloudflare]).
+
+```typescript
+import { newQuickJSWASMModule, DEBUG_SYNC as baseVariant, newVariant } from 'quickjs-emscripten';
+import cloudflareWasmModule from './DEBUG_SYNC.wasm';
+import cloudflareWasmModuleSourceMap from './DEBUG_SYNC.wasm.map.txt';
+
+/**
+ * We need to make a new variant that directly passes the imported WebAssembly.Module
+ * to Emscripten. Normally we'd load the wasm file as bytes from a URL, but
+ * that's forbidden in Cloudflare workers.
+ */
+const cloudflareVariant = newVariant(baseVariant, {
+  wasmModule: cloudflareWasmModule,
+  wasmSourceMapData: cloudflareWasmModuleSourceMap,
+});
+```
+
+[newVariant]: https://github.com/justjake/quickjs-emscripten/blob/main/doc/quickjs-emscripten/interfaces/CustomizeVariantOptions.md
+
+#### Using in the browser without a build step
 
 You can use quickjs-emscripten directly from an HTML file in two ways:
 
@@ -548,16 +637,6 @@ You can use quickjs-emscripten directly from an HTML file in two ways:
    </script>
    ```
 
-### quickjs-emscripten-core, variants, and advanced packaging
-
-Them main `quickjs-emscripten` package includes several build variants of the WebAssembly module.
-If these variants are too large for you, you can instead use the `quickjs-emscripten-core` package,
-and manually select your own build variant.
-
-See the [documentation of quickjs-emscripten-core][core] for more details.
-
-[core]: https://github.com/justjake/quickjs-emscripten/blob/main/doc/quickjs-emscripten-core/README.md
-
 ### Debugging
 
 - Switch to a DEBUG build variant of the WebAssembly module to see debug log messages from the C part of this library:
@@ -604,7 +683,7 @@ See the [documentation of quickjs-emscripten-core][core] for more details.
   - Edge 79+
   - Safari 11.1+
   - Firefox 58+
-- NodeJS: requires v16.0.0 or later for WebAssembly compatibility. Tested with node@18. See the [node-typescript example][tsx-example].
+- NodeJS: requires v16.0.0 or later for WebAssembly compatibility. Tested with node@18. See the [node-typescript][tsx-example] and [node-minimal][minimal] examples.
 - Typescript: tested with typescript@4.5.5 and typescript@5.3.3. See the [node-typescript example][tsx-example].
 - Vite: tested with vite@5.0.10. See the [Vite/Vue example][vite].
 - Create react app: tested with react-scripts@5.0.1. See the [create-react-app example][cra].
@@ -619,6 +698,7 @@ See the [documentation of quickjs-emscripten-core][core] for more details.
 [cra]: https://github.com/justjake/quickjs-emscripten/blob/main/examples/create-react-app
 [cloudflare]: https://github.com/justjake/quickjs-emscripten/blob/main/examples/cloudflare-workers
 [tsx-example]: https://github.com/justjake/quickjs-emscripten/blob/main/examples/node-typescript
+[minimal]: https://github.com/justjake/quickjs-emscripten/blob/main/examples/node-minimal
 
 ### More Documentation
 

examples/node-minimal/main.mjs

@@ -0,0 +1,2 @@
+import { QuickJS } from './quickjs.mjs'
+console.log(QuickJS.evalCode("1 + 1"))

examples/node-minimal/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "quickjs-emscripten-minimal-smoketest",
+  "version": "1.0.0",
+  "type": "commonjs",
+  "main": "main.mjs",
+  "license": "MIT",
+  "private": true,
+  "scripts": {
+    "start": "node ./main.mjs"
+  },
+  "dependencies": {
+    "@jitl/quickjs-ffi-types": "file:../../build/tar/@jitl-quickjs-ffi-types.tgz",
+    "@jitl/quickjs-wasmfile-release-sync": "file:../../build/tar/@jitl-quickjs-wasmfile-release-sync.tgz",
+    "quickjs-emscripten-core": "file:../../build/tar/quickjs-emscripten-core.tgz"
+  }
+}

examples/node-minimal/quickjs.mjs

@@ -0,0 +1,7 @@
+import { newQuickJSWASMModuleFromVariant } from 'quickjs-emscripten-core'
+import RELEASE_SYNC from '@jitl/quickjs-wasmfile-release-sync'
+
+/**
+ * Export the QuickJSWASMModule instance as a singleton.
+ */
+export const QuickJS = await newQuickJSWASMModuleFromVariant(RELEASE_SYNC)

packages/quickjs-emscripten-core/src/from-variant.ts

@@ -127,7 +127,25 @@ export interface CustomizeVariantOptions {
   wasmSourceMapLocation?: string
   /** If given, we will provide the source map to Emscripten directly. This may only be respected if wasmModule is also provided. */
   wasmSourceMapData?: OrLoader<string | SourceMapData>
-  /** Provide a locateFile callback, see {@link EmscriptenModuleLoaderOptions#locateFile} */
+  /**
+   * If set, this method will be called when the runtime needs to load a file,
+   * such as a .wasm WebAssembly file, .mem memory init file, or a file
+   * generated by the file packager.
+   *
+   * The function receives two parameters:
+   *
+   * - `fileName`, the relative path to the file as configured in build
+   * process, eg `"emscripten-module.wasm"`.
+   * - `prefix` (path to the main JavaScript file’s directory). This may be `''`
+   * (empty string) in some cases if the Emscripten Javascript code can't locate
+   * itself. Try logging it in your environment.
+   *
+   * It should return the actual URI or path to the requested file.
+   *
+   * This lets you host file packages on a different location than the directory
+   * of the JavaScript file (which is the default expectation), for example if
+   * you want to host them on a CDN.
+   */
   locateFile?: EmscriptenModuleLoaderOptions["locateFile"]
   /** The enumerable properties of this object will be passed verbatim, although they may be overwritten if you pass other options. */
   emscriptenModule?: EmscriptenModuleLoaderOptions

packages/quickjs-ffi-types/src/emscripten-types.ts

@@ -68,13 +68,35 @@ export interface QuickJSEmscriptenExtensions {
   receiveWasmOffsetConverter?(bytes: ArrayBuffer, mod: WebAssembly.Module): void
 }
 
-/** It's possible to provide these parameters to an emscripten module loader. */
+/**
+ * This structure is defined by Emscripten.
+ * It's possible to provide these parameters to an emscripten module loader.
+ * See [the Emscripten Module API reference](https://emscripten.org/docs/api_reference/module.html).
+ */
 export interface EmscriptenModuleLoaderOptions {
-  /** Give a path or URL where Emscripten should locate the given file */
+  /**
+   * If set, this method will be called when the runtime needs to load a file,
+   * such as a .wasm WebAssembly file, .mem memory init file, or a file
+   * generated by the file packager.
+   *
+   * The function receives two parameters:
+   *
+   * - `fileName`, the relative path to the file as configured in build
+   * process, eg `"emscripten-module.wasm"`.
+   * - `prefix` (path to the main JavaScript file’s directory). This may be `''`
+   * (empty string) in some cases if the Emscripten Javascript code can't locate
+   * itself. Try logging it in your environment.
+   *
+   * It should return the actual URI or path to the requested file.
+   *
+   * This lets you host file packages on a different location than the directory
+   * of the JavaScript file (which is the default expectation), for example if
+   * you want to host them on a CDN.
+   */
   locateFile?(
     fileName: "emscripten-module.wasm" | "emscripten-module.wasm.map" | string,
     /** Often `''` (empty string) */
-    relativeTo: string,
+    prefix: string,
   ): string
 
   /** Compile this to WebAssembly.Module */

scripts/helpers.ts

@@ -40,41 +40,34 @@ export function exec(command: string) {
 }
 
 class InstallFlow {
-  steps: Array<() => void> = []
+  removed = new Set<string>()
+  added = new Set<string>()
 
   constructor(
     public into: string,
     public commands: { install: string; remove: string; fromRegistry: boolean },
   ) {}
 
   add(packageName: string) {
-    const removed = new Set<string>()
-    const added = new Set<string>()
     this.visit(packageName, {
       after: (pkg) => {
-        removed.add(pkg.name)
+        this.removed.add(pkg.name)
       },
     })
-    this.steps.push(() =>
-      exec(`cd ${this.into} && ${this.commands.remove} ${[...removed].join(" ")}`),
-    )
 
     this.visit(packageName, {
       after: (pkg) => {
         if (this.commands.fromRegistry) {
           if (pkg.name === packageName) {
-            added.add(`${pkg.name}@${pkg.version}`)
+            this.added.add(`${pkg.name}@${pkg.version}`)
           }
           return
         }
 
         const tarFile = getTarFile(pkg.name)
-        added.add(`${pkg.name}@${tarFile}`)
+        this.added.add(`${pkg.name}@${tarFile}`)
       },
     })
-    this.steps.push(() =>
-      exec(`cd ${this.into} && ${this.commands.install} ${[...added].join(" ")}`),
-    )
 
     return this
   }
@@ -92,15 +85,19 @@ class InstallFlow {
   }
 
   run() {
-    for (const step of this.steps) {
-      step()
+    if (this.removed.size > 0) {
+      exec(`cd ${this.into} && ${this.commands.remove} ${[...this.removed].join(" ")}`)
+    }
+
+    if (this.added.size > 0) {
+      exec(`cd ${this.into} && ${this.commands.install} ${[...this.added].join(" ")}`)
     }
   }
 }
 
 export function installDependencyGraphFromTar(
   into: string,
-  packageName: string,
+  packageNameOrNames: string | string[],
   cmds: {
     install: string
     remove: string
@@ -111,7 +108,12 @@ export function installDependencyGraphFromTar(
     fromRegistry: INSTALL_FROM_REGISTRY,
   },
 ) {
-  new InstallFlow(into, cmds).add(packageName).run()
+  const packageNames = Array.isArray(packageNameOrNames) ? packageNameOrNames : [packageNameOrNames]
+  const flow = new InstallFlow(into, cmds)
+  for (const packageName of packageNames) {
+    flow.add(packageName)
+  }
+  flow.run()
 }
 
 export function resolve(...parts: string[]) {

scripts/smoketest-node-minimal.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env -S npx tsx
+import * as sh from "./helpers"
+
+const target = sh.resolve(__dirname, "../examples/node-minimal")
+sh.installDependencyGraphFromTar(target, [
+  "quickjs-emscripten-core",
+  "@jitl/quickjs-wasmfile-release-sync",
+])
+sh.exec(`cd ${target} && node main.mjs`)