[POC] Dynamic Extension Loading

.changeset/funny-ghosts-attack.md

@@ -0,0 +1,5 @@
+---
+'@journeyapps/wa-sqlite': minor
+---
+
+Load PowerSync SQLite core as a dynamic sidemodule

.changeset/perfect-radios-hunt.md

@@ -1,5 +1,5 @@
 ---
-'@journeyapps/wa-sqlite': minor
+'@journeyapps/wa-sqlite': major
 ---
 
-Updated from upstream
+Updated from upstream to v1.0.3

.changeset/stale-cups-grow.md

@@ -0,0 +1,5 @@
+---
+'@journeyapps/wa-sqlite': patch
+---
+
+Fix bug where table change update notifications would access invalid memory locations under certain conditions.

.github/workflows/ci.yml

@@ -38,11 +38,6 @@ jobs:
       - run: |
             ${{ steps.setup-chrome.outputs.chrome-path }} --version
 
-      - name: Rust Setup
-        run: |
-          rustup toolchain install nightly-2023-08-28-x86_64-unknown-linux-gnu
-          rustup component add rust-src --toolchain nightly-2023-08-28-x86_64-unknown-linux-gnu
-
       # Install yarn dependencies.
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
@@ -55,10 +50,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
       - run: yarn install
-      - name: Rust Setup
-        run: |
-          rustup toolchain install nightly-2024-05-18-x86_64-unknown-linux-gnu
-          rustup component add rust-src --toolchain nightly-2024-05-18-x86_64-unknown-linux-gnu
 
       - name: Test with checked-in WASM files
         run: yarn test

.gitignore

@@ -9,6 +9,9 @@
 !.yarn/versions
 .pnp.*
 
+# PowerSync Rust Core
+libpowersync*.wasm
+
 /cache
 /debug
 /deps

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "powersync-sqlite-core"]
-	path = powersync-sqlite-core
-	url = git@github.com:powersync-ja/powersync-sqlite-core.git

Makefile

@@ -1,12 +1,13 @@
 # dependencies
-# TODO this should be 3.46.0, but there are build errors
-SQLITE_VERSION = version-3.44.0
+SQLITE_VERSION = version-3.46.0
 SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE_VERSION}
 
 EXTENSION_FUNCTIONS = extension-functions.c
 EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
 EXTENSION_FUNCTIONS_SHA3 = ee39ddf5eaa21e1d0ebcbceeab42822dd0c4f82d8039ce173fd4814807faabfa
 
+
+
 # source files
 CFILES = \
 	sqlite3.c \
@@ -37,21 +38,14 @@ ASYNCIFY_EXPORTS = src/asyncify_exports.json
 OBJ_FILES_DEBUG = $(patsubst %.c,tmp/obj/debug/%.o,$(CFILES))
 OBJ_FILES_DIST = $(patsubst %.c,tmp/obj/dist/%.o,$(CFILES))
 
-RS_LIB = powersync
-RS_LIB_DIR = ./powersync-sqlite-core
-RS_WASM_TGT = wasm32-unknown-emscripten
-RS_WASM_TGT_DIR = ${RS_LIB_DIR}/target/$(RS_WASM_TGT)
-RS_RELEASE_BC = $(RS_WASM_TGT_DIR)/wasm/deps/$(RS_LIB).bc
-RS_DEBUG_BC = $(RS_WASM_TGT_DIR)/debug/deps/$(RS_LIB).bc
-
 # build options
 EMCC ?= emcc
 
 CFLAGS_COMMON = \
 	-I'deps/$(SQLITE_VERSION)' \
 	-Wno-non-literal-null-conversion \
 	$(CFLAGS_EXTRA)
-CFLAGS_DEBUG = -g $(CFLAGS_COMMON)
+CFLAGS_DEBUG = -g -fPIC $(CFLAGS_COMMON)
 CFLAGS_DIST =  -Oz -flto $(CFLAGS_COMMON)
 
 EMFLAGS_COMMON = \
@@ -72,6 +66,18 @@ EMFLAGS_DIST = \
 	-flto \
 	$(EMFLAGS_COMMON)
 
+# Need to export all the Main module's symbols
+# Setting MAIN_MODULE=2 can cause a runtime error if the 
+# main module does not export a particular function.
+# With this being a runtime error, it's tricky to determine all
+# the methods that a submodule might require - also for this to have value
+# other submodules would need to be compatible, but we cannot gaurentee their
+# required exports are present. 
+# This setting increases the main WASM side considerably.
+# Rougly increases from 1.5mb to 3.7mb.
+EMFLAGS_DYNAMIC = \
+	-s MAIN_MODULE=1 
+
 EMFLAGS_INTERFACES = \
 	-s EXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS) \
 	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
@@ -111,7 +117,6 @@ WASQLITE_DEFINES = \
 	-DSQLITE_OMIT_AUTOINIT \
 	-DSQLITE_OMIT_DECLTYPE \
 	-DSQLITE_OMIT_DEPRECATED \
-	-DSQLITE_OMIT_LOAD_EXTENSION \
 	-DSQLITE_OMIT_SHARED_CACHE \
 	-DSQLITE_THREADSAFE=0 \
 	-DSQLITE_USE_ALLOCA \
@@ -165,13 +170,23 @@ else
 endif
 
 
+# This hacks away a:
+# ```C
+#  #define COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE=1
+# ```
+# statement in `extension-functions.c`. 
+# The default setting causes ``extension-functions.c` to import SQLite3 with
+# ```C
+#  #include "sqlite3ext.h"
+# ```
+# This results in undefined symbols during runtime when compiling the WASM with MAIN_MODULE=2.
 deps/$(EXTENSION_FUNCTIONS): cache/$(EXTENSION_FUNCTIONS)
 	mkdir -p deps
 	bash -c "$(OPENSSL_CHECK_CMD)"
 	bash -c "$(HASH_CHECK_CMD)"
 	rm -rf deps/sha3 $@
-	cp 'cache/$(EXTENSION_FUNCTIONS)' $@
-
+	sed -i '' "/#define COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE/d" cache/$(EXTENSION_FUNCTIONS)
+	cp cache/$(EXTENSION_FUNCTIONS) $@
 ## tmp
 .PHONY: clean-tmp
 clean-tmp:
@@ -185,48 +200,48 @@ tmp/obj/dist/%.o: %.c
 	mkdir -p tmp/obj/dist
 	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
 
-# Use Linker true command switch which differs per OS
-ifeq ($(shell uname), Darwin)
-    TRUE_CMD := /usr/bin/true
-else
-    TRUE_CMD := /bin/true
-endif
-
-$(RS_DEBUG_BC): FORCE
-	mkdir -p tmp/bc/dist
-	cd $(RS_LIB_DIR); \
-	RUSTFLAGS="--emit=llvm-bc -C linker=${TRUE_CMD}" cargo build -p powersync_loadable --profile wasm --no-default-features --features "powersync_core/static powersync_core/omit_load_extension sqlite_nostd/static sqlite_nostd/omit_load_extension" -Z build-std=panic_abort,core,alloc --target $(RS_WASM_TGT)
-
-$(RS_RELEASE_BC): FORCE
-	mkdir -p tmp/bc/dist
-	cd $(RS_LIB_DIR); \
-	RUSTFLAGS="--emit=llvm-bc -C linker=${TRUE_CMD}" cargo build -p powersync_loadable --profile wasm --no-default-features --features "powersync_core/static powersync_core/omit_load_extension sqlite_nostd/static sqlite_nostd/omit_load_extension" -Z build-std=panic_abort,core,alloc --target $(RS_WASM_TGT)
-
 
 ## debug
 .PHONY: clean-debug
 clean-debug:
 	rm -rf debug
 
 .PHONY: debug
-debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs
+debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-main.mjs debug/wa-sqlite-main-async.mjs
 
-debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# TODO link static
+debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-	  $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
-	  $(OBJ_FILES_DEBUG) *.o -o $@
+	  $(OBJ_FILES_DEBUG) -o $@
 
-debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# TODO link static
+debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DEBUG) \
-	  $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
-	  $(OBJ_FILES_DEBUG) *.o -o $@
+	  $(OBJ_FILES_DEBUG) -o $@
+
+# Main module distributable
+debug/wa-sqlite-main.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(OBJ_FILES_DEBUG) -o $@
+
+# Main module distributable
+debug/wa-sqlite-main-async.mjs: $(OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_ASYNCIFY_DEBUG) \
+	  $(OBJ_FILES_DEBUG) -o $@
 
 ## Debug FTS builds
 # .PHONY: debug
@@ -249,13 +264,13 @@ debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTION
 # 		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
 # 	  $(OBJ_FILES_DEBUG_FTS) *.o -o $@
 
+# TODO link static
 debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p debug
-	$(EMCC) $(EMFLAGS_DEBUG) \
+	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC)  \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_JSPI) \
-	  $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
 	  $(OBJ_FILES_DEBUG) -o $@
 
 ## dist
@@ -264,32 +279,47 @@ clean-dist:
 	rm -rf dist
 
 .PHONY: dist
-dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs
+dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-main.mjs dist/wa-sqlite-async-main.mjs
 
-dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# TODO link static
+dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-	  $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
 	  $(OBJ_FILES_DIST)  -o $@
 
-dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# TODO link static
+dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DIST) \
-	  $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
 	  $(OBJ_FILES_DIST) -o $@
 
+dist/wa-sqlite-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) $(EMFLAGS_DYNAMIC) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(OBJ_FILES_DIST)  -o $@
+
+dist/wa-sqlite-async-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) $(EMFLAGS_DYNAMIC) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_ASYNCIFY_DIST) \
+	  $(OBJ_FILES_DIST) -o $@
+
+# TODO link static
 dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_JSPI) \
-	  $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
 	  $(OBJ_FILES_DIST) -o $@
 
 FORCE:

README.md

@@ -46,19 +46,15 @@ git clone [this repo]
 ```
 
 ```bash
-git submodule init
+make -B
 ```
 
-```bash
-git submodule update --init --recursive
-```
+### PowerSync Core
 
-```bash
-yarn install
-```
+To update the PowerSync SQLite core see the [script](./scripts/powersync-core.js). Edit the `CORE_VERSION` and run the script with
 
 ```bash
-make -B
+node scripts/powersync-core.js
 ```
 
 ## API

demo/demo-worker.js

@@ -11,6 +11,12 @@ const BUILDS = new Map([
   // ['jspi', '../debug/wa-sqlite-jspi.mjs'],
 ]);
 
+const EXT_WASM = new Map([
+  ['default', 'libpowersync.wasm'],
+  ['asyncify', 'libpowersync-async.wasm'],
+  ['jspi', 'libpowersync-jspi.wasm'],
+]);
+
 /**
  * @typedef Config
  * @property {string} name
@@ -77,15 +83,37 @@ maybeReset().then(async () => {
   const buildName = searchParams.get('build') || BUILDS.keys().next().value;
   const configName = searchParams.get('config') || VFS_CONFIGS.keys().next().value;
   const config = VFS_CONFIGS.get(configName);
+  const extWasm = EXT_WASM.get(buildName);
 
   const dbName = searchParams.get('dbName') ?? 'hello';
   const vfsName = searchParams.get('vfsName') ?? config.vfsName ?? 'demo';
 
   // Instantiate SQLite.
   const { default: moduleFactory } = await import(BUILDS.get(buildName));
-  const module = await moduleFactory();
+  const module = await moduleFactory({
+    // locateFile(path) {
+    //   // (OPTIONAL) locateFile can be specified to change WASM module resolution.
+    //   // The returned url is passed to fetch, and can be a relative path
+    //   // or absolute URL.
+    //   // This applies to both the main wasm file, and the extension.
+    //   return `../dist/${path}`;
+    // },
+  });
+
   const sqlite3 = SQLite.Factory(module);
 
+  // Load the extension library into this scope
+  let extScope = {};
+  await module.loadDynamicLibrary(
+    extWasm,
+    { loadAsync: true },
+    extScope
+  );
+
+  // This calls sqlite3_auto_extension(sqlite3_powersync_init).
+  // For generic extensions, we'd need to call sqlite3_auto_extension directly.
+  extScope.powersync_init_static();
+
   if (config.vfsModule) {
     // Create the VFS and register it as the default file system.
     const namespace = await import(config.vfsModule);

package.json

@@ -20,10 +20,13 @@
     "src/WebLocksMixin.js",
     "src/examples/*",
     "dist/*",
-    "test/*"
+    "test/*",
+    "scripts/download-powersync-core.js"
   ],
   "scripts": {
     "build-docs": "typedoc",
+    "postinstall": "npm run powersync-core:download",
+    "powersync-core:download": "node scripts/download-dynamic-core.js",
     "release": "yarn changeset publish",
     "start": "web-dev-server --node-resolve",
     "test": "web-test-runner",