pgdump with emscripten

.changeset/old-tables-yawn.md

@@ -0,0 +1,6 @@
+---
+'@electric-sql/pglite-tools': patch
+'@electric-sql/pglite': patch
+---
+
+Updated pg_dump to use callback data exchange; built pg_dump with emscripten

.github/workflows/build_and_test.yml

@@ -75,12 +75,13 @@ jobs:
           name: pglite-interim-build-files-node-v20.x
           path: ./packages/pglite/release/**
           retention-days: 60
-      - name: Upload PGlite tools to Gihub artifacts
-        id: upload-pglite-tools-dist-files
+
+      - name: Upload pglite-tools build artifacts to Github artifacts
+        id: upload-pglite-tools-release-files
         uses: actions/upload-artifact@v4
         with:
-          name: pglite-tools-dist-files-node-v20.x
-          path: ./packages/pglite-tools/dist/**
+          name: pglite-tools-release-files-node-v20.x
+          path: ./packages/pglite-tools/release/**
           retention-days: 60
 
   build-and-test-pglite:
@@ -111,6 +112,12 @@ jobs:
           name: pglite-interim-build-files-node-v20.x
           path: ./packages/pglite/release
 
+      - name: Download pglite-tools WASM build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pglite-tools-release-files-node-v20.x
+          path: ./packages/pglite-tools/release
+
       - name: Install dependencies
         run: |
           pnpm install --frozen-lockfile
@@ -157,7 +164,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: Built bundles
+          body-includes: '- PGlite:'
 
       - name: Create or update build outputs comment
         uses: peter-evans/create-or-update-comment@v4
@@ -191,6 +198,12 @@ jobs:
           name: pglite-dist-node-v${{ matrix.node }}
           path: ./packages/pglite/dist/
 
+      - name: Download pglite-tools WASM build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pglite-tools-release-files-node-v20.x
+          path: ./packages/pglite-tools/release/
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
@@ -206,6 +219,13 @@ jobs:
       - name: Test packages
         run: pnpm --filter="...^pglite" test
 
+      - name: Upload pglite-tools distribution artifact
+        id: upload-pglite-tools-dist
+        uses: actions/upload-artifact@v4
+        with:
+          name: pglite-tools-dist-node-v${{ matrix.node }}
+          path: ./packages/pglite-tools/dist/*        
+
   publish-website-with-demos:
     name: Publish website with demos
     runs-on: ubuntu-22.04
@@ -227,6 +247,12 @@ jobs:
           name: pglite-interim-build-files-node-v20.x
           path: ./packages/pglite/release
 
+      - name: Download pglite-tools WASM build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pglite-tools-release-files-node-v20.x
+          path: ./packages/pglite-tools/release          
+
       - name: Download PGlite build artifacts
         uses: actions/download-artifact@v4
         with:
@@ -245,10 +271,10 @@ jobs:
           name: pglite-interim-build-files-node-v20.x
           path: /tmp/web
 
-      - name: Download PGlite tools build artifacts
+      - name: Download pglite-tools dist artifacts to tmp
         uses: actions/download-artifact@v4
         with:
-          name: pglite-tools-dist-files-node-v20.x
+          name: pglite-tools-dist-node-v20.x
           path: /tmp/pglite-tools
 
       - name: Build demo site
@@ -333,6 +359,13 @@ jobs:
         with:
           name: pglite-interim-build-files-node-v20.x
           path: ./packages/pglite/release
+
+      - name: Download pglite-tools build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pglite-tools-release-files-node-v20.x
+          path: ./packages/pglite-tools/release/
+
       - run: pnpm install --frozen-lockfile
       - run: pnpm --filter "./packages/**" build
       - name: Create Release Pull Request or Publish

package.json

@@ -12,7 +12,9 @@
     "ci:publish": "pnpm changeset publish",
     "ts:build": "pnpm -r --filter \"./packages/**\" build",
     "ts:build:debug": "DEBUG=true pnpm ts:build",
-    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/",
+    "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release",
+    "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/",
+    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump",
     "wasm:build:debug": "DEBUG=true pnpm wasm:build",
     "build:all": "pnpm wasm:build && pnpm ts:build",
     "build:all:debug": "DEBUG=true pnpm build:all"

packages/pglite-tools/package.json

@@ -45,7 +45,7 @@
     }
   },
   "scripts": {
-    "build": "tsup && tsx ./scripts/bundle-wasm.ts",
+    "build": "tsup",
     "check:exports": "attw . --pack --profile node16",
     "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0",
     "format": "prettier --write ./src ./tests",

packages/pglite-tools/src/pgDumpModFactory.ts

@@ -0,0 +1,41 @@
+import PgDumpModFactory from '../release/pg_dump'
+
+type IDBFS = Emscripten.FileSystemType & {
+  quit: () => void
+  dbs: Record<string, IDBDatabase>
+}
+
+export type FS = typeof FS & {
+  filesystems: {
+    MEMFS: Emscripten.FileSystemType
+    NODEFS: Emscripten.FileSystemType
+    IDBFS: IDBFS
+  }
+  quit: () => void
+}
+
+export interface PgDumpMod
+  extends Omit<EmscriptenModule, 'preInit' | 'preRun' | 'postRun'> {
+  preInit: Array<{ (mod: PgDumpMod): void }>
+  preRun: Array<{ (mod: PgDumpMod): void }>
+  postRun: Array<{ (mod: PgDumpMod): void }>
+  FS: FS
+  WASM_PREFIX: string
+  INITIAL_MEMORY: number
+  _set_read_write_cbs: (read_cb: number, write_cb: number) => void
+  addFunction: (
+    cb: (ptr: any, length: number) => void,
+    signature: string,
+  ) => number
+  removeFunction: (f: number) => void
+  _main: (args: string[]) => number
+  onExit: (status: number) => void
+  print: (test: string) => void
+  printErr: (text: string) => void
+}
+
+type PgDumpFactory<T extends PgDumpMod = PgDumpMod> = (
+  moduleOverrides?: Partial<T>,
+) => Promise<T>
+
+export default PgDumpModFactory as PgDumpFactory<PgDumpMod>