Automatically migrate pg16 to pg17

package.json

@@ -101,6 +101,7 @@
   },
   "dependencies": {
     "@electric-sql/pglite": "^0.3.3",
+    "@electric-sql/pglite-tools": "^0.2.8",
     "@google-cloud/cloud-sql-connector": "^1.3.3",
     "@google-cloud/pubsub": "^4.5.0",
     "@inquirer/prompts": "^7.4.0",
@@ -150,6 +151,7 @@
     "p-limit": "^3.0.1",
     "pg": "^8.11.3",
     "pg-gateway": "^0.3.0-beta.4",
+    "pglite-2": "npm:@electric-sql/pglite@0.2.17",
     "portfinder": "^1.0.32",
     "progress": "^2.0.3",
     "proxy-agent": "^6.3.0",

src/emulator/dataconnect/pgliteServer.ts

@@ -5,10 +5,11 @@
 // They are only available as ESM, and if we import them normally,
 // our tsconfig will convert them to requires, which will cause errors
 // during module resolution.
 const { dynamicImport } = require(true && "../../dynamicImport");
 import * as net from "node:net";
 import { Readable, Writable } from "node:stream";
 import * as fs from "fs";
+import * as path from "node:path";
 
 import {
   getMessages,
@@ -40,12 +41,12 @@
   public db: PGlite | undefined = undefined;
   private server: net.Server | undefined = undefined;
 
   public async createPGServer(host: string = "127.0.0.1", port: number): Promise<net.Server> {
     const getDb = this.getDb.bind(this);
 
     const server = net.createServer(async (socket) => {
-      const connection: PostgresConnection = await fromNodeSocket(socket, {
-        serverVersion: "16.3 (PGlite 0.3.3)",
+      await fromNodeSocket(socket, {
+        serverVersion: "17.4 (PGlite 0.3.3)",
         auth: { method: "trust" },
 
         async *onMessage(data: Uint8Array, { isAuthenticated }: { isAuthenticated: boolean }) {
@@ -93,14 +94,9 @@
       }
       // Not all schemas will need vector installed, but we don't have an good way
       // to swap extensions after starting PGLite, so we always include it.
-      const vector = (await dynamicImport("@electric-sql/pglite/vector")).vector;
-      const uuidOssp = (await dynamicImport("@electric-sql/pglite/contrib/uuid_ossp")).uuid_ossp;
       const pgliteArgs: PGliteOptions = {
         debug: this.debug,
-        extensions: {
-          vector,
-          uuidOssp,
-        },
+        extensions: await this.getExtensions(),
         dataDir: this.dataDirectory,
       };
       if (this.importPath) {
@@ -114,6 +110,12 @@
     return this.db;
   }
 
+  private async getExtensions() {
+    const vector = (await dynamicImport("@electric-sql/pglite/vector")).vector;
+    const uuidOssp = (await dynamicImport("@electric-sql/pglite/contrib/uuid_ossp")).uuid_ossp;
+    return { vector, uuidOssp };
+  }
+
   public async clearDb(): Promise<void> {
     const db = await this.getDb();
     await db.query(TRUNCATE_TABLES_SQL);
@@ -126,7 +128,76 @@
     fs.writeFileSync(exportPath, new Uint8Array(arrayBuff));
   }
 
+  private async migrateDb(pgliteArgs: PGliteOptions): Promise<PGlite> {
+    if (!pgliteArgs.dataDir) {
+      throw new FirebaseError("Cannot migrate database without a data directory.");
+    }
+    const dataDir = pgliteArgs.dataDir;
+
+    // 1. Import old PGlite and pgDump
+    const { PGlite: PGlite02 } = await dynamicImport("pglite-2");
+    const pgDump = (await dynamicImport("@electric-sql/pglite-tools/pg_dump")).pgDump;
+
+    // 2. Open old DB with old PGlite
+
+    logger.info("Opening database with Postgres 16...");
+    const extensions = await this.getExtensions();
+    const oldDb = new PGlite02({ dataDir, extensions });
+    await oldDb.waitReady;
+
+    const oldVersion = await (oldDb as PGlite).query<{ version: string }>("SELECT version();");
+    logger.debug(`Old database version: ${oldVersion.rows[0].version}`);
+    if (!oldVersion.rows[0].version.includes("PostgreSQL 16")) {
+      await oldDb.close();
+      throw new FirebaseError("Migration started, but DB version is not PostgreSQL 16.");
+    }
+
+    // 3. Dump data
+    logger.info("Dumping data from old database...");
+    const dumpDir = await oldDb.dumpDataDir("none");
+    const tempOldDb = await PGlite02.create({
+      loadDataDir: dumpDir,
+      extensions,
+    });
+
+    const dumpResult = await pgDump({ pg: tempOldDb, args: ["--verbose", "--verbose"] });
+    await tempOldDb.close();
+    await oldDb.close();
+
+    // 4. Nuke old data directory
+    logger.info("Removing old database directory...");
+    fs.rmSync(dataDir, { force: true, recursive: true });
+
+    // 5. Create new DB with new PGlite
+    logger.info("Creating new database with Postgres 17...");
+    const newDb = new PGlite(pgliteArgs);
+    await newDb.waitReady;
+
+    // 6. Import data
+    logger.info("Importing data into new database...");
+    const dumpText = await dumpResult.text();
+    await newDb.exec(dumpText);
+    await newDb.exec("SET SEARCH_PATH = public;");
+
+    logger.info("Postgres database migration successful.");
+    return newDb;
+  }
+
   async forceCreateDB(pgliteArgs: PGliteOptions): Promise<PGlite> {
+    if (pgliteArgs.dataDir && fs.existsSync(pgliteArgs.dataDir)) {
+      const versionFilePath = path.join(pgliteArgs.dataDir, "PG_VERSION");
+      if (fs.existsSync(versionFilePath)) {
+        const version = fs.readFileSync(versionFilePath, "utf-8").trim();
+        logger.debug(`Found Postgres version file with version: ${version}`);
+        if (version === "16") {
+          logger.info(
+            "Detected a Postgres 16 data directory from an older version of firebase-tools. Migrating to Postgres 17...",
+          );
+          return this.migrateDb(pgliteArgs);
+        }
+      }
+    }
+
     try {
       const db = new PGlite(pgliteArgs);
       await db.waitReady;
@@ -161,7 +232,6 @@
   }
 }
 
-
 /**
  * Creates a `PostgresConnection` from a Node.js TCP/Unix `Socket`.
  *
@@ -181,4 +251,4 @@
     : undefined;
 
   return new PostgresConnection({ readable: rs, writable: ws }, opts);
-}
+}