Import/export support for emulators (#1968)

Author: samtstern

CHANGELOG.md

@@ -0,0 +1 @@
+* Adds import/export support to `emulators` commands for Firestore (#1167).

package-lock.json

@@ -97,6 +97,7 @@
     "plist": "^3.0.1",
     "portfinder": "^1.0.23",
     "progress": "^2.0.3",
+    "rimraf": "^3.0.0",
     "request": "^2.87.0",
     "semver": "^5.7.1",
     "superstatic": "^6.0.1",
@@ -129,6 +130,7 @@
     "@types/plist": "^3.0.1",
     "@types/progress": "^2.0.3",
     "@types/request": "^2.48.1",
+    "@types/rimraf": "^2.0.3",
     "@types/semver": "^6.0.0",
     "@types/sinon": "^5.0.5",
     "@types/sinon-chai": "^3.2.2",
@@ -156,7 +158,6 @@
     "nock": "^9.3.3",
     "nyc": "^14.0.0",
     "prettier": "1.14.3",
-    "rimraf": "^3.0.0",
     "sinon": "^6.3.4",
     "sinon-chai": "^3.2.0",
     "source-map-support": "^0.5.9",

package.json

@@ -9,6 +9,9 @@ database-debug.log*
 firestore-debug.log*
 pubsub-debug.log*
 
+# NPM
+package-lock.json
+
 # Firebase cache
 .firebase/
 
@@ -65,4 +68,4 @@ node_modules/
 .yarn-integrity
 
 # dotenv environment variables file
-.env
+.env

scripts/triggers-end-to-end-tests/.gitignore

@@ -5,6 +5,9 @@
   },
   "functions": {},
   "emulators": {
+    "hub": {
+      "port": 4000
+    },
     "database": {
       "port": 9000
     },

scripts/triggers-end-to-end-tests/firebase.json

@@ -6,6 +6,7 @@ const chai = require("chai");
 const expect = chai.expect;
 const assert = chai.assert;
 const fs = require("fs");
+const os = require("os");
 
 const Firestore = require("@google-cloud/firestore");
 
@@ -32,7 +33,6 @@ const ALL_EMULATORS_STARTED_LOG = "All emulators started, it is now safe to conn
  * parallel emulator subprocesses.
  */
 const TEST_SETUP_TIMEOUT = 60000;
-const EMULATORS_STARTUP_DELAY_TIMEOUT = 60000;
 const EMULATORS_WRITE_DELAY_MS = 5000;
 const EMULATORS_SHUTDOWN_DELAY_MS = 5000;
 const EMULATOR_TEST_TIMEOUT = EMULATORS_WRITE_DELAY_MS * 2;
@@ -44,6 +44,66 @@ const EMULATOR_TEST_TIMEOUT = EMULATORS_WRITE_DELAY_MS * 2;
 const FIRESTORE_COMPLETION_MARKER = "test/done_from_firestore";
 const DATABASE_COMPLETION_MARKER = "test/done_from_database";
 
+function CLIProcess(name) {
+  this.name = name;
+  this.process = undefined;
+}
+CLIProcess.prototype.constructor = CLIProcess;
+
+CLIProcess.prototype.start = function(cmd, additionalArgs, logDoneFn) {
+  const args = [PROJECT_ROOT + "/lib/bin/firebase.js", cmd, "--project", FIREBASE_PROJECT];
+
+  if (additionalArgs) {
+    args.push(...additionalArgs);
+  }
+
+  this.process = subprocess.spawn("node", args);
+
+  this.process.stdout.on("data", (data) => {
+    process.stdout.write(`[${this.name} stdout] ` + data);
+  });
+
+  this.process.stderr.on("data", (data) => {
+    console.log(`[${this.name} stderr] ` + data);
+  });
+
+  let started;
+  if (logDoneFn) {
+    started = new Promise((resolve) => {
+      this.process.stdout.on("data", (data) => {
+        if (logDoneFn(data)) {
+          resolve();
+        }
+      });
+    });
+  } else {
+    started = new Promise((resolve) => {
+      this.process.once("close", () => {
+        this.process = undefined;
+        resolve();
+      });
+    });
+  }
+
+  return started;
+};
+
+CLIProcess.prototype.stop = function() {
+  if (!this.process) {
+    return Promise.resolve();
+  }
+
+  const stopped = new Promise((resolve) => {
+    this.process.once("close", (/* exitCode, signal */) => {
+      this.process = undefined;
+      resolve();
+    });
+  });
+
+  this.process.kill("SIGINT");
+  return stopped;
+};
+
 function TriggerEndToEndTest(config) {
   this.rtdb_emulator_host = "localhost";
   this.rtdb_emulator_port = config.emulators.database.port;
@@ -69,7 +129,7 @@ function TriggerEndToEndTest(config) {
   this.rtdb_from_rtdb = false;
   this.firestore_from_firestore = false;
 
-  this.emulators_process = null;
+  this.cli_process = null;
 }
 
 /*
@@ -86,50 +146,36 @@ TriggerEndToEndTest.prototype.success = function success() {
 };
 
 TriggerEndToEndTest.prototype.startEmulators = function startEmulators(additionalArgs) {
-  var self = this;
-  const args = [
-    PROJECT_ROOT + "/lib/bin/firebase.js",
-    "emulators:start",
-    "--project",
-    FIREBASE_PROJECT,
-  ];
-
-  if (additionalArgs) {
-    args.push(...additionalArgs);
-  }
-
-  self.emulators_process = subprocess.spawn("node", args);
+  const cli = new CLIProcess("default");
+  const started = cli.start("emulators:start", additionalArgs, (data) => {
+    return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+  });
 
-  self.emulators_process.stdout.on("data", function(data) {
-    process.stdout.write("[emulators stdout] " + data);
+  cli.process.stdout.on("data", (data) => {
     if (data.indexOf(RTDB_FUNCTION_LOG) > -1) {
-      self.rtdb_trigger_count++;
+      this.rtdb_trigger_count++;
     }
     if (data.indexOf(FIRESTORE_FUNCTION_LOG) > -1) {
-      self.firestore_trigger_count++;
+      this.firestore_trigger_count++;
     }
     if (data.indexOf(PUBSUB_FUNCTION_LOG) > -1) {
-      self.pubsub_trigger_count++;
-    }
-    if (data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1) {
-      self.all_emulators_started = true;
+      this.pubsub_trigger_count++;
     }
   });
 
-  self.emulators_process.stderr.on("data", function(data) {
-    console.log("[emulators stderr] " + data);
-  });
+  this.cli_process = cli;
+  return started;
 };
 
-TriggerEndToEndTest.prototype.stopEmulators = function stopEmulators(done) {
-  this.emulators_process.once("close", function(/* exitCode, signal */) {
-    done();
-  });
+TriggerEndToEndTest.prototype.startEmulatorsAndWait = function startEmulatorsAndWait(
+  additionalArgs,
+  done
+) {
+  this.startEmulators(additionalArgs).then(done);
+};
 
-  /*
-   * CLI process only shuts down emulators cleanly on SIGINT.
-   */
-  this.emulators_process.kill("SIGINT");
+TriggerEndToEndTest.prototype.stopEmulators = function stopEmulators(done) {
+  this.cli_process.stop().then(done);
 };
 
 TriggerEndToEndTest.prototype.invokeHttpFunction = function invokeHttpFunction(name, done) {
@@ -217,15 +263,7 @@ describe("database and firestore emulator function triggers", function() {
           });
         },
         function(done) {
-          test.startEmulators(["--only", "functions,database,firestore"]);
-          test.waitForCondition(
-            () => test.all_emulators_started,
-            EMULATORS_STARTUP_DELAY_TIMEOUT,
-            (err) => {
-              expect(err).to.be.undefined;
-              done();
-            }
-          );
+          test.startEmulatorsAndWait(["--only", "functions,database,firestore"], done);
         },
         function(done) {
           test.firestore_client = new Firestore({
@@ -390,15 +428,7 @@ describe("pubsub emulator function triggers", function() {
           });
         },
         function(done) {
-          test.startEmulators(["--only", "functions,pubsub"]);
-          test.waitForCondition(
-            () => test.all_emulators_started,
-            EMULATORS_STARTUP_DELAY_TIMEOUT,
-            (err) => {
-              expect(err).to.be.undefined;
-              done();
-            }
-          );
+          test.startEmulatorsAndWait(["--only", "functions,pubsub"], done);
         },
       ],
       done
@@ -429,3 +459,33 @@ describe("pubsub emulator function triggers", function() {
     done();
   });
 });
+
+describe("import/export end to end", () => {
+  it("should be able to import/export firestore data", async () => {
+    // Start up emulator suite
+    const emulatorsCLI = new CLIProcess("1");
+    await emulatorsCLI.start("emulators:start", ["--only", "firestore"], (data) => {
+      return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+    });
+
+    // Ask for export
+    const exportCLI = new CLIProcess("2");
+    const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data"));
+    await exportCLI.start("emulators:export", [exportPath]);
+
+    // Stop the suite
+    await emulatorsCLI.stop();
+
+    // Attempt to import
+    const importCLI = new CLIProcess("3");
+    await importCLI.start(
+      "emulators:start",
+      ["--only", "firestore", "--import", exportPath],
+      (data) => {
+        return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+      }
+    );
+
+    await importCLI.stop();
+  }).timeout(2 * TEST_SETUP_TIMEOUT);
+});

scripts/triggers-end-to-end-tests/run.spec.js

@@ -84,6 +84,7 @@ module.exports = new Command("emulators:exec <script>")
   )
   .option(commandUtils.FLAG_ONLY, commandUtils.DESC_ONLY)
   .option(commandUtils.FLAG_INSPECT_FUNCTIONS, commandUtils.DESC_INSPECT_FUNCTIONS)
+  .option(commandUtils.FLAG_IMPORT, commandUtils.DESC_IMPORT)
   .action(async (script: string, options: any) => {
     const projectId = getProjectId(options, true);
     const extraEnv: Record<string, string> = {};