Apphosting emulator - local secrets support via apphosting:config:export command

src/apphosting/config.spec.ts

@@ -2,7 +2,6 @@ import { expect } from "chai";
 import * as sinon from "sinon";
 import * as yaml from "yaml";
 import * as path from "path";
-
 import * as fsImport from "../fsutils";
 import * as promptImport from "../prompt";
 import * as dialogs from "./secrets/dialogs";
@@ -23,15 +22,19 @@ describe("config", () => {
 
     it("finds apphosting.yaml at cwd", () => {
       fs.fileExistsSync.withArgs("/cwd/apphosting.yaml").returns(true);
-      expect(config.yamlPath("/cwd")).equals("/cwd/apphosting.yaml");
+      expect(config.yamlPath("/cwd", config.APPHOSTING_BASE_YAML_FILE)).equals(
+        "/cwd/apphosting.yaml",
+      );
     });
 
     it("finds apphosting.yaml in a parent directory", () => {
       fs.fileExistsSync.withArgs("/parent/cwd/apphosting.yaml").returns(false);
       fs.fileExistsSync.withArgs("/parent/cwd/firebase.json").returns(false);
       fs.fileExistsSync.withArgs("/parent/apphosting.yaml").returns(true);
 
-      expect(config.yamlPath("/parent/cwd")).equals("/parent/apphosting.yaml");
+      expect(config.yamlPath("/parent/cwd", config.APPHOSTING_BASE_YAML_FILE)).equals(
+        "/parent/apphosting.yaml",
+      );
     });
 
     it("returns null if it finds firebase.json without finding apphosting.yaml", () => {
@@ -40,7 +43,7 @@ describe("config", () => {
       fs.fileExistsSync.withArgs("/parent/apphosting.yaml").returns(false);
       fs.fileExistsSync.withArgs("/parent/firebase.json").returns(true);
 
-      expect(config.yamlPath("/parent/cwd")).equals(null);
+      expect(config.yamlPath("/parent/cwd", config.APPHOSTING_BASE_YAML_FILE)).equals(null);
     });
 
     it("returns if it reaches the fs root", () => {
@@ -51,7 +54,7 @@ describe("config", () => {
       fs.fileExistsSync.withArgs("/apphosting.yaml").returns(false);
       fs.fileExistsSync.withArgs("/firebase.json").returns(false);
 
-      expect(config.yamlPath("/parent/cwd")).equals(null);
+      expect(config.yamlPath("/parent/cwd", config.APPHOSTING_BASE_YAML_FILE)).equals(null);
     });
   });
 
@@ -210,4 +213,37 @@ env:
       expect(store).to.have.been.calledWithMatch(path.join("CWD", "apphosting.yaml"), doc);
     });
   });
+
+  describe("discoverConfigsInProject", () => {
+    let fs: sinon.SinonStubbedInstance<typeof fsImport>;
+
+    beforeEach(() => {
+      fs = sinon.stub(fsImport);
+    });
+
+    afterEach(() => {
+      sinon.verifyAndRestore();
+    });
+
+    it("moves up the tree finding apphosting.*.yaml files till firebase.json file found", () => {
+      fs.fileExistsSync.withArgs("/parent-parent/firebase.json").returns(true);
+      fs.fileExistsSync.withArgs("/parent-parent/parent/cwd/firebase.json").returns(false);
+      fs.fileExistsSync.withArgs("/parent-parent/parent/firebase.json").returns(false);
+
+      fs.listFiles
+        .withArgs("/parent-parent/parent/cwd")
+        .returns(["apphosting.staging.yaml", "blah.js", "apphosting.foo.bar.yaml"]);
+
+      fs.listFiles
+        .withArgs("/parent-parent/parent")
+        .returns(["apphosting.local.yaml", "bloh.txt", "apphosting.yaml"]);
+
+      const apphostingYamls = config.discoverConfigsInProject("/parent-parent/parent/cwd");
+      expect(apphostingYamls).to.deep.equal([
+        "/parent-parent/parent/cwd/apphosting.staging.yaml",
+        "/parent-parent/parent/apphosting.local.yaml",
+        "/parent-parent/parent/apphosting.yaml",
+      ]);
+    });
+  });
 });

src/apphosting/config.ts

@@ -7,6 +7,10 @@
 import * as prompt from "../prompt";
 import * as dialogs from "./secrets/dialogs";
 
+export const APPHOSTING_BASE_YAML_FILE = "apphosting.yaml";
+export const APPHOSTING_LOCAL_YAML_FILE = "apphosting.local.yaml";
+export const APPHOSTING_YAML_FILE_REGEX = /^apphosting(\.[a-zA-Z0-9]+)?\.yaml$/;
+
 export interface RunConfig {
   concurrency?: number;
   cpu?: number;
@@ -33,15 +37,15 @@
 }
 
 /**
- * Finds the path of apphosting.yaml.
- * Starts with cwd and walks up the tree until apphosting.yaml is found or
+ * Finds the path of a yaml file.
+ * Starts with cwd and walks up the tree until yamlFileName is found or
  * we find the project root (where firebase.json is) or the filesystem root;
  * in these cases, returns null.
  */
-export function yamlPath(cwd: string): string | null {
+export function yamlPath(cwd: string, yamlFileName: string): string | null {
   let dir = cwd;
 
-  while (!fs.fileExistsSync(resolve(dir, "apphosting.yaml"))) {
+  while (!fs.fileExistsSync(resolve(dir, yamlFileName))) {
     // We've hit project root
     if (fs.fileExistsSync(resolve(dir, "firebase.json"))) {
       return null;
@@ -54,7 +58,42 @@
     }
     dir = parent;
   }
-  return resolve(dir, "apphosting.yaml");
+  return resolve(dir, yamlFileName);
+}
+
+/**
+ * Finds all paths to `apphosting.*.yaml` configs within a project.
+ *
+ * This function starts at the provided directory (`cwd`) and traverses
+ * upwards through the file system until it finds a `firebase.json` file
+ * (indicating the project root) or reaches the root of the filesystem.
+ * Along the way, it collects the paths of all encountered `apphosting.*.yaml` files.
+ *
+ * @param cwd The directory to start the search from.
+ * @returns An array of strings representing the paths to all found `apphosting.*.yaml` files,
+ *          or `null` if no such files are found.
+ */
+export function discoverConfigsInProject(cwd: string): string[] | null {
+  let dir = cwd;
+  const files: string[] = [];
+
+  do {
+    const apphostingYamlFiles = fs
+      .listFiles(dir)
+      .filter((file) => APPHOSTING_YAML_FILE_REGEX.test(file));
+    const apphostingYamlFilePaths = apphostingYamlFiles.map((file) => join(dir, file));
+    files.push(...apphostingYamlFilePaths);
+
+    const parent = dirname(dir);
+    // We've hit the filesystem root
+    if (parent === dir) {
+      break;
+    }
+
+    dir = parent;
+  } while (!fs.fileExistsSync(resolve(dir, "firebase.json"))); // We've hit project root
+
+  return files.length > 0 ? files : null;
 }
 
 /** Load apphosting.yaml */
@@ -120,7 +159,7 @@
     store: typeof store;
   };
   // Note: The API proposal suggested that we would check if the env exists. This is stupidly hard because the YAML may not exist yet.
-  let path = dynamicDispatch.yamlPath(process.cwd());
+  let path = dynamicDispatch.yamlPath(process.cwd(), APPHOSTING_BASE_YAML_FILE);
   let projectYaml: yaml.Document;
   if (path) {
     projectYaml = dynamicDispatch.load(path);
@@ -144,7 +183,7 @@
         "It looks like you don't have an apphosting.yaml yet. Where would you like to store it?",
       default: process.cwd(),
     });
-    path = join(path, "apphosting.yaml");
+    path = join(path, APPHOSTING_BASE_YAML_FILE);
   }
   const envName = await dialogs.envVarForSecret(secretName);
   dynamicDispatch.upsertEnv(projectYaml, {

src/apphosting/secrets/index.spec.ts

@@ -9,6 +9,10 @@
 import * as gcsmImport from "../../gcp/secretManager";
 import * as utilsImport from "../../utils";
 import * as promptImport from "../../prompt";
+import { AppHostingYamlConfig } from "../yaml";
+
+import { Secret } from "../yaml";
+import { FirebaseError } from "../../error";
 
 describe("secrets", () => {
   let gcsm: sinon.SinonStubbedInstance<typeof gcsmImport>;
@@ -33,7 +37,7 @@
     it("uses explicit account", () => {
       const backend = {
         serviceAccount: "sa",
       } as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: "sa",
         runServiceAccount: "sa",
@@ -41,7 +45,7 @@
     });
 
     it("has a fallback for legacy SAs", () => {
       const backend = {} as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: gcb.getDefaultServiceAccount("number"),
         runServiceAccount: gce.getDefaultServiceAccount("number"),
@@ -256,4 +260,185 @@
       expect(gcsm.setIamPolicy).to.be.calledWithMatch(secret, newBindings);
     });
   });
+
+  describe("fetchSecrets", () => {
+    const projectId = "randomProject";
+    it("correctly attempts to fetch secret and it's version", async () => {
+      const secretSource: Secret[] = [
+        {
+          variable: "PINNED_API_KEY",
+          secret: "myApiKeySecret@5",
+        },
+      ];
+
+      gcsm.accessSecretVersion.returns(Promise.resolve("some-value"));
+      await secrets.fetchSecrets(projectId, secretSource);
+
+      expect(gcsm.accessSecretVersion).calledOnce;
+      expect(gcsm.accessSecretVersion).calledWithExactly(projectId, "myApiKeySecret", "5");
+    });
+
+    it("fetches latest version if version not explicitely provided", async () => {
+      const secretSource: Secret[] = [
+        {
+          variable: "VERBOSE_API_KEY",
+          secret: "projects/test-project/secrets/secretID",
+        },
+      ];
+
+      gcsm.accessSecretVersion.returns(Promise.resolve("some-value"));
+      await secrets.fetchSecrets(projectId, secretSource);
+
+      expect(gcsm.accessSecretVersion).calledOnce;
+      expect(gcsm.accessSecretVersion).calledWithExactly(
+        projectId,
+        "projects/test-project/secrets/secretID",
+        "latest",
+      );
+    });
+  });
+
+  describe("promptForAppHostingYaml", () => {
+    it("should prompt with the correct options", async () => {
+      const apphostingFileNameToPathMap = new Map<string, string>([
+        ["apphosting.yaml", "/parent/cwd/apphosting.yaml"],
+        ["apphosting.staging.yaml", "/parent/apphosting.staging.yaml"],
+      ]);
+
+      prompt.promptOnce.returns(Promise.resolve());
+
+      await secrets.promptForAppHostingYaml(apphostingFileNameToPathMap);
+
+      expect(prompt.promptOnce).to.have.been.calledWith({
+        name: "apphosting-yaml",
+        type: "list",
+        message: "Which environment would you like to export secrets from Secret Manager for?",
+        choices: [
+          {
+            name: "base (apphosting.yaml)",
+            value: "/parent/cwd/apphosting.yaml",
+          },
+          {
+            name: "staging (apphosting.yaml + apphosting.staging.yaml)",
+            value: "/parent/apphosting.staging.yaml",
+          },
+        ],
+      });
+    });
+  });
+
+  describe("getConfigToExport", () => {
+    let loadFromFileStub: sinon.SinonStub;
+    let baseAppHostingYaml: AppHostingYamlConfig;
+    let stagingAppHostingYaml: AppHostingYamlConfig;
+
+    const apphostingYamlPaths = ["/parent/cwd/apphosting.yaml", "/parent/apphosting.staging.yaml"];
+
+    beforeEach(() => {
+      loadFromFileStub = sinon.stub(AppHostingYamlConfig, "loadFromFile");
+      baseAppHostingYaml = AppHostingYamlConfig.empty();
+      baseAppHostingYaml.addEnvironmentVariable({
+        variable: "ENV_1",
+        value: "base_env_1",
+      });
+      baseAppHostingYaml.addEnvironmentVariable({
+        variable: "ENV_3",
+        value: "base_env_3",
+      });
+      baseAppHostingYaml.addSecret({
+        variable: "SECRET_1",
+        secret: "base_secret_1",
+      });
+      baseAppHostingYaml.addSecret({
+        variable: "SECRET_2",
+        secret: "base_secret_2",
+      });
+      baseAppHostingYaml.addSecret({
+        variable: "SECRET_3",
+        secret: "base_secret_3",
+      });
+
+      stagingAppHostingYaml = AppHostingYamlConfig.empty();
+      stagingAppHostingYaml.addEnvironmentVariable({
+        variable: "ENV_1",
+        value: "staging_env_1",
+      });
+      stagingAppHostingYaml.addEnvironmentVariable({
+        variable: "ENV_2",
+        value: "staging_env_2",
+      });
+      stagingAppHostingYaml.addSecret({
+        variable: "SECRET_1",
+        secret: "staging_secret_1",
+      });
+      stagingAppHostingYaml.addSecret({
+        variable: "SECRET_2",
+        secret: "staging_secret_2",
+      });
+
+      loadFromFileStub.callsFake(async (filePath) => {
+        if (filePath?.includes("apphosting.staging.yaml")) {
+          return Promise.resolve(stagingAppHostingYaml);
+        }
+        return Promise.resolve(baseAppHostingYaml);
+      });
+    });
+
+    afterEach(() => {
+      sinon.verifyAndRestore();
+    });
+
+    it("returns a config that complies with the expected precendence", async () => {
+      prompt.promptOnce.onFirstCall().returns(Promise.resolve("/parent/apphosting.staging.yaml"));
+
+      const resultingConfig = await secrets.getConfigToExport(apphostingYamlPaths);
+      expect(JSON.stringify(resultingConfig.environmentVariables)).to.equal(
+        JSON.stringify([
+          { variable: "ENV_1", value: "staging_env_1" },
+          { variable: "ENV_3", value: "base_env_3" },
+          { variable: "ENV_2", value: "staging_env_2" },
+        ]),
+      );
+
+      expect(JSON.stringify(resultingConfig.secrets)).to.equal(
+        JSON.stringify([
+          { variable: "SECRET_1", secret: "staging_secret_1" },
+          { variable: "SECRET_2", secret: "staging_secret_2" },
+          { variable: "SECRET_3", secret: "base_secret_3" },
+        ]),
+      );
+    });
+
+    it("returns appropriate config if only base file was selected", async () => {
+      prompt.promptOnce.onFirstCall().returns(Promise.resolve("/parent/apphosting.yaml"));
+
+      const resultingConfig = await secrets.getConfigToExport(apphostingYamlPaths);
+      expect(JSON.stringify(resultingConfig.environmentVariables)).to.equal(
+        JSON.stringify([
+          { variable: "ENV_1", value: "base_env_1" },
+          { variable: "ENV_3", value: "base_env_3" },
+        ]),
+      );
+
+      expect(JSON.stringify(resultingConfig.secrets)).to.equal(
+        JSON.stringify([
+          { variable: "SECRET_1", secret: "base_secret_1" },
+          { variable: "SECRET_2", secret: "base_secret_2" },
+          { variable: "SECRET_3", secret: "base_secret_3" },
+        ]),
+      );
+    });
+
+    it("returns throws an error if an invalid apphosting yaml if provided", async () => {
+      await expect(secrets.getConfigToExport(apphostingYamlPaths, "blah.txt")).to.be.rejectedWith(
+        FirebaseError,
+        /Invalid apphosting yaml file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'/,
+      );
+    });
+
+    it("does not prompt user if an appHostingfileToExportPath is provided", async () => {
+      await secrets.getConfigToExport(apphostingYamlPaths, "apphosting.staging.yaml");
+      expect(prompt.promptOnce).to.not.be.called;
+    });
+  });
 });