Apphosting emulator - local secrets support via apphosting:config:export command

src/apphosting/config.spec.ts

@@ -214,37 +214,7 @@ env:
     });
   });
 
-  describe("listAppHostingYamlsInCwd", () => {
-    let fs: sinon.SinonStubbedInstance<typeof fsImport>;
-
-    beforeEach(() => {
-      fs = sinon.stub(fsImport);
-    });
-
-    afterEach(() => {
-      sinon.verifyAndRestore();
-    });
-
-    it("lists apphosting yamls only", () => {
-      fs.listFiles
-        .withArgs("/cwd")
-        .returns([
-          "apphosting.staging.yaml",
-          "blah.js",
-          "apphosting.yaml",
-          "apphosting.local.yaml",
-          "blah.txt",
-        ]);
-
-      const apphostingYamls = config.list("/cwd");
-      expect(apphostingYamls).to.deep.equal([
-        "/cwd/apphosting.staging.yaml",
-        "/cwd/apphosting.yaml",
-        "/cwd/apphosting.local.yaml",
-      ]);
-    });
-  });
-  describe("allYamlPaths", () => {
+  describe("discoverConfigsInProject", () => {
     let fs: sinon.SinonStubbedInstance<typeof fsImport>;
 
     beforeEach(() => {
@@ -262,13 +232,13 @@ env:
 
       fs.listFiles
         .withArgs("/parent-parent/parent/cwd")
-        .returns(["apphosting.staging.yaml", "blah.js"]);
+        .returns(["apphosting.staging.yaml", "blah.js", "apphosting.foo.bar.yaml"]);
 
       fs.listFiles
         .withArgs("/parent-parent/parent")
         .returns(["apphosting.local.yaml", "bloh.txt", "apphosting.yaml"]);
 
-      const apphostingYamls = config.allYamlPaths("/parent-parent/parent/cwd");
+      const apphostingYamls = config.discoverConfigsInProject("/parent-parent/parent/cwd");
       expect(apphostingYamls).to.deep.equal([
         "/parent-parent/parent/cwd/apphosting.staging.yaml",
         "/parent-parent/parent/apphosting.local.yaml",

src/apphosting/config.ts

@@ -8,7 +8,8 @@
 import * as dialogs from "./secrets/dialogs";
 
 export const APPHOSTING_BASE_YAML_FILE = "apphosting.yaml";
-export const APPHOSTING_LOCAL_YAML = "apphosting.local.yaml";
+export const APPHOSTING_LOCAL_YAML_FILE = "apphosting.local.yaml";
+export const APPHOSTING_YAML_FILE_REGEX = /^apphosting(\.[a-zA-Z0-9]+)?\.yaml$/;
 
 export interface RunConfig {
   concurrency?: number;
@@ -36,15 +37,15 @@
 }
 
 /**
- * Finds the path of apphosting.yaml.
- * Starts with cwd and walks up the tree until apphosting.yaml is found or
+ * Finds the path of a yaml file.
+ * Starts with cwd and walks up the tree until yamlFileName is found or
  * we find the project root (where firebase.json is) or the filesystem root;
  * in these cases, returns null.
  */
-export function yamlPath(cwd: string, fileName: string): string | null {
+export function yamlPath(cwd: string, yamlFileName: string): string | null {
   let dir = cwd;
 
-  while (!fs.fileExistsSync(resolve(dir, fileName))) {
+  while (!fs.fileExistsSync(resolve(dir, yamlFileName))) {
     // We've hit project root
     if (fs.fileExistsSync(resolve(dir, "firebase.json"))) {
       return null;
@@ -57,27 +58,31 @@
     }
     dir = parent;
   }
-  return resolve(dir, fileName);
+  return resolve(dir, yamlFileName);
 }
 
 /**
- * Finds all paths to `apphosting.*.yaml` files within a project.
+ * Finds all paths to `apphosting.*.yaml` configs within a project.
  *
  * This function starts at the provided directory (`cwd`) and traverses
  * upwards through the file system until it finds a `firebase.json` file
  * (indicating the project root) or reaches the root of the filesystem.
  * Along the way, it collects the paths of all encountered `apphosting.*.yaml` files.
  *
  * @param cwd The directory to start the search from.
  * @returns An array of strings representing the paths to all found `apphosting.*.yaml` files,
  *          or `null` if no such files are found.
  */
-export function allYamlPaths(cwd: string): string[] | null {
+export function discoverConfigsInProject(cwd: string): string[] | null {
   let dir = cwd;
   const files: string[] = [];
 
   do {
-    files.push(...list(dir));
+    const apphostingYamlFiles = fs
+      .listFiles(dir)
+      .filter((file) => APPHOSTING_YAML_FILE_REGEX.test(file));
+    const apphostingYamlFilePaths = apphostingYamlFiles.map((file) => join(dir, file));
+    files.push(...apphostingYamlFilePaths);
 
     const parent = dirname(dir);
     // We've hit the filesystem root
@@ -91,19 +96,6 @@
   return files.length > 0 ? files : null;
 }
 
-/**
- * Lists all apphosting.*.yaml files in the given directory.
- */
-export function list(cwd: string): string[] {
-  const paths: string[] = [];
-  for (const file of fs.listFiles(cwd)) {
-    if (file.startsWith("apphosting.") && file.endsWith(".yaml")) {
-      paths.push(join(cwd, file));
-    }
-  }
-  return paths;
-}
-
 /** Load apphosting.yaml */
 export function load(yamlPath: string): yaml.Document {
   const raw = fs.readFile(yamlPath);

src/apphosting/secrets/index.spec.ts

@@ -9,9 +9,10 @@
 import * as gcsmImport from "../../gcp/secretManager";
 import * as utilsImport from "../../utils";
 import * as promptImport from "../../prompt";
-import * as apphostingYamlImport from "../yaml";
+import { AppHostingYamlConfig } from "../yaml";
 
 import { Secret } from "../yaml";
+import { FirebaseError } from "../../error";
 
 describe("secrets", () => {
   let gcsm: sinon.SinonStubbedInstance<typeof gcsmImport>;
@@ -36,7 +37,7 @@
     it("uses explicit account", () => {
       const backend = {
         serviceAccount: "sa",
       } as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: "sa",
         runServiceAccount: "sa",
@@ -44,7 +45,7 @@
     });
 
     it("has a fallback for legacy SAs", () => {
       const backend = {} as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: gcb.getDefaultServiceAccount("number"),
         runServiceAccount: gce.getDefaultServiceAccount("number"),
@@ -327,16 +328,15 @@
   });
 
   describe("getConfigToExport", () => {
-    let loadAppHostingYamlStub: sinon.SinonStub;
-    let baseAppHostingYaml: apphostingYamlImport.AppHostingYamlConfig;
-    let stagingAppHostingYaml: apphostingYamlImport.AppHostingYamlConfig;
+    let loadFromFileStub: sinon.SinonStub;
+    let baseAppHostingYaml: AppHostingYamlConfig;
+    let stagingAppHostingYaml: AppHostingYamlConfig;
 
     const apphostingYamlPaths = ["/parent/cwd/apphosting.yaml", "/parent/apphosting.staging.yaml"];
 
     beforeEach(() => {
-      loadAppHostingYamlStub = sinon.stub(apphostingYamlImport, "loadAppHostingYaml");
-
-      baseAppHostingYaml = new apphostingYamlImport.AppHostingYamlConfig();
+      loadFromFileStub = sinon.stub(AppHostingYamlConfig, "loadFromFile");
+      baseAppHostingYaml = AppHostingYamlConfig.empty();
       baseAppHostingYaml.addEnvironmentVariable({
         variable: "ENV_1",
         value: "base_env_1",
@@ -358,7 +358,7 @@
         secret: "base_secret_3",
       });
 
-      stagingAppHostingYaml = new apphostingYamlImport.AppHostingYamlConfig();
+      stagingAppHostingYaml = AppHostingYamlConfig.empty();
       stagingAppHostingYaml.addEnvironmentVariable({
         variable: "ENV_1",
         value: "staging_env_1",
@@ -376,8 +376,8 @@
         secret: "staging_secret_2",
       });
 
-      loadAppHostingYamlStub.callsFake(async (filePath) => {
+      loadFromFileStub.callsFake(async (filePath) => {
         if (filePath?.includes("apphosting.staging.yaml")) {
           return Promise.resolve(stagingAppHostingYaml);
         }
         return Promise.resolve(baseAppHostingYaml);
@@ -428,5 +428,17 @@
         ]),
       );
     });
+
+    it("returns throws an error if an invalid apphosting yaml if provided", async () => {
+      await expect(secrets.getConfigToExport(apphostingYamlPaths, "blah.txt")).to.be.rejectedWith(
+        FirebaseError,
+        /Invalid apphosting yaml file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'/,
+      );
+    });
+
+    it("does not prompt user if an appHostingfileToExportPath is provided", async () => {
+      await secrets.getConfigToExport(apphostingYamlPaths, "apphosting.staging.yaml");
+      expect(prompt.promptOnce).to.not.be.called;
+    });
   });
 });

src/apphosting/secrets/index.ts

@@ -9,9 +9,9 @@
 import * as utils from "../../utils";
 import * as prompt from "../../prompt";
 import { basename } from "path";
-import { APPHOSTING_BASE_YAML_FILE } from "../config";
+import { APPHOSTING_BASE_YAML_FILE, APPHOSTING_YAML_FILE_REGEX } from "../config";
 import { getEnvironmentName } from "../utils";
-import { AppHostingYamlConfig, Secret, loadAppHostingYaml } from "../yaml";
+import { AppHostingYamlConfig, Secret } from "../yaml";
 
 /** Interface for holding the service account pair for a given Backend. */
 export interface ServiceAccounts {
@@ -94,10 +94,10 @@
   let existingBindings;
   try {
     existingBindings = (await gcsm.getIamPolicy({ projectId, name: secretName })).bindings || [];
   } catch (err: any) {
     throw new FirebaseError(
       `Failed to get IAM bindings on secret: ${secretName}. Ensure you have the permissions to do so and try again.`,
       { original: err },
     );
   }
 
@@ -105,10 +105,10 @@
     // TODO: Merge with existing bindings with the same role
     const updatedBindings = existingBindings.concat(newBindings);
     await gcsm.setIamPolicy({ projectId, name: secretName }, updatedBindings);
   } catch (err: any) {
     throw new FirebaseError(
       `Failed to set IAM bindings ${JSON.stringify(newBindings)} on secret: ${secretName}. Ensure you have the permissions to do so and try again.`,
       { original: err },
     );
   }
 
@@ -171,35 +171,35 @@
 
 /**
  * Fetches secrets from Google Secret Manager and returns their values in plain text.
- *
- * This function takes an array of `Secret` objects, each containing the name
- * and URL of a secret stored in Google Secret Manager. It retrieves the
- * secrets from Secret Manager and returns a Map where the keys are the secret
- * names and the values are the corresponding secret values in plain text.
- *
- * @param projectId The ID of the Google Cloud project where the secrets are stored.
- * @param secrets An array of `Secret` objects, each specifying the name and URL of a secret.
- * @returns A Promise that resolves to a Map containing the secret names as keys and their
- *          plain text values as values.
  */
 export async function fetchSecrets(
   projectId: string,
   secrets: Secret[],
 ): Promise<Map<string, string>> {
-  const secretsKeyValuePairs: Map<string, string> = new Map();
+  let secretsKeyValuePairs: Map<string, string>;
 
   try {
-    for (const secretConfig of secrets) {
+    const secretPromises: Promise<[string, string]>[] = secrets.map(async (secretConfig) => {
+      /**
+       * secretConfig.secret expected to be in fromat "myApiKeySecret@5",
+       * "projects/test-project/secrets/secretID", or
+       * "projects/test-project/secrets/secretID/versions/5"
+       */
       let [name, version] = secretConfig.secret!.split("@");
       if (!version) {
         version = "latest";
       }
 
       const value = await gcsm.accessSecretVersion(projectId, name, version);
-      secretsKeyValuePairs.set(secretConfig.variable, value);
-    }
-  } catch (e) {
-    throw new FirebaseError(`Error exporting secrets: ${e}`);
+      return [secretConfig.variable, value] as [string, string];
+    });
+
+    const secretEntries = await Promise.all(secretPromises);
+    secretsKeyValuePairs = new Map(secretEntries);
+  } catch (e: any) {
+    throw new FirebaseError(`Error exporting secrets`, {
+      original: e,
+    });
   }
 
   return secretsKeyValuePairs;
@@ -215,25 +215,33 @@
  * to ensure the exported secrets accurately reflect the chosen environment.
  *
  * @param allAppHostingYamlPaths An array of paths to all available `apphosting.*.yaml` files.
- * @returns A Promise that resolves to an `AppHostingYamlConfig` object representing the combined
- *          configuration to be used for exporting configurations.
  */
 export async function getConfigToExport(
   allAppHostingYamlPaths: string[],
+  appHostingfileToExportPath?: string,
 ): Promise<AppHostingYamlConfig> {
+  if (appHostingfileToExportPath && !APPHOSTING_YAML_FILE_REGEX.test(appHostingfileToExportPath)) {
+    throw new FirebaseError(
+      "Invalid apphosting yaml file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
+    );
+  }
+
   const fileNameToPathMap: Map<string, string> = new Map();
   for (const path of allAppHostingYamlPaths) {
     const fileName = basename(path);
     fileNameToPathMap.set(fileName, path);
   }
   const baseFilePath = fileNameToPathMap.get(APPHOSTING_BASE_YAML_FILE);
-  const appHostingfileToExportPath = await promptForAppHostingYaml(fileNameToPathMap);
+  if (!appHostingfileToExportPath) {
+    // If file is not provided, prompt the user
+    appHostingfileToExportPath = await promptForAppHostingYaml(fileNameToPathMap);
+  }
 
-  const envConfig = await loadAppHostingYaml(appHostingfileToExportPath);
+  const envConfig = await AppHostingYamlConfig.loadFromFile(appHostingfileToExportPath);
 
   // if the base file exists we'll include it
   if (baseFilePath) {
-    const baseConfig = await loadAppHostingYaml(baseFilePath);
+    const baseConfig = await AppHostingYamlConfig.loadFromFile(baseFilePath);
 
     // if the user had selected the base file only, thats okay becuase logic below won't alter the config or cause duplicates
     baseConfig.merge(envConfig);
@@ -250,7 +258,9 @@
  * will prompt the user for an apphosting configuration. It returns the path
  * of the chosen apphosting yaml file.
  */
-export async function promptForAppHostingYaml(apphostingFileNameToPathMap: Map<string, string>) {
+export async function promptForAppHostingYaml(
+  apphostingFileNameToPathMap: Map<string, string>,
+): Promise<string> {
   const fileNames = Array.from(apphostingFileNameToPathMap.keys());
 
   const baseFilePath = apphostingFileNameToPathMap.get(APPHOSTING_BASE_YAML_FILE);

src/apphosting/yaml.spec.ts

@@ -5,7 +5,7 @@ describe("yaml", () => {
   describe("environment variables", () => {
     let apphostingYaml: AppHostingYamlConfig;
     beforeEach(() => {
-      apphostingYaml = new AppHostingYamlConfig();
+      apphostingYaml = AppHostingYamlConfig.empty();
     });
 
     it("adds environment variables and retrieves them correctly", () => {
@@ -48,31 +48,12 @@ describe("yaml", () => {
         JSON.stringify([{ variable: "TEST", value: "overwritten_value" }]),
       );
     });
-
-    it("converts environment variables to records correctly", () => {
-      apphostingYaml.addEnvironmentVariable({
-        variable: "TEST_1",
-        value: "value_1",
-      });
-
-      apphostingYaml.addEnvironmentVariable({
-        variable: "TEST_2",
-        value: "value_2",
-      });
-
-      expect(JSON.stringify(apphostingYaml.environmentVariablesAsRecord())).to.equal(
-        JSON.stringify({
-          TEST_1: "value_1",
-          TEST_2: "value_2",
-        }),
-      );
-    });
   });
 
   describe("secrets", () => {
     let apphostingYaml: AppHostingYamlConfig;
     beforeEach(() => {
-      apphostingYaml = new AppHostingYamlConfig();
+      apphostingYaml = AppHostingYamlConfig.empty();
     });
 
     it("adds environment variables and retrieves them correctly", () => {
@@ -120,7 +101,7 @@ describe("yaml", () => {
   describe("merge", () => {
     let apphostingYaml: AppHostingYamlConfig;
     beforeEach(() => {
-      apphostingYaml = new AppHostingYamlConfig();
+      apphostingYaml = AppHostingYamlConfig.empty();
     });
 
     it("merges incoming apphosting yaml config with precendence", () => {
@@ -141,7 +122,7 @@ describe("yaml", () => {
         secret: "secret_2",
       });
 
-      const incomingAppHostingYaml = new AppHostingYamlConfig();
+      const incomingAppHostingYaml = AppHostingYamlConfig.empty();
       incomingAppHostingYaml.addEnvironmentVariable({
         variable: "ENV_1",
         value: "incoming_env_1",