Apphosting emulator - environment variable support (#7781)

src/emulator/apphosting/config.spec.ts

@@ -0,0 +1,150 @@
+import * as emulatorUtils from "../../utils";
+import * as fsExtra from "fs-extra";
+import * as path from "path";
+import * as utils from "./utils";
+
+import * as sinon from "sinon";
+import { expect } from "chai";
+import { getLocalAppHostingConfiguration, loadAppHostingYaml } from "./config";
+
+const SAMPLE_APPHOSTING_YAML_CONFIG_ONE = {
+  env: [
+    {
+      variable: "STORAGE_BUCKET",
+      value: "mybucket.appspot.com",
+      availability: ["BUILD", "RUNTIME"],
+    },
+    {
+      variable: "API_KEY",
+      secret: "myApiKeySecret",
+    },
+    {
+      variable: "PINNED_API_KEY",
+      secret: "myApiKeySecret@5",
+    },
+    {
+      variable: "VERBOSE_API_KEY",
+      secret: "projects/test-project/secrets/secretID",
+    },
+    {
+      variable: "PINNED_VERBOSE_API_KEY",
+      secret: "projects/test-project/secrets/secretID/versions/5",
+    },
+  ],
+};
+
+const SAMPLE_APPHOSTING_YAML_CONFIG_TWO = {
+  env: [
+    {
+      variable: "randomEnvOne",
+      value: "envOne",
+    },
+    {
+      variable: "randomEnvTwo",
+      value: "envTwo",
+    },
+    {
+      variable: "randomEnvThree",
+      value: "envThree",
+    },
+    { variable: "randomSecretOne", secret: "secretOne" },
+    { variable: "randomSecretTwo", secret: "secretTwo" },
+    { variable: "randomSecretThree", secret: "secretThree" },
+  ],
+};
+
+const SAMPLE_APPHOSTING_YAML_CONFIG_THREE = {
+  env: [
+    {
+      variable: "randomEnvOne",
+      value: "envOne",
+    },
+    {
+      variable: "randomEnvTwo",
+      value: "blah",
+    },
+    {
+      variable: "randomEnvFour",
+      value: "envFour",
+    },
+    { variable: "randomSecretOne", secret: "bleh" },
+    { variable: "randomSecretTwo", secret: "secretTwo" },
+    { variable: "randomSecretFour", secret: "secretFour" },
+  ],
+};
+
+describe("environments", () => {
+  let pathExistsStub: sinon.SinonStub;
+  let joinStub: sinon.SinonStub;
+  let loggerStub: sinon.SinonStub;
+  let readFileFromDirectoryStub: sinon.SinonStub;
+  let wrappedSafeLoadStub: sinon.SinonStub;
+
+  beforeEach(() => {
+    pathExistsStub = sinon.stub(fsExtra, "pathExists");
+    joinStub = sinon.stub(path, "join");
+    loggerStub = sinon.stub(utils, "logger");
+    readFileFromDirectoryStub = sinon.stub(emulatorUtils, "readFileFromDirectory");
+    wrappedSafeLoadStub = sinon.stub(emulatorUtils, "wrappedSafeLoad");
+  });
+
+  afterEach(() => {
+    pathExistsStub.restore();
+    joinStub.restore();
+    loggerStub.restore();
+    readFileFromDirectoryStub.restore();
+    wrappedSafeLoadStub.restore();
+  });
+
+  describe("loadAppHostingYaml", () => {
+    it("should return a configuration in the correct format", async () => {
+      readFileFromDirectoryStub.returns({ source: "blah" });
+      wrappedSafeLoadStub.returns(SAMPLE_APPHOSTING_YAML_CONFIG_ONE);
+
+      const res = await loadAppHostingYaml("test", "test.yaml");
+      expect(JSON.stringify(res)).to.equal(
+        JSON.stringify({
+          environmentVariables: {
+            STORAGE_BUCKET: "mybucket.appspot.com",
+          },
+          secrets: {
+            API_KEY: "myApiKeySecret",
+            PINNED_API_KEY: "myApiKeySecret@5",
+            VERBOSE_API_KEY: "projects/test-project/secrets/secretID",
+            PINNED_VERBOSE_API_KEY: "projects/test-project/secrets/secretID/versions/5",
+          },
+        }),
+      );
+    });
+  });
+
+  describe("getLocalAppHostingConfiguration", () => {
+    it("should combine apphosting yaml files according to precedence", async () => {
+      pathExistsStub.returns(true);
+      readFileFromDirectoryStub.returns({ source: "blah" });
+
+      // Second config takes precedence
+      wrappedSafeLoadStub.onFirstCall().returns(SAMPLE_APPHOSTING_YAML_CONFIG_TWO);
+      wrappedSafeLoadStub.onSecondCall().returns(SAMPLE_APPHOSTING_YAML_CONFIG_THREE);
+
+      const apphostingConfig = await getLocalAppHostingConfiguration("test");
+
+      expect(JSON.stringify(apphostingConfig)).to.equal(
+        JSON.stringify({
+          environmentVariables: {
+            randomEnvOne: "envOne",
+            randomEnvTwo: "blah",
+            randomEnvThree: "envThree",
+            randomEnvFour: "envFour",
+          },
+          secrets: {
+            randomSecretOne: "bleh",
+            randomSecretTwo: "secretTwo",
+            randomSecretThree: "secretThree",
+            randomSecretFour: "secretFour",
+          },
+        }),
+      );
+    });
+  });
+});

src/emulator/apphosting/config.ts

@@ -0,0 +1,95 @@
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { readFileFromDirectory, wrappedSafeLoad } from "../../utils";
+import { logger } from "./utils";
+import { Emulators } from "../types";
+
+type EnvironmentAvailability = "BUILD" | "RUNTIME";
+
+const APPHOSTING_YAML = "apphosting.yaml";
+const APPHOSTING_LOCAL_YAML = "apphosting.local.yaml";
+
+// Schema of apphosting.*.yaml files
+interface AppHostingYaml {
+  env?: {
+    variable: string;
+    secret?: string;
+    value?: string;
+    availability?: EnvironmentAvailability[];
+  }[];
+}
+
+interface AppHostingConfiguration {
+  environmentVariables?: Record<string, string>;
+  secrets?: Record<string, string>;
+}
+
+/**
+ * Reads an apphosting.*.yaml file, parses, and returns in an easy to use
+ * format.
+ */
+export async function loadAppHostingYaml(
+  sourceDirectory: string,
+  fileName: string,
+): Promise<AppHostingConfiguration> {
+  const file = await readFileFromDirectory(sourceDirectory, fileName);
+  const apphostingYaml: AppHostingYaml = await wrappedSafeLoad(file.source);
+
+  const environmentVariables: Record<string, string> = {};
+  const secrets: Record<string, string> = {};
+
+  if (apphostingYaml.env) {
+    for (const env of apphostingYaml.env) {
+      if (env.value) {
+        environmentVariables[env.variable] = env.value;
+      }
+
+      if (env.secret) {
+        secrets[env.variable] = env.secret;
+      }
+    }
+  }
+
+  return { environmentVariables, secrets };
+}
+
+/**
+ * Loads in apphosting.yaml & apphosting.local.yaml, giving
+ * apphosting.local.yaml precedence if present.
+ */
+export async function getLocalAppHostingConfiguration(
+  sourceDirectory: string,
+): Promise<AppHostingConfiguration> {
+  let apphostingBaseConfig: AppHostingConfiguration = {};
+  let apphostingLocalConfig: AppHostingConfiguration = {};
+
+  if (await pathExists(join(sourceDirectory, APPHOSTING_YAML))) {
+    logger.logLabeled(
+      "SUCCESS",
+      Emulators.APPHOSTING,
+      `${APPHOSTING_YAML} found, loading configuration`,
+    );
+    apphostingBaseConfig = await loadAppHostingYaml(sourceDirectory, APPHOSTING_YAML);
+  }
+
+  if (await pathExists(join(sourceDirectory, APPHOSTING_LOCAL_YAML))) {
+    logger.logLabeled(
+      "SUCCESS",
+      Emulators.APPHOSTING,
+      `${APPHOSTING_LOCAL_YAML} found, loading configuration`,
+    );
+    apphostingLocalConfig = await loadAppHostingYaml(sourceDirectory, APPHOSTING_LOCAL_YAML);
+  }
+
+  // Combine apphosting configurations in order of lowest precedence to highest
+  return {
+    environmentVariables: {
+      ...apphostingBaseConfig.environmentVariables,
+      ...apphostingLocalConfig.environmentVariables,
+    },
+    secrets: {
+      ...apphostingBaseConfig.secrets,
+      ...apphostingLocalConfig.secrets,
+    },
+  };
+}

src/emulator/apphosting/index.ts

@@ -1,6 +1,6 @@
-import { EmulatorLogger } from "../emulatorLogger";
 import { EmulatorInfo, EmulatorInstance, Emulators } from "../types";
 import { start as apphostingStart } from "./serve";
+import { logger } from "./utils";
 interface AppHostingEmulatorArgs {
   options?: any;
   port?: number;
@@ -12,24 +12,23 @@ interface AppHostingEmulatorArgs {
  * environment for testing App Hosting features locally.
  */
 export class AppHostingEmulator implements EmulatorInstance {
-  private logger = EmulatorLogger.forEmulator(Emulators.APPHOSTING);
   constructor(private args: AppHostingEmulatorArgs) {}
 
   async start(): Promise<void> {
-    this.logger.logLabeled("INFO", Emulators.APPHOSTING, "starting apphosting emulator");
+    logger.logLabeled("INFO", Emulators.APPHOSTING, "starting apphosting emulator");
 
     const { hostname, port } = await apphostingStart();
     this.args.options.host = hostname;
     this.args.options.port = port;
   }
 
   connect(): Promise<void> {
-    this.logger.logLabeled("INFO", Emulators.APPHOSTING, "connecting apphosting emulator");
+    logger.logLabeled("INFO", Emulators.APPHOSTING, "connecting apphosting emulator");
     return Promise.resolve();
   }
 
   stop(): Promise<void> {
-    this.logger.logLabeled("INFO", Emulators.APPHOSTING, "stopping apphosting emulator");
+    logger.logLabeled("INFO", Emulators.APPHOSTING, "stopping apphosting emulator");
     return Promise.resolve();
   }
 

src/emulator/apphosting/serve.spec.ts

@@ -1,34 +1,38 @@
-import * as portUtils from "../portUtils";
 import * as sinon from "sinon";
-import * as spawn from "../../init/spawn";
 import { expect } from "chai";
+import * as portUtils from "../portUtils";
+import * as spawn from "../../init/spawn";
 import * as serve from "./serve";
 import { DEFAULT_PORTS } from "../constants";
 import * as utils from "./utils";
+import * as configs from "./config";
 
 describe("serve", () => {
   let checkListenableStub: sinon.SinonStub;
   let wrapSpawnStub: sinon.SinonStub;
   let discoverPackageManagerStub: sinon.SinonStub;
+  let getLocalAppHostingConfigurationStub: sinon.SinonStub;
 
   beforeEach(() => {
     checkListenableStub = sinon.stub(portUtils, "checkListenable");
     wrapSpawnStub = sinon.stub(spawn, "wrapSpawn");
     discoverPackageManagerStub = sinon.stub(utils, "discoverPackageManager");
+    getLocalAppHostingConfigurationStub = sinon.stub(configs, "getLocalAppHostingConfiguration");
   });
 
   afterEach(() => {
     checkListenableStub.restore();
     wrapSpawnStub.restore();
     discoverPackageManagerStub.restore();
+    getLocalAppHostingConfigurationStub.restore();
   });
 
   describe("start", () => {
     it("should only select an available port to serve", async () => {
       checkListenableStub.onFirstCall().returns(false);
       checkListenableStub.onSecondCall().returns(false);
       checkListenableStub.onThirdCall().returns(true);
-
+      getLocalAppHostingConfigurationStub.returns({ environmentVariables: {}, secrets: {} });
       const res = await serve.start();
       expect(res.port).to.equal(DEFAULT_PORTS.apphosting + 2);
     });

src/emulator/apphosting/serve.ts

@@ -8,6 +8,7 @@ import { checkListenable } from "../portUtils";
 import { discoverPackageManager } from "./utils";
 import { DEFAULT_HOST, DEFAULT_PORTS } from "../constants";
 import { wrapSpawn } from "../../init/spawn";
+import { getLocalAppHostingConfiguration } from "./config";
 
 /**
  * Spins up a project locally by running the project's dev command.
@@ -32,8 +33,12 @@ export async function start(): Promise<{ hostname: string; port: number }> {
 async function serve(port: number): Promise<void> {
   const rootDir = process.cwd();
   const packageManager = await discoverPackageManager(rootDir);
+  const apphostingLocalConfig = await getLocalAppHostingConfiguration(rootDir);
 
-  await wrapSpawn(packageManager, ["run", "dev"], rootDir, { PORT: port });
+  await wrapSpawn(packageManager, ["run", "dev"], rootDir, {
+    ...apphostingLocalConfig.environmentVariables,
+    PORT: port,
+  });
 }
 
 function availablePort(host: string, port: number): Promise<boolean> {

src/emulator/apphosting/utils.ts

@@ -1,8 +1,12 @@
 import { pathExists } from "fs-extra";
 import { join } from "path";
+import { EmulatorLogger } from "../emulatorLogger";
+import { Emulators } from "../types";
+
+export const logger = EmulatorLogger.forEmulator(Emulators.APPHOSTING);
 
 /**
- * Exported for unit testing
+ * Supported package managers. This mirrors production.
  */
 export type PackageManager = "npm" | "yarn" | "pnpm";