Add localBuild and zip upload support into firebase deploy (#9193)

Author: annajowang

npm-shrinkwrap.json

@@ -102,6 +102,8 @@
     ]
   },
   "dependencies": {
+    "@apphosting/build": "^0.1.6",
+    "@apphosting/common": "^0.0.8",
     "@electric-sql/pglite": "^0.3.3",
     "@electric-sql/pglite-tools": "^0.2.8",
     "@google-cloud/cloud-sql-connector": "^1.3.3",

package.json

@@ -1107,6 +1107,9 @@
                             },
                             "type": "array"
                         },
+                        "localBuild": {
+                            "type": "boolean"
+                        },
                         "rootDir": {
                             "type": "string"
                         }
@@ -1134,6 +1137,9 @@
                                 },
                                 "type": "array"
                             },
+                            "localBuild": {
+                                "type": "boolean"
+                            },
                             "rootDir": {
                                 "type": "string"
                             }

schema/firebase-config.json

@@ -0,0 +1,47 @@
+import * as sinon from "sinon";
+import { expect } from "chai";
+import * as localBuildModule from "@apphosting/build";
+import { localBuild } from "./localbuilds";
+
+describe("localBuild", () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it("returns the expected output", async () => {
+    const bundleConfig = {
+      version: "v1" as const,
+      runConfig: {
+        runCommand: "npm run build:prod",
+      },
+      metadata: {
+        adapterPackageName: "@apphosting/angular-adapter",
+        adapterVersion: "14.1",
+        framework: "nextjs",
+      },
+      outputFiles: {
+        serverApp: {
+          include: ["./next/standalone"],
+        },
+      },
+    };
+    const expectedAnnotations = {
+      adapterPackageName: "@apphosting/angular-adapter",
+      adapterVersion: "14.1",
+      framework: "nextjs",
+    };
+    const expectedOutputFiles = ["./next/standalone"];
+    const expectedBuildConfig = {
+      runCommand: "npm run build:prod",
+      env: [],
+    };
+    const localApphostingBuildStub: sinon.SinonStub = sinon
+      .stub(localBuildModule, "localBuild")
+      .resolves(bundleConfig);
+    const { outputFiles, annotations, buildConfig } = await localBuild("./", "nextjs");
+    expect(annotations).to.deep.equal(expectedAnnotations);
+    expect(buildConfig).to.deep.equal(expectedBuildConfig);
+    expect(outputFiles).to.deep.equal(expectedOutputFiles);
+    sinon.assert.calledWith(localApphostingBuildStub, "./", "nextjs");
+  });
+});

src/apphosting/localbuilds.spec.ts

@@ -0,0 +1,37 @@
+import { BuildConfig, Env } from "../gcp/apphosting";
+import { localBuild as localAppHostingBuild } from "@apphosting/build";
+
+/**
+ * Triggers a local apphosting build.
+ */
+export async function localBuild(
+  projectRoot: string,
+  framework: string,
+): Promise<{
+  outputFiles: string[];
+  annotations: Record<string, string>;
+  buildConfig: BuildConfig;
+}> {
+  const apphostingBuildOutput = await localAppHostingBuild(projectRoot, framework);
+
+  const annotations: Record<string, string> = Object.fromEntries(
+    Object.entries(apphostingBuildOutput.metadata).map(([key, value]) => [key, String(value)]),
+  );
+
+  const env: Env[] | undefined = apphostingBuildOutput.runConfig.environmentVariables?.map(
+    ({ variable, value, availability }) => ({
+      variable,
+      value,
+      availability,
+    }),
+  );
+
+  return {
+    outputFiles: apphostingBuildOutput.outputFiles?.serverApp.include ?? [],
+    annotations,
+    buildConfig: {
+      runCommand: apphostingBuildOutput.runConfig.runCommand,
+      env: env ?? [],
+    },
+  };
+}

src/apphosting/localbuilds.ts

@@ -1,7 +1,15 @@
 import { AppHostingSingle } from "../../firebaseConfig";
+import { BuildConfig } from "../../gcp/apphosting";
+
+export interface LocalBuild {
+  buildConfig: BuildConfig;
+  buildDir: string;
+  annotations: Record<string, string>;
+}
 
 export interface Context {
   backendConfigs: Record<string, AppHostingSingle>;
   backendLocations: Record<string, string>;
   backendStorageUris: Record<string, string>;
+  backendLocalBuilds: Record<string, LocalBuild>;
 }

src/deploy/apphosting/args.ts

@@ -30,9 +30,22 @@ function initializeContext(): Context {
         rootDir: "/",
         ignore: [],
       },
+      fooLocalBuild: {
+        backendId: "fooLocalBuild",
+        rootDir: "/",
+        ignore: [],
+        localBuild: true,
+      },
     },
-    backendLocations: { foo: "us-central1" },
+    backendLocations: { foo: "us-central1", fooLocalBuild: "us-central1" },
     backendStorageUris: {},
+    backendLocalBuilds: {
+      fooLocalBuild: {
+        buildDir: "./nextjs/standalone",
+        buildConfig: {},
+        annotations: {},
+      },
+    },
   };
 }
 
@@ -59,17 +72,25 @@ describe("apphosting", () => {
     sinon.verifyAndRestore();
   });
 
-  describe("deploy", () => {
+  describe("deploy local source", () => {
     const opts = {
       ...BASE_OPTS,
       projectId: "my-project",
       only: "apphosting",
       config: new Config({
-        apphosting: {
-          backendId: "foo",
-          rootDir: "/",
-          ignore: [],
-        },
+        apphosting: [
+          {
+            backendId: "foo",
+            rootDir: "/",
+            ignore: [],
+          },
+          {
+            backendId: "fooLocalBuild",
+            rootDir: "/",
+            ignore: [],
+            localBuild: true,
+          },
+        ],
       }),
     };
 
@@ -78,18 +99,26 @@ describe("apphosting", () => {
       const projectNumber = "000000000000";
       const location = "us-central1";
       const bucketName = `firebaseapphosting-sources-${projectNumber}-${location}`;
-
       getProjectNumberStub.resolves(projectNumber);
       upsertBucketStub.resolves(bucketName);
-      createArchiveStub.resolves("path/to/foo-1234.zip");
-      uploadObjectStub.resolves({
+      createArchiveStub.onFirstCall().resolves("path/to/foo-1234.zip");
+      createArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.zip");
+
+      uploadObjectStub.onFirstCall().resolves({
         bucket: bucketName,
         object: "foo-1234",
       });
+      uploadObjectStub.onSecondCall().resolves({
+        bucket: bucketName,
+        object: "foo-local-build-1234",
+      });
+
       createReadStreamStub.returns("stream" as any);
 
       await deploy(context, opts);
 
+      // assert backend foo calls
+
       expect(upsertBucketStub).to.be.calledWith({
         product: "apphosting",
         createMessage: `Creating Cloud Storage bucket in ${location} to store App Hosting source code uploads at ${bucketName}...`,
@@ -108,26 +137,65 @@ describe("apphosting", () => {
           },
         },
       });
+
+      // assert backend fooLocalBuild calls
+      expect(upsertBucketStub).to.be.calledWith({
+        product: "apphosting",
+        createMessage:
+          "Creating Cloud Storage bucket in us-central1 to store App Hosting source code uploads at firebaseapphosting-sources-000000000000-us-central1...",
+        projectId: "my-project",
+        req: {
+          baseName: "firebaseapphosting-sources-000000000000-us-central1",
+          purposeLabel: `apphosting-source-${location}`,
+          location: "us-central1",
+          lifecycle: {
+            rule: [
+              {
+                action: { type: "Delete" },
+                condition: { age: 30 },
+              },
+            ],
+          },
+        },
+      });
+      expect(createArchiveStub).to.be.calledWithExactly(
+        context.backendConfigs["fooLocalBuild"],
+        process.cwd(),
+        "./nextjs/standalone",
+      );
+      expect(uploadObjectStub).to.be.calledWithMatch(
+        sinon.match.any,
+        "firebaseapphosting-sources-000000000000-us-central1",
+      );
     });
 
     it("correctly creates and sets storage URIs", async () => {
       const context = initializeContext();
       const projectNumber = "000000000000";
       const location = "us-central1";
       const bucketName = `firebaseapphosting-sources-${projectNumber}-${location}`;
-
       getProjectNumberStub.resolves(projectNumber);
       upsertBucketStub.resolves(bucketName);
-      createArchiveStub.resolves("path/to/foo-1234.zip");
-      uploadObjectStub.resolves({
+      createArchiveStub.onFirstCall().resolves("path/to/foo-1234.zip");
+      createArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.zip");
+
+      uploadObjectStub.onFirstCall().resolves({
         bucket: bucketName,
         object: "foo-1234",
       });
+
+      uploadObjectStub.onSecondCall().resolves({
+        bucket: bucketName,
+        object: "foo-local-build-1234",
+      });
       createReadStreamStub.returns("stream" as any);
 
       await deploy(context, opts);
 
       expect(context.backendStorageUris["foo"]).to.equal(`gs://${bucketName}/foo-1234.zip`);
+      expect(context.backendStorageUris["fooLocalBuild"]).to.equal(
+        `gs://${bucketName}/foo-local-build-1234.zip`,
+      );
     });
   });
 });

src/deploy/apphosting/deploy.spec.ts

@@ -26,7 +26,13 @@ export default async function (context: Context, options: Options): Promise<void
   // Ensure that a bucket exists in each region that a backend is or will be deployed to
   const bucketsPerLocation: Record<string, string> = {};
   await Promise.all(
-    Object.values(context.backendLocations).map(async (loc) => {
+    Object.entries(context.backendLocations).map(async ([backendId, loc]) => {
+      const cfg = context.backendConfigs[backendId];
+      if (!cfg) {
+        throw new FirebaseError(
+          `Failed to find config for backend ${backendId}. Please contact support with the contents of your firebase-debug.log to report your issue.`,
+        );
+      }
       const baseName = `firebaseapphosting-sources-${options.projectNumber}-${loc.toLowerCase()}`;
       const resolvedName = await gcs.upsertBucket({
         product: "apphosting",
@@ -54,10 +60,23 @@ export default async function (context: Context, options: Options): Promise<void
     }),
   );
 
+  // Zip and upload code to GCS bucket.
   await Promise.all(
     Object.values(context.backendConfigs).map(async (cfg) => {
-      const projectSourcePath = options.projectRoot ? options.projectRoot : process.cwd();
-      const zippedSourcePath = await createArchive(cfg, projectSourcePath);
+      const rootDir = options.projectRoot ?? process.cwd();
+      let builtAppDir;
+      if (cfg.localBuild) {
+        builtAppDir = context.backendLocalBuilds[cfg.backendId].buildDir;
+        if (!builtAppDir) {
+          throw new FirebaseError(`No local build dir found for ${cfg.backendId}`);
+        }
+      }
+      const zippedSourcePath = await createArchive(cfg, rootDir, builtAppDir);
+      logLabeledBullet(
+        "apphosting",
+        `Zipped ${cfg.localBuild ? "built app" : "source"} for backend ${cfg.backendId}`,
+      );
+
       const backendLocation = context.backendLocations[cfg.backendId];
       if (!backendLocation) {
         throw new FirebaseError(
@@ -66,7 +85,7 @@ export default async function (context: Context, options: Options): Promise<void
       }
       logLabeledBullet(
         "apphosting",
-        `Uploading source code at ${projectSourcePath} for backend ${cfg.backendId}...`,
+        `Uploading ${cfg.localBuild ? "built app" : "source"} for backend ${cfg.backendId}...`,
       );
       const bucketName = bucketsPerLocation[backendLocation]!;
       const { bucket, object } = await gcs.uploadObject(
@@ -76,7 +95,7 @@ export default async function (context: Context, options: Options): Promise<void
         },
         bucketName,
       );
-      logLabeledBullet("apphosting", `Source code uploaded at gs://${bucket}/${object}`);
+      logLabeledBullet("apphosting", `Uploaded at gs://${bucket}/${object}`);
       context.backendStorageUris[cfg.backendId] =
         `gs://${bucketName}/${path.basename(zippedSourcePath)}`;
     }),