🏗 Persist merge commit for reuse with later CI jobs (#33165)

* Fetch and store merge_commit at the beginning of the build * Log MERGE_SHA during Setup ENV stage * Fix GET_MERGE_SHA * Simplify and speed up fetch_merge_commit * Curl and bash * Fixes * Try Docker image for faster Setup? * Add comment describing docker image * Report merge SHA to bundle-size endpoint * Review updates * Fix docker executor * Rename ciMergeSha
ampproject · Mar 10, 2021 · 980f0ef · 980f0ef
1 parent c121292
commit 980f0ef
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 7 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -29,14 +29,32 @@ executors:
     machine:
       image: ubuntu-2004:202010-01
     resource_class: xlarge
+  amphtml-docker-executor:
+    docker:
+      # The official, bare bones Docker Image from CircleCI
+      # https://circleci.com/docs/2.0/circleci-images/?section=executors-and-images#circleci-base-image
+      - image: cimg/base:2020.01
 
 commands:
+  save_merge_commit:
+    steps:
+      - save_cache:
+          name: 'Save Merge Commit'
+          key: git-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - .CIRCLECI_WORKFLOW_MERGE_COMMIT
+  restore_merge_commit:
+    steps:
+      - restore_cache:
+          name: 'Restore Merge Commit'
+          key: git-{{ .Branch }}-{{ .Revision }}
   setup_vm:
     steps:
       - checkout
+      - restore_merge_commit
       - run:
           name: 'Fetch Merge Commit'
-          command: ./.circleci/fetch_merge_commit.sh
+          command: ./.circleci/fetch_merge_commit.sh merge
       - run:
           name: 'Check Config'
           command: ./.circleci/check_config.sh
@@ -61,6 +79,15 @@ commands:
           command: ./.circleci/fail_fast.sh
 
 jobs:
+  'Compute Merge Commit':
+    executor:
+      name: amphtml-docker-executor
+    steps:
+      - run:
+          name: 'Fetch Merge Commit'
+          # TODO: This needs to be changed to ampproject/amphtml/master after merging #33165
+          command: curl -o- https://raw.githubusercontent.com/jridgewell/amphtml/build-size-base/.circleci/fetch_merge_commit.sh | bash /dev/stdin fetch
+      - save_merge_commit
   'Checks':
     executor:
       name: amphtml-medium-executor
@@ -231,30 +258,47 @@ jobs:
 workflows:
   'CircleCI':
     jobs:
+      - 'Compute Merge Commit':
+          <<: *push_and_pr_builds
       - 'Checks':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Unminified Build':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Nomodule Build':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Module Build':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Bundle Size':
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Nomodule Build'
             - 'Module Build'
       - 'Validator Tests':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Visual Diff Tests':
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Nomodule Build'
       - 'Unit Tests':
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Unminified Tests':
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Unminified Build'
       - 'Nomodule Tests':
           name: 'Nomodule Tests (<< matrix.config >>)'
@@ -263,6 +307,7 @@ workflows:
               config: ['prod', 'canary']
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Nomodule Build'
       - 'Module Tests':
           name: 'Module Tests (<< matrix.config >>)'
@@ -271,28 +316,34 @@ workflows:
               config: ['prod', 'canary']
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Nomodule Build'
             - 'Module Build'
       - 'End-to-End Tests':
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Nomodule Build'
       - 'Experiment Build':
           name: 'Experiment << matrix.exp >> Build'
           matrix:
             parameters:
               exp: ['A', 'B', 'C']
           <<: *push_and_pr_builds
+          requires:
+            - 'Compute Merge Commit'
       - 'Experiment Tests':
           name: 'Experiment << matrix.exp >> Tests'
           matrix:
             parameters:
               exp: ['A', 'B', 'C']
           <<: *push_and_pr_builds
           requires:
+            - 'Compute Merge Commit'
             - 'Experiment << matrix.exp >> Build'
       # TODO(wg-performance, #12128): This takes 30 mins and fails regularly.
       # - 'Performance Tests':
       #     <<: *push_builds_only
       #     requires:
+      #       - 'Compute Merge Commit'
       #       - 'Nomodule Build'
diff --git a/.circleci/fetch_merge_commit.sh b/.circleci/fetch_merge_commit.sh
@@ -25,6 +25,11 @@ GREEN() { echo -e "\n\033[0;32m$1\033[0m"; }
 YELLOW() { echo -e "\n\033[0;33m$1\033[0m"; }
 RED() { echo -e "\n\033[0;31m$1\033[0m"; }
 
+if [[ -z "$1" ]]; then
+  echo "Usage: fetch_merge_commit.sh [fetch | merge]"
+  exit 1
+fi
+
 # Push builds are only run against master and amp-release branches.
 if [[ "$CIRCLE_BRANCH" == "master" || "$CIRCLE_BRANCH" =~ ^amp-release-* ]]; then
   echo $(GREEN "Nothing to do because $CIRCLE_BRANCH is not a PR branch.")
@@ -53,12 +58,22 @@ if [[ -z "$PR_NUMBER" ]]; then
   exit 0
 fi
 
-# GitHub provides refs/pull/<PR_NUMBER>/merge, an up-to-date merge branch for
-# every PR branch that can be cleanly merged to master. For more details, see:
-# https://discuss.circleci.com/t/show-test-results-for-prospective-merge-of-a-github-pr/1662
-MERGE_BRANCH="refs/pull/$PR_NUMBER/merge"
-echo $(GREEN "Fetching merge commit from $MERGE_BRANCH...")
-(set -x && git pull --ff-only origin "$MERGE_BRANCH") || err=$?
+if [[ "$1" == "fetch" ]]; then
+  # GitHub provides refs/pull/<PR_NUMBER>/merge, an up-to-date merge branch for
+  # every PR branch that can be cleanly merged to master. For more details, see:
+  # https://discuss.circleci.com/t/show-test-results-for-prospective-merge-of-a-github-pr/1662
+  MERGE_BRANCH="refs/pull/$PR_NUMBER/merge"
+  echo $(GREEN "Fetching merge SHA from $MERGE_BRANCH...")
+
+  CIRCLE_MERGE_SHA="$(git ls-remote https://github.com/ampproject/amphtml.git "$MERGE_BRANCH" | awk '{print $1}')"
+  echo $(GREEN "Fetched merge SHA $CIRCLE_MERGE_SHA...")
+  echo "$CIRCLE_MERGE_SHA" > .CIRCLECI_WORKFLOW_MERGE_COMMIT
+  exit 0
+fi
+
+export CIRCLE_MERGE_SHA="$(cat .CIRCLECI_WORKFLOW_MERGE_COMMIT)"
+echo $(GREEN "Fetching merge commit $CIRCLE_MERGE_SHA...")
+(set -x && git pull --ff-only origin "$CIRCLE_MERGE_SHA") || err=$?
 
 # If a clean merge is not possible, do not proceed with the build. GitHub's UI
 # will show an error indicating there was a merge conflict.

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 .DS_Store
 .g4ignore
+.CIRCLECI_WORKFLOW_MERGE_COMMIT
 build/
 .amp-dep-check
 c

diff --git a/build-system/common/ci.js b/build-system/common/ci.js
@@ -194,6 +194,15 @@ function ciJobUrl() {
     : '';
 }
 
+/**
+ * Returns the merge commits SHA when running a CI Pull Request build.
+ * @return {string}
+ */
+function circleciPrMergeSha() {
+  // CIRCLE_MERGE_SHA is populated by .circleci/fetch_merge_commit.sh.
+  return isCircleci ? env('CIRCLE_MERGE_SHA') : '';
+}
+
 /**
  * Returns the repo slug for the ongoing build.
  * @return {string}
@@ -221,6 +230,7 @@ module.exports = {
   ciCommitSha,
   ciJobId,
   ciJobUrl,
+  circleciPrMergeSha,
   ciPullRequestBranch,
   ciPullRequestSha,
   ciPushBranch,

diff --git a/build-system/tasks/bundle-size/index.js b/build-system/tasks/bundle-size/index.js
@@ -30,6 +30,7 @@ const {
   isPushBuild,
   ciPushBranch,
   ciRepoSlug,
+  circleciPrMergeSha,
 } = require('../../common/ci');
 const {
   VERSION: internalRuntimeVersion,
@@ -187,6 +188,7 @@ async function reportBundleSize() {
         json: true,
         body: {
           baseSha,
+          mergeSha: circleciPrMergeSha(),
           bundleSizes: await getBrotliBundleSizes(),
         },
       });