[ci] Add Bazel cache GCP key to more jobs

Signed-off-by: James Wainwright <james.wainwright@lowrisc.org>
lowRISC · Jan 15, 2024 · ecfc442 · ecfc442
1 parent 2aa0bb3
commit ecfc442
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 33 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -46,6 +46,7 @@ jobs:
   - publish: $(Pipeline.Workspace)/opentitan-repo.tar.gz
     artifact: opentitan-repo
     displayName: Upload repository
+
 - job: lint
   displayName: Quality (quick lint)
   # Run code quality checks (quick lint)
@@ -179,23 +180,17 @@ jobs:
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
   - task: DownloadSecureFile@1
     condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
+    name: GCP_BAZEL_CACHE_KEY
     inputs:
       secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
   - bash: |
       set -x -e
+
       # Check the entire build graph for conflicts in loading or analysis
       # phases. For context, see issue #18726.
       # First, test with an empty bitstream cache entry.
@@ -214,7 +209,6 @@ jobs:
       #   shallow exclusion; tests deeper under //hw will still be found.
       # * It excludes targets that depend on bitstream_splice rules, since the
       #   environment does not have access to Vivado.
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=target_pattern.txt
       echo //... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
@@ -260,21 +254,14 @@ jobs:
   timeoutInMinutes: 120
   dependsOn: sw_build
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
   - task: DownloadSecureFile@1
     condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
+    name: GCP_BAZEL_CACHE_KEY
     inputs:
       secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
   - download: current
     artifact: target_pattern_file
   - bash: |
@@ -317,24 +304,16 @@ jobs:
   pool: ci-public
   timeoutInMinutes: 240
   dependsOn: lint
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
   - task: DownloadSecureFile@1
     condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
+    name: GCP_BAZEL_CACHE_KEY
     inputs:
       secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
   - bash: |
       set -x -e
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       ci/scripts/run-verilator-tests.sh
     displayName: Build & execute tests
   - template: ci/publish-bazel-test-results.yml
@@ -374,6 +353,12 @@ jobs:
     parameters:
       downloadPartialBuildBinFrom:
         - chip_englishbreakfast_verilator
+    # FIXME: do we still want this
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       . util/build_consts.sh
       ci/scripts/run-english-breakfast-verilator-tests.sh
@@ -437,13 +422,9 @@ jobs:
   - template: ci/install-package-dependencies.yml
   - task: DownloadSecureFile@1
     condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
+    name: GCP_BAZEL_CACHE_KEY
     inputs:
       secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
   - bash: |
       ci/bazelisk.sh test --test_tag_filters=-nightly //sw/otbn/crypto/...
     displayName: Execute tests
@@ -560,6 +541,11 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       set -e
       . util/build_consts.sh
@@ -586,6 +572,11 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       set -e
       . util/build_consts.sh
@@ -613,6 +604,11 @@ jobs:
         - chip_earlgrey_cw310
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       set -e
       . util/build_consts.sh
@@ -639,6 +635,11 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   # We run the update command twice to workaround an issue with udev on the container.
   # Where rusb cannot dynamically update its device list in CI (udev is not completely
   # functional). If the device is in normal mode, the first thing that opentitantool
@@ -678,6 +679,11 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw340
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       set -e
       . util/build_consts.sh
@@ -704,6 +710,11 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"
   - bash: |
       set -e
       . util/build_consts.sh

diff --git a/ci/bazelisk.sh b/ci/bazelisk.sh
@@ -15,12 +15,13 @@ echo "Running bazelisk in $(pwd)."
 
 # An additional bazelrc must be synthesized to specify precisely how to use the
 # GCP bazel cache.
+GCP_CREDS_FILE="$GCP_BAZEL_CACHE_KEY_SECUREFILEPATH"
 GCP_BAZELRC="$(mktemp /tmp/XXXXXX.bazelrc)"
 trap 'rm ${GCP_BAZELRC}' EXIT
 
-if [[ -n "${GCP_BAZEL_CACHE_KEY}" && -f "${GCP_BAZEL_CACHE_KEY}" ]]; then
+if [[ -n "$GCP_CREDS_FILE" && -f "$GCP_CREDS_FILE" ]]; then
     echo "Applying GCP cache key; will upload to the cache."
-    echo "build --google_credentials=${GCP_BAZEL_CACHE_KEY}" >> "${GCP_BAZELRC}"
+    echo "build --google_credentials=${GCP_CREDS_FILE}" >> "${GCP_BAZELRC}"
 else
     echo "No key/invalid path to key. Download from cache only."
     echo "build --remote_upload_local_results=false" >> "${GCP_BAZELRC}"