From ecfc442f64af9489e6341f6be46c576f3fab6edf Mon Sep 17 00:00:00 2001 From: James Wainwright Date: Mon, 15 Jan 2024 14:01:03 +0000 Subject: [PATCH] [ci] Add Bazel cache GCP key to more jobs Signed-off-by: James Wainwright --- azure-pipelines.yml | 73 ++++++++++++++++++++++++++------------------- ci/bazelisk.sh | 5 ++-- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c48b9b0de6d159..91000a28de1ff6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -46,6 +46,7 @@ jobs: - publish: $(Pipeline.Workspace)/opentitan-repo.tar.gz artifact: opentitan-repo displayName: Upload repository + - job: lint displayName: Quality (quick lint) # Run code quality checks (quick lint) @@ -179,23 +180,17 @@ jobs: dependsOn: lint condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0')) pool: ci-public - variables: - - name: bazelCacheGcpKeyPath - value: '' steps: - template: ci/checkout-template.yml - template: ci/install-package-dependencies.yml - task: DownloadSecureFile@1 condition: eq(variables['Build.SourceBranchName'], 'master') - name: bazelCacheGcpKey + name: GCP_BAZEL_CACHE_KEY inputs: secureFile: "bazel_cache_gcp_key.json" - # Set the remote cache GCP key path - - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)" - condition: eq(variables['Build.SourceBranchName'], 'master') - displayName: GCP key path - bash: | set -x -e + # Check the entire build graph for conflicts in loading or analysis # phases. For context, see issue #18726. # First, test with an empty bitstream cache entry. @@ -214,7 +209,6 @@ jobs: # shallow exclusion; tests deeper under //hw will still be found. # * It excludes targets that depend on bitstream_splice rules, since the # environment does not have access to Vivado. - export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath) TARGET_PATTERN_FILE=target_pattern.txt echo //... > "${TARGET_PATTERN_FILE}" echo -//quality/... >> "${TARGET_PATTERN_FILE}" @@ -260,21 +254,14 @@ jobs: timeoutInMinutes: 120 dependsOn: sw_build pool: ci-public - variables: - - name: bazelCacheGcpKeyPath - value: '' steps: - template: ci/checkout-template.yml - template: ci/install-package-dependencies.yml - task: DownloadSecureFile@1 condition: eq(variables['Build.SourceBranchName'], 'master') - name: bazelCacheGcpKey + name: GCP_BAZEL_CACHE_KEY inputs: secureFile: "bazel_cache_gcp_key.json" - # Set the remote cache GCP key path - - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)" - condition: eq(variables['Build.SourceBranchName'], 'master') - displayName: GCP key path - download: current artifact: target_pattern_file - bash: | @@ -317,24 +304,16 @@ jobs: pool: ci-public timeoutInMinutes: 240 dependsOn: lint - variables: - - name: bazelCacheGcpKeyPath - value: '' steps: - template: ci/checkout-template.yml - template: ci/install-package-dependencies.yml - task: DownloadSecureFile@1 condition: eq(variables['Build.SourceBranchName'], 'master') - name: bazelCacheGcpKey + name: GCP_BAZEL_CACHE_KEY inputs: secureFile: "bazel_cache_gcp_key.json" - - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)" - condition: eq(variables['Build.SourceBranchName'], 'master') - displayName: GCP key path - # Set the remote cache GCP key path - bash: | set -x -e - export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath) ci/scripts/run-verilator-tests.sh displayName: Build & execute tests - template: ci/publish-bazel-test-results.yml @@ -374,6 +353,12 @@ jobs: parameters: downloadPartialBuildBinFrom: - chip_englishbreakfast_verilator + # FIXME: do we still want this + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | . util/build_consts.sh ci/scripts/run-english-breakfast-verilator-tests.sh @@ -437,13 +422,9 @@ jobs: - template: ci/install-package-dependencies.yml - task: DownloadSecureFile@1 condition: eq(variables['Build.SourceBranchName'], 'master') - name: bazelCacheGcpKey + name: GCP_BAZEL_CACHE_KEY inputs: secureFile: "bazel_cache_gcp_key.json" - - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)" - condition: eq(variables['Build.SourceBranchName'], 'master') - displayName: GCP key path - # Set the remote cache GCP key path - bash: | ci/bazelisk.sh test --test_tag_filters=-nightly //sw/otbn/crypto/... displayName: Execute tests @@ -560,6 +541,11 @@ jobs: downloadPartialBuildBinFrom: - chip_earlgrey_cw310 - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | set -e . util/build_consts.sh @@ -586,6 +572,11 @@ jobs: downloadPartialBuildBinFrom: - chip_earlgrey_cw310 - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | set -e . util/build_consts.sh @@ -613,6 +604,11 @@ jobs: - chip_earlgrey_cw310 - chip_earlgrey_cw310_hyperdebug - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | set -e . util/build_consts.sh @@ -639,6 +635,11 @@ jobs: downloadPartialBuildBinFrom: - chip_earlgrey_cw310_hyperdebug - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" # We run the update command twice to workaround an issue with udev on the container. # Where rusb cannot dynamically update its device list in CI (udev is not completely # functional). If the device is in normal mode, the first thing that opentitantool @@ -678,6 +679,11 @@ jobs: downloadPartialBuildBinFrom: - chip_earlgrey_cw340 - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | set -e . util/build_consts.sh @@ -704,6 +710,11 @@ jobs: downloadPartialBuildBinFrom: - chip_earlgrey_cw310 - sw_build + - task: DownloadSecureFile@1 + condition: eq(variables['Build.SourceBranchName'], 'master') + name: GCP_BAZEL_CACHE_KEY + inputs: + secureFile: "bazel_cache_gcp_key.json" - bash: | set -e . util/build_consts.sh diff --git a/ci/bazelisk.sh b/ci/bazelisk.sh index 89722b1e916394..e765a457dbc57e 100755 --- a/ci/bazelisk.sh +++ b/ci/bazelisk.sh @@ -15,12 +15,13 @@ echo "Running bazelisk in $(pwd)." # An additional bazelrc must be synthesized to specify precisely how to use the # GCP bazel cache. +GCP_CREDS_FILE="$GCP_BAZEL_CACHE_KEY_SECUREFILEPATH" GCP_BAZELRC="$(mktemp /tmp/XXXXXX.bazelrc)" trap 'rm ${GCP_BAZELRC}' EXIT -if [[ -n "${GCP_BAZEL_CACHE_KEY}" && -f "${GCP_BAZEL_CACHE_KEY}" ]]; then +if [[ -n "$GCP_CREDS_FILE" && -f "$GCP_CREDS_FILE" ]]; then echo "Applying GCP cache key; will upload to the cache." - echo "build --google_credentials=${GCP_BAZEL_CACHE_KEY}" >> "${GCP_BAZELRC}" + echo "build --google_credentials=${GCP_CREDS_FILE}" >> "${GCP_BAZELRC}" else echo "No key/invalid path to key. Download from cache only." echo "build --remote_upload_local_results=false" >> "${GCP_BAZELRC}"