diff --git a/.github/actions/sign_artifacts/action.yaml b/.github/actions/sign_artifacts/action.yaml deleted file mode 100644 index 65100f312c4..00000000000 --- a/.github/actions/sign_artifacts/action.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: sign-artifacts -description: Sign all artifacts in the given directory using the provided GPG key. -inputs: - artifacts-dir: - description: 'Directory with artifacts to sign' - required: true - skip-shas: - description: 'Whether to skip signing of sha256 files' - default: 'true' - BUILDBOT_GPG_KEY_ID: - description: 'GPG key ID of key to sign with' - required: true - BUILDBOT_GPG_KEY_B64: - description: 'GPG private key base64 encoded' - required: true -runs: - using: "composite" - steps: - - name: Import GPG key - shell: bash - env: - BUILDBOT_GPG_KEY_B64: ${{ inputs.BUILDBOT_GPG_KEY_B64 }} - run: | - echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - - name: Sign Artifacts - shell: bash - env: - ARTIFACTS_DIR: ${{ inputs.artifacts-dir }} - SKIP_SHAS: ${{ inputs.skip-shas }} - BUILDBOT_GPG_KEY_ID: ${{ inputs.BUILDBOT_GPG_KEY_ID }} - # yamllint disable rule:indentation - run: | - for artifact in "${ARTIFACTS_DIR}/"*; do - if [[ "${SKIP_SHAS}" == "true" ]] && [[ "${artifact}" == *".sha256" ]]; then - continue - fi - gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "${artifact}" - done - # yamllint enable rule:indentation diff --git a/.github/workflows/cli_release.yaml b/.github/workflows/cli_release.yaml index 3ca1fb92f37..30df5725352 100644 --- a/.github/workflows/cli_release.yaml +++ b/.github/workflows/cli_release.yaml @@ -36,11 +36,11 @@ jobs: # With some kernel configs (eg. COS), podman only works with legacy iptables. update-alternatives --set iptables /usr/sbin/iptables-legacy update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy - - name: Get GPG Key from secrets + - name: Import GPG key env: BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} run: | - echo "$BUILDBOT_GPG_KEY_B64" | base64 --decode > /tmp/gpg.key + echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - id: gcloud-creds uses: ./.github/actions/gcloud_creds with: @@ -49,7 +49,6 @@ jobs: env: REF: ${{ github.event.ref }} BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} - BUILDBOT_GPG_KEY_FILE: "/tmp/gpg.key" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_NUMBER: ${{ github.run_attempt }} JOB_NAME: ${{ github.job }} @@ -115,11 +114,11 @@ jobs: - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: macos-artifacts - - name: Get GPG Key from secrets + - name: Import GPG key env: BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} run: | - echo "$BUILDBOT_GPG_KEY_B64" | base64 --decode > /tmp/gpg.key + echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - id: gcloud-creds uses: ./.github/actions/gcloud_creds with: @@ -128,7 +127,6 @@ jobs: env: REF: ${{ github.event.ref }} BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} - BUILDBOT_GPG_KEY_FILE: "/tmp/gpg.key" GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} shell: bash run: | diff --git a/.github/workflows/cloud_release.yaml b/.github/workflows/cloud_release.yaml index e22b1cb5b07..fd2e95ac310 100644 --- a/.github/workflows/cloud_release.yaml +++ b/.github/workflows/cloud_release.yaml @@ -35,6 +35,11 @@ jobs: uses: ./.github/actions/gcloud_creds with: SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }} + - name: Import GPG key + env: + BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} + run: | + echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - name: Build & Push Artifacts env: REF: ${{ github.event.ref }} @@ -44,6 +49,7 @@ jobs: COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} + BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} shell: bash run: | export TAG_NAME="${REF#*/tags/}" @@ -51,12 +57,6 @@ jobs: mkdir -p "${ARTIFACTS_DIR}" ./ci/save_version_info.sh ./ci/cloud_build_release.sh - - name: Sign Artifacts - uses: ./.github/actions/sign_artifacts - with: - artifacts-dir: artifacts - BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} - BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: cloud-artifacts diff --git a/.github/workflows/operator_release.yaml b/.github/workflows/operator_release.yaml index 15808bb6b18..57e0374b1b9 100644 --- a/.github/workflows/operator_release.yaml +++ b/.github/workflows/operator_release.yaml @@ -35,6 +35,11 @@ jobs: uses: ./.github/actions/gcloud_creds with: SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }} + - name: Import GPG key + env: + BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} + run: | + echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - name: Build & Push Artifacts env: REF: ${{ github.event.ref }} @@ -44,6 +49,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} GH_REPO: ${{ github.repository }} + BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} shell: bash run: | export TAG_NAME="${REF#*/tags/}" @@ -57,12 +63,6 @@ jobs: ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public" GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} run: ./ci/update_artifact_manifest.sh - - name: Sign Artifacts - uses: ./.github/actions/sign_artifacts - with: - artifacts-dir: artifacts - BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} - BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: operator-artifacts diff --git a/.github/workflows/vizier_release.yaml b/.github/workflows/vizier_release.yaml index 852e705ab99..56a4fd44f66 100644 --- a/.github/workflows/vizier_release.yaml +++ b/.github/workflows/vizier_release.yaml @@ -35,6 +35,11 @@ jobs: uses: ./.github/actions/gcloud_creds with: SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }} + - name: Import GPG key + env: + BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} + run: | + echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import - name: Build & Push Artifacts env: REF: ${{ github.event.ref }} @@ -43,6 +48,7 @@ jobs: COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} + BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} shell: bash run: | export TAG_NAME="${REF#*/tags/}" @@ -62,12 +68,6 @@ jobs: ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public" GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }} run: ./ci/update_artifact_manifest.sh - - name: Sign Artifacts - uses: ./.github/actions/sign_artifacts - with: - artifacts-dir: artifacts - BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }} - BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }} - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: vizier-artifacts diff --git a/ci/artifact_mirrors.yaml b/ci/artifact_mirrors.yaml new file mode 100644 index 00000000000..f31b3a72e24 --- /dev/null +++ b/ci/artifact_mirrors.yaml @@ -0,0 +1,11 @@ +--- +# The order of the list is important, as its the order of the mirrors. +# So the first mirror in the list will be pulled from first. +- name: gh-releases + type: gh-releases + url_format: 'https://github.com/pixie-io/pixie/releases/download/release/${component}/v${version}/${artifact_name}' +- name: pixie-oss-gcs + type: gcs + bucket: pixie-dev-public + path_format: '${component}/${version}/${artifact_name}' + url_format: 'https://storage.googleapis.com/pixie-dev-public/${component}/${version}/${artifact_name}' diff --git a/ci/artifact_utils.sh b/ci/artifact_utils.sh new file mode 100644 index 00000000000..4d37c897463 --- /dev/null +++ b/ci/artifact_utils.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +gh_artifacts_dir="${ARTIFACTS_DIR}" +workspace=$(git rev-parse --show-toplevel) +mirrors_file="${workspace}/ci/artifact_mirrors.yaml" + +BUILDBOT_GPG_KEY_ID="${BUILDBOT_GPG_KEY_ID:?}" + +upload_artifact_to_mirrors() { + component="$1" + version="$2" + artifact_path="$3" + artifact_name="$4" + + # Create SHA and signature files for the artifact. + sha256sum "${artifact_path}" | awk '{print $1}' > "${artifact_path}.sha256" + gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "${artifact_path}" + + while read -r mirror; do + mirror_def="$(yq '.[] | select(.name == "'"${mirror}"'")' "${mirrors_file}")" + mirror_type="$(echo "${mirror_def}" | yq '.type')" + case "${mirror_type}" in + gh-releases) + if [[ "${version}" == "latest" ]]; then + # gh-releases have no consistent latest across multiple components, + # so we only upload "latest" artifacts to GCS. + continue; + fi + gh_release_mirror "${artifact_path}" "${artifact_name}" + ;; + gcs) + bucket="$(echo "${mirror_def}" | yq '.bucket' )" + path_format="$(echo "${mirror_def}" | yq '.path_format')" + path="$(env - component="${component}" \ + version="${version}" \ + artifact_name="${artifact_name}" \ + bash -c "echo ${path_format}")" + upload_to_gcs "${artifact_path}" "${bucket}" "${path}" + ;; + esac + done < <(yq '.[].name' "${mirrors_file}") +} + +gh_release_mirror() { + artifact_path="$1" + artifact_name="$2" + # This is expected to run in a github action with ARTIFACTS_DIR defined + if [[ -z "${gh_artifacts_dir}" ]]; then + echo "Must run in github actions to use gh-releases mirror" + exit 1 + fi + + cp "${artifact_path}" "${gh_artifacts_dir}/${artifact_name}" + cp "${artifact_path}.sha256" "${gh_artifacts_dir}/${artifact_name}.sha256" + cp "${artifact_path}.asc" "${gh_artifacts_dir}/${artifact_name}.asc" +} + +upload_to_gcs() { + artifact_path="$1" + gcs_bucket="$2" + gcs_path="$3" + + gsutil cp "${artifact_path}" "gs://${gcs_bucket}/${gcs_path}" + gsutil cp "${artifact_path}.sha256" "gs://${gcs_bucket}/${gcs_path}.sha256" + gsutil cp "${artifact_path}.asc" "gs://${gcs_bucket}/${gcs_path}.asc" +} diff --git a/ci/cli_build_release.sh b/ci/cli_build_release.sh index 42e08f5a808..2985635b1b9 100755 --- a/ci/cli_build_release.sh +++ b/ci/cli_build_release.sh @@ -19,13 +19,12 @@ set -ex repo_path=$(bazel info workspace) -# shellcheck source=ci/gcs_utils.sh -. "${repo_path}/ci/gcs_utils.sh" +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" printenv versions_file="$(realpath "${VERSIONS_FILE:?}")" -artifacts_dir="${ARTIFACTS_DIR:-$(mktemp -d)}" release_tag=${TAG_NAME##*/v} linux_arch=x86_64 pkg_prefix="pixie-px-${release_tag}.${linux_arch}" @@ -86,44 +85,21 @@ if [[ ! "$release_tag" == *"-"* ]]; then # TODO(james): Add push to docker hub/quay.io. fi -gpg --no-tty --batch --yes --import "${BUILDBOT_GPG_KEY_FILE}" - -write_artifacts_to_gcs() { - output_path=$1 - copy_artifact_to_gcs "${output_path}" "${darwin_amd64_binary}" "cli_darwin_amd64_unsigned" - copy_artifact_to_gcs "${output_path}" "${darwin_arm64_binary}" "cli_darwin_arm64_unsigned" - copy_artifact_to_gcs "${output_path}" "${linux_binary}" "cli_linux_amd64" +upload_artifacts() { + version="$1" + upload_artifact_to_mirrors "cli" "${version}" "${darwin_amd64_binary}" "cli_darwin_amd64_unsigned" + upload_artifact_to_mirrors "cli" "${version}" "${darwin_arm64_binary}" "cli_darwin_arm64_unsigned" + upload_artifact_to_mirrors "cli" "${version}" "${linux_binary}" "cli_linux_amd64" if [[ ! "$release_tag" == *"-"* ]]; then # RPM/DEB only exists for release builds. - copy_artifact_to_gcs "${output_path}" "$(pwd)/${pkg_prefix}.deb" "pixie-px.${linux_arch}.deb" - copy_artifact_to_gcs "${output_path}" "$(pwd)/${pkg_prefix}.rpm" "pixie-px.${linux_arch}.rpm" + upload_artifact_to_mirrors "cli" "${version}" "$(pwd)/${pkg_prefix}.deb" "pixie-px.${linux_arch}.deb" + upload_artifact_to_mirrors "cli" "${version}" "$(pwd)/${pkg_prefix}.rpm" "pixie-px.${linux_arch}.rpm" fi } -sign_artifacts() { - cp "${linux_binary}" "${artifacts_dir}/cli_linux_amd64" - cp "$(pwd)/${pkg_prefix}.deb" "${artifacts_dir}/pixie-px.${linux_arch}.deb" - cp "$(pwd)/${pkg_prefix}.rpm" "${artifacts_dir}/pixie-px.${linux_arch}.rpm" - - pushd "${artifacts_dir}" - gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "cli_linux_amd64" - gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "pixie-px.${linux_arch}.deb" - gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "pixie-px.${linux_arch}.rpm" - popd -} - -public="True" -bucket="pixie-dev-public" -if [[ $release_tag == *"-"* ]]; then - public="False" - # Use the same bucket for RCs. -fi -output_path="gs://${bucket}/cli/${release_tag}" -write_artifacts_to_gcs "${output_path}" +upload_artifacts "${release_tag}" # Check to see if it's production build. If so we should also write it to the latest directory. -if [[ $public == "True" ]]; then - output_path="gs://${bucket}/cli/latest" - write_artifacts_to_gcs "${output_path}" - sign_artifacts +if [[ ! $release_tag == *"-"* ]]; then + upload_artifacts "latest" fi diff --git a/ci/cli_upload_signed.sh b/ci/cli_upload_signed.sh index 42e1dac14ff..d56b227f08c 100755 --- a/ci/cli_upload_signed.sh +++ b/ci/cli_upload_signed.sh @@ -18,31 +18,21 @@ repo_path=$(bazel info workspace) -# shellcheck source=ci/gcs_utils.sh -. "${repo_path}/ci/gcs_utils.sh" +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" set -ex printenv -artifacts_dir="${ARTIFACTS_DIR:-?}" release_tag=${TAG_NAME##*/v} -bucket="pixie-dev-public" -gpg --no-tty --batch --yes --import "${BUILDBOT_GPG_KEY_FILE}" - -output_path="gs://${bucket}/cli/${release_tag}" for arch in amd64 arm64 universal do - copy_artifact_to_gcs "$output_path" "cli_darwin_${arch}" "cli_darwin_${arch}" + upload_artifact_to_mirrors "cli" "${release_tag}" "cli_darwin_${arch}" "cli_darwin_${arch}" # Check to see if it's production build. If so we should also write it to the latest directory. if [[ ! "$release_tag" == *"-"* ]]; then - output_path="gs://${bucket}/cli/latest" - copy_artifact_to_gcs "$output_path" "cli_darwin_${arch}" "cli_darwin_${arch}" - - gpg --no-tty --batch --yes --local-user "${BUILDBOT_GPG_KEY_ID}" --armor --detach-sign "cli_darwin_${arch}" - cp "cli_darwin_${arch}" "${artifacts_dir}" - cp "cli_darwin_${arch}.asc" "${artifacts_dir}" + upload_artifact_to_mirrors "cli" "latest" "cli_darwin_${arch}" "cli_darwin_${arch}" fi done diff --git a/ci/cloud_build_release.sh b/ci/cloud_build_release.sh index f30854393e8..84fe6cbf6c9 100755 --- a/ci/cloud_build_release.sh +++ b/ci/cloud_build_release.sh @@ -19,7 +19,10 @@ set -ex printenv -artifacts_dir="${ARTIFACTS_DIR:?}" +repo_path=$(git rev-parse --show-toplevel) + +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" repo_path=$(pwd) @@ -46,8 +49,12 @@ all_licenses_opts=("//tools/licenses:all_licenses" "--action_env=GOOGLE_APPLICAT all_licenses_path="$(bazel cquery "${all_licenses_opts[@]}" --output starlark --starlark:expr "target.files.to_list()[0].path" 2> /dev/null)" bazel build "${all_licenses_opts[@]}" +upload_artifact_to_mirrors "cloud" "${release_tag}" "${all_licenses_path}" "licenses.json" +# The licenses file uses a non-standard path (outside of the "component/version/artifact" convention) +# so for now we'll also copy it to the legacy path. gsutil cp "${all_licenses_path}" "gs://pixie-dev-public/oss-licenses/${release_tag}.json" if [[ "${release}" == "true" ]]; then + upload_artifact_to_mirrors "cloud" "latest" "${all_licenses_path}" "licenses.json" gsutil cp "${all_licenses_path}" "gs://pixie-dev-public/oss-licenses/latest.json" fi @@ -69,9 +76,11 @@ bazel run @com_github_mikefarah_yq_v4//:v4 -- '..|.image?|select(.|type == "!!st cd "${repo_path}" tar -czvf "${repo_path}/pixie_cloud.tar.gz" "pixie_cloud" -gsutil cp "${repo_path}/pixie_cloud.tar.gz" "gs://pixie-dev-public/cloud/${release_tag}/pixie_cloud.tar.gz" + +upload_artifact_to_mirrors "cloud" "${release_tag}" "${repo_path}/pixie_cloud.tar.gz" "pixie_cloud.tar.gz" + if [[ "${release}" == "true" ]]; then - gsutil cp "${repo_path}/pixie_cloud.tar.gz" "gs://pixie-dev-public/cloud/latest/pixie_cloud.tar.gz" + upload_artifact_to_mirrors "cloud" "latest" "${repo_path}/pixie_cloud.tar.gz" "pixie_cloud.tar.gz" fi sha256sum "${repo_path}/pixie_cloud.tar.gz" | awk '{print $1}' > sha diff --git a/ci/operator_build_release.sh b/ci/operator_build_release.sh index c96937e8443..d7a4cf40709 100755 --- a/ci/operator_build_release.sh +++ b/ci/operator_build_release.sh @@ -24,6 +24,8 @@ release_tag=${TAG_NAME##*/v} # shellcheck source=ci/image_utils.sh . "${repo_path}/ci/image_utils.sh" +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" echo "The release tag is: ${release_tag}" @@ -118,9 +120,7 @@ cd "${repo_path}" output_path="gs://${bucket}/operator/${release_tag}" bazel build //k8s/operator:operator_templates yamls_tar="${repo_path}/bazel-bin/k8s/operator/operator_templates.tar" -sha256sum "${yamls_tar}" | awk '{print $1}' > tmplSha -gsutil cp "${yamls_tar}" "${output_path}/operator_template_yamls.tar" -gsutil cp tmplSha "${output_path}/operator_template_yamls.tar.sha256" +upload_artifact_to_mirrors "operator" "${release_tag}" "${yamls_tar}" "operator_template_yamls.tar" ./ci/operator_helm_build_release.sh "${release_tag}" diff --git a/ci/operator_helm_build_release.sh b/ci/operator_helm_build_release.sh index fd1c68bbe0b..b1b68ceb84b 100755 --- a/ci/operator_helm_build_release.sh +++ b/ci/operator_helm_build_release.sh @@ -33,7 +33,6 @@ parse_args() { parse_args "$@" tmp_dir="$(mktemp -d)" -artifacts_dir="${ARTIFACTS_DIR:?}" index_file="${INDEX_FILE:?}" gh_repo="${GH_REPO:?}" @@ -43,6 +42,8 @@ if [[ $VERSION == *"-"* ]]; then fi repo_path=$(pwd) +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" helm_path="${repo_path}/k8s/operator/helm" # Create Chart.yaml for this release for Helm3. @@ -84,9 +85,7 @@ helm package "${helm_path}2" -d "${tmp_dir}/${helm_gcs_bucket}" # Update the index file. helm repo index "${tmp_dir}/${helm_gcs_bucket}" --url "https://${helm_gcs_bucket}.storage.googleapis.com" -cp "${tmp_dir}/${helm_gcs_bucket}/pixie-operator-chart-${VERSION}.tgz" "${artifacts_dir}/pixie-operator-chart-${VERSION}.tgz" -sha256sum "${tmp_dir}/${helm_gcs_bucket}/pixie-operator-chart-${VERSION}.tgz" | awk '{print $1}' > sha -cp sha "${artifacts_dir}/pixie-operator-chart-${VERSION}.tgz.sha256" +upload_artifact_to_mirrors "operator" "${VERSION}" "${tmp_dir}/${helm_gcs_bucket}/pixie-operator-chart-${VERSION}.tgz" "pixie-operator-chart-${VERSION}.tgz" # Upload the new index and tar to gcs by syncing. This will help keep the timestamps for pre-existing tars the same. gsutil rsync "${tmp_dir}/${helm_gcs_bucket}" "gs://${helm_gcs_bucket}" diff --git a/ci/vizier_build_release.sh b/ci/vizier_build_release.sh index f4eeeb7c634..7e4d2b05333 100755 --- a/ci/vizier_build_release.sh +++ b/ci/vizier_build_release.sh @@ -20,13 +20,14 @@ set -ex printenv -artifacts_dir="${ARTIFACTS_DIR:?}" versions_file="$(realpath "${VERSIONS_FILE:?}")" repo_path=$(pwd) release_tag=${TAG_NAME##*/v} # shellcheck source=ci/image_utils.sh . "${repo_path}/ci/image_utils.sh" +# shellcheck source=ci/artifact_utils.sh +. "${repo_path}/ci/artifact_utils.sh" echo "The release tag is: ${release_tag}" @@ -34,44 +35,30 @@ bazel run -c opt //src/utils/artifacts/versions_gen:versions_gen -- \ --repo_path "${repo_path}" --artifact_name vizier --versions_file "${versions_file}" build_type="--//k8s:build_type=public" -bucket="pixie-dev-public" if [[ $release_tag == *"-"* ]]; then build_type="--//k8s:build_type=dev" - # Use the same bucket as above for RCs fi -output_path="gs://${bucket}/vizier/${release_tag}" -latest_output_path="gs://${bucket}/vizier/latest" - push_all_multiarch_images "//k8s/vizier:vizier_images_push" "//k8s/vizier:list_image_bundle" "${release_tag}" "${build_type}" bazel build --config=stamp -c opt --//k8s:image_version="${release_tag}" \ --config=stamp "${build_type}" //k8s/vizier:vizier_yamls -output_path="gs://${bucket}/vizier/${release_tag}" yamls_tar="${repo_path}/bazel-bin/k8s/vizier/vizier_yamls.tar" -sha256sum "${yamls_tar}" | awk '{print $1}' > sha -gsutil cp "${yamls_tar}" "${output_path}/vizier_yamls.tar" -gsutil cp sha "${output_path}/vizier_yamls.tar.sha256" - -cp "${yamls_tar}" "${artifacts_dir}/vizier_yamls.tar" -cp sha "${artifacts_dir}/vizier_yamls.tar.sha256" +upload_artifact_to_mirrors "vizier" "${release_tag}" "${yamls_tar}" "vizier_yamls.tar" # Upload templated YAMLs. tmp_dir="$(mktemp -d)" bazel run -c opt //src/utils/template_generator:template_generator -- \ --base "${yamls_tar}" --version "${release_tag}" --out "${tmp_dir}" tmpl_path="${tmp_dir}/yamls.tar" -sha256sum "${tmpl_path}" | awk '{print $1}' > tmplSha -gsutil cp "${tmpl_path}" "${output_path}/vizier_template_yamls.tar" -gsutil cp tmplSha "${output_path}/vizier_template_yamls.tar.sha256" +upload_artifact_to_mirrors "vizier" "${release_tag}" "${tmpl_path}" "vizier_template_yamls.tar" # Update helm chart if it is a release. if [[ $public == "True" ]]; then # Update Vizier YAMLS in latest. - gsutil cp "${yamls_tar}" "${latest_output_path}/vizier_yamls.tar" - gsutil cp sha "${latest_output_path}/vizier_yamls.tar.sha256" + upload_artifact_to_mirrors "vizier" "latest" "${yamls_tar}" "vizier_yamls.tar" ./ci/helm_build_release.sh "${release_tag}" "${tmpl_path}" fi