Merge branch 'main' into onboarding-card-design

Author: rshen91

.buildkite/scripts/common/activate_service_account.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
+BUCKET_OR_EMAIL="${1:-}"
+GCLOUD_EMAIL_POSTFIX="elastic-kibana-ci.iam.gserviceaccount.com"
+GCLOUD_SA_PROXY_EMAIL="kibana-ci-sa-proxy@$GCLOUD_EMAIL_POSTFIX"
+
+if [[ -z "$BUCKET_OR_EMAIL" ]]; then
+  echo "Usage: $0 <bucket_name|email>"
+  exit 1
+elif [[ "$BUCKET_OR_EMAIL" == "--unset-impersonation" ]]; then
+  echo "Unsetting impersonation"
+  gcloud config unset auth/impersonate_service_account
+  exit 0
+elif [[ "$BUCKET_OR_EMAIL" == "--logout-gcloud" ]]; then
+  echo "Logging out of gcloud"
+  if [[ -x "$(command -v gcloud)" ]] && [[ "$(gcloud auth list 2>/dev/null | grep $GCLOUD_SA_PROXY_EMAIL)" != "" ]]; then
+    gcloud auth revoke $GCLOUD_SA_PROXY_EMAIL --no-user-output-enabled
+  fi
+  exit 0
+fi
+
+CURRENT_GCLOUD_USER=$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")
+
+# Verify that the service account proxy is activated
+if [[ "$CURRENT_GCLOUD_USER" != "$GCLOUD_SA_PROXY_EMAIL" ]]; then
+    if [[ -x "$(command -v gcloud)" ]]; then
+      if [[ -z "${KIBANA_SERVICE_ACCOUNT_PROXY_KEY:-}" ]]; then
+        echo "KIBANA_SERVICE_ACCOUNT_PROXY_KEY is not set, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      fi
+
+      AUTH_RESULT=$(gcloud auth activate-service-account --key-file="$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" || "FAILURE")
+      if [[ "$AUTH_RESULT" == "FAILURE" ]]; then
+        echo "Failed to activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      else
+        echo "Activated service account $GCLOUD_SA_PROXY_EMAIL"
+      fi
+    else
+      echo "gcloud is not installed, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+      exit 1
+    fi
+fi
+
+# Check if the arg is a service account e-mail or a bucket name
+EMAIL=""
+if [[ "$BUCKET_OR_EMAIL" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then
+  EMAIL="$BUCKET_OR_EMAIL"
+elif [[ "$BUCKET_OR_EMAIL" =~ ^gs://* ]]; then
+  BUCKET_NAME="${BUCKET_OR_EMAIL:5}"
+else
+  BUCKET_NAME="$BUCKET_OR_EMAIL"
+fi
+
+if [[ -z "$EMAIL" ]]; then
+  case "$BUCKET_NAME" in
+    "elastic-kibana-coverage-live")
+      EMAIL="kibana-ci-access-coverage@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-ci-es-snapshots-daily")
+      EMAIL="kibana-ci-access-es-snapshots@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-so-types-snapshots")
+      EMAIL="kibana-ci-access-so-snapshots@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-performance")
+      EMAIL="kibana-ci-access-perf-stats@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "ci-artifacts.kibana.dev")
+      EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    *)
+      EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+  esac
+fi
+
+# Activate the service account
+echo "Impersonating $EMAIL"
+gcloud config set auth/impersonate_service_account "$EMAIL"
+echo "Activated service account $EMAIL"

.buildkite/scripts/common/setup_bazel.sh

@@ -2,6 +2,8 @@
 
 source .buildkite/scripts/common/util.sh
 
+echo '--- Setting up bazel'
+
 echo "[bazel] writing .bazelrc"
 cat <<EOF > $KIBANA_DIR/.bazelrc
   # Generated by .buildkite/scripts/common/setup_bazel.sh
@@ -27,16 +29,16 @@ if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
 
   echo "[bazel] using GCS bucket: $BAZEL_BUCKET"
 
-cat <<EOF >> $KIBANA_DIR/.bazelrc
+  cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET
-  build --google_default_credentials
+  build --google_credentials=$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
 EOF
 fi
 
 if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
   echo "[bazel] enabling caching with GCS buckets for local dev"
 
-cat <<EOF >> $KIBANA_DIR/.bazelrc
+  cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
   build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
 EOF

.buildkite/scripts/common/util.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
 is_pr() {
   [[ "${GITHUB_PR_NUMBER-}" ]] && return
   false
@@ -170,48 +172,3 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
-
-# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
-if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
-  VAULT_KV_PREFIX="secret/kibana-issues/dev"
-  IS_LEGACY_VAULT_ADDR=true
-else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
-  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
-  IS_LEGACY_VAULT_ADDR=false
-fi
-export IS_LEGACY_VAULT_ADDR
-
-vault_get() {
-  key_path=$1
-  field=$2
-
-  fullPath="$VAULT_PATH_PREFIX/$key_path"
-
-  if [[ -z "${2:-}" || "${2:-}" =~ ^-.* ]]; then
-    retry 5 5 vault read "$fullPath" "${@:2}"
-  else
-    retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
-  fi
-}
-
-vault_set() {
-  key_path=$1
-  shift
-  fields=("$@")
-
-
-  fullPath="$VAULT_PATH_PREFIX/$key_path"
-
-  # shellcheck disable=SC2068
-  retry 5 5 vault write "$fullPath" ${fields[@]}
-}
-
-vault_kv_set() {
-  kv_path=$1
-  shift
-  fields=("$@")
-
-  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
-}

.buildkite/scripts/common/vault_fns.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  VAULT_KV_PREFIX="secret/kibana-issues/dev"
+  IS_LEGACY_VAULT_ADDR=true
+else
+  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
+  IS_LEGACY_VAULT_ADDR=false
+fi
+export IS_LEGACY_VAULT_ADDR
+
+retry() {
+  local retries=$1; shift
+  local delay=$1; shift
+  local attempts=1
+
+  until "$@"; do
+    retry_exit_status=$?
+    echo "Exited with $retry_exit_status" >&2
+    if (( retries == "0" )); then
+      return $retry_exit_status
+    elif (( attempts == retries )); then
+      echo "Failed $attempts retries" >&2
+      return $retry_exit_status
+    else
+      echo "Retrying $((retries - attempts)) more times..." >&2
+      attempts=$((attempts + 1))
+      sleep "$delay"
+    fi
+  done
+}
+
+vault_get() {
+  key_path=${1:-}
+  field=${2:-}
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
+
+  if [[ -z "$field" || "$field" =~ ^-.* ]]; then
+    retry 5 5 vault read "$fullPath" "${@:2}"
+  else
+    retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
+  fi
+}
+
+vault_set() {
+  key_path=$1
+  shift
+  fields=("$@")
+
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
+
+  # shellcheck disable=SC2068
+  retry 5 5 vault write "$fullPath" ${fields[@]}
+}
+
+vault_kv_set() {
+  kv_path=$1
+  shift
+  fields=("$@")
+
+  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
+}

.buildkite/scripts/lifecycle/post_command.sh

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+echo '--- Log out of gcloud'
+./.buildkite/scripts/common/activate_service_account.sh --unset-impersonation || echo "Failed to unset impersonation"
+./.buildkite/scripts/common/activate_service_account.sh --logout-gcloud || echo "Failed to log out of gcloud"
+
 echo '--- Agent Debug Info'
 ts-node .buildkite/scripts/lifecycle/print_agent_links.ts || true
 

.buildkite/scripts/lifecycle/pre_command.sh

@@ -167,6 +167,16 @@ BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-loca
 export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
 vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
 
+# Export key for accessing bazel remote cache's GCS bucket
+BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json"
+export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
+vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE"
+
+# Setup GCS Service Account Proxy for CI
+KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json"
+export KIBANA_SERVICE_ACCOUNT_PROXY_KEY
+vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY"
+
 PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
 if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
   source "$PIPELINE_PRE_COMMAND"

.buildkite/scripts/steps/archive_so_migration_snapshot.sh

@@ -3,15 +3,16 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-SO_MIGRATIONS_SNAPSHOT_FOLDER=kibana-so-types-snapshots
+SO_MIGRATIONS_SNAPSHOT_BUCKET="gs://kibana-so-types-snapshots"
 SNAPSHOT_FILE_PATH="${1:-target/plugin_so_types_snapshot.json}"
 
 echo "--- Creating snapshot of Saved Object migration info"
 node scripts/snapshot_plugin_types snapshot --outputPath "$SNAPSHOT_FILE_PATH"
 
 echo "--- Uploading as ${BUILDKITE_COMMIT}.json"
-SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_FOLDER}/${BUILDKITE_COMMIT}.json"
-gsutil cp "$SNAPSHOT_FILE_PATH" "gs://$SNAPSHOT_PATH"
+SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_BUCKET}/${BUILDKITE_COMMIT}.json"
+.buildkite/scripts/common/activate_service_account.sh "$SO_MIGRATIONS_SNAPSHOT_BUCKET"
+gsutil cp "$SNAPSHOT_FILE_PATH" "$SNAPSHOT_PATH"
 
 buildkite-agent annotate --context so_migration_snapshot --style success \
   'Saved Object type snapshot is available at <a href="https://storage.cloud.google.com/'"$SNAPSHOT_PATH"'">'"$SNAPSHOT_PATH"'</a>'

.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh

@@ -6,6 +6,7 @@ set -euo pipefail
 gsutil -m cp -r gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/previous.txt . || echo "### Previous Pointer NOT FOUND?"
 
 # TODO: Activate after the above is removed
+#.buildkite/scripts/common/activate_service_account.sh gs://elastic-kibana-coverage-live
 #gsutil -m cp -r gs://elastic-kibana-coverage-live/previous_pointer/previous.txt . || echo "### Previous Pointer NOT FOUND?"
 
 if [ -e ./previous.txt ]; then

.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh

@@ -12,4 +12,5 @@ collectPrevious
 # TODO: Safe to remove this after 2024-03-01 (https://github.com/elastic/kibana/issues/175904)
 gsutil cp previous.txt gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/
 
+.buildkite/scripts/common/activate_service_account.sh gs://elastic-kibana-coverage-live
 gsutil cp previous.txt gs://elastic-kibana-coverage-live/previous_pointer/

.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh

@@ -27,5 +27,6 @@ uploadRest() {
 
 echo "--- Uploading static site"
 
+.buildkite/scripts/common/activate_service_account.sh gs://elastic-kibana-coverage-live
 uploadBase
 uploadRest