Postmigration script cleanup

.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh

@@ -2,15 +2,18 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/util.sh
-
 export KUBECONFIG="${WORKSPACE}/kubecfg"
 export BIN="${WORKSPACE}/bin"
-add_bin_path
 
-echo "--- Installing kind & kubectl"
+echo "--- Add ${BIN} to PATH"
+if [[ ! -d "${BIN}" ]]; then
+  mkdir -p "${BIN}"
+fi
+export PATH="${PATH}:${BIN}"
+
+echo "~~~ Installing kind & kubectl"
 asdf plugin add kind
-asdf install kind $ASDF_KIND_VERSION
+asdf install kind "$ASDF_KIND_VERSION"
 
 echo "~~~ Setting up kind"
 max_retries=3

.buildkite/hooks/pre-command

@@ -2,56 +2,9 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/util.sh
-
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
-DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
-GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then
-  source .buildkite/env-scripts/env.sh
-  if [[ -z "${GO_VERSION-""}" ]]; then
-    export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
-  fi
-
-  # Images with prefix "platform-ingest-beats-*" has Go pre-setup.
-  # Image itself takes care of Go version download/installation based on .go-version file
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    if [[ -z "${GO_VERSION-""}" ]]; then
-      GO_VERSION=$(cat "${WORKSPACE}/.go-version")
-      export GO_VERSION
-    fi
-
-    # To prevent "OSError: [Errno 24] Too many open files"
-    ulimit -Sn 150000
-    echo "--- Ulimit: $(ulimit)"
-
-    echo "--- Setting up environment"
-    add_bin_path
-    with_go
-    with_mage
-  fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then
-    DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
-
-    export DOCKER_USERNAME_SECRET
-    export DOCKER_PASSWORD_SECRET
-    export GITHUB_TOKEN_SECRET
-
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-
-    github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
-
-    git config user.name "$github_username"
-    git config user.email "$github_email"
-  fi
-fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(
   "auditbeat"
@@ -74,10 +27,8 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do
   fi
 done
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
-    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then
+    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-  fi
 fi
 

.buildkite/hooks/pre-exit

@@ -2,14 +2,24 @@
 
 set -euo pipefail
 
-source .buildkite/hooks/scripts/util.sh
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
-    docker logout "${DOCKER_REGISTRY}"
+cleanup() {
+  if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
+  echo "Deleting temporary files..."
+    if [[ -n "${BIN:-}" ]]; then
+      rm -rf "${BIN}"
+    fi
+  echo "Done."
   fi
+}
+
+unset_secrets
+cleanup
 
-  # Ensure that any temporal files created during any step are removed
-  cleanup
-  unset_secrets
-fi

.buildkite/scripts/initCloudEnv.sh

@@ -8,8 +8,8 @@ exportAwsSecrets() {
   local awsSecretKey
   local awsAccessKey
 
-  awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-  awsAccessKey=$(retry -t 5 -- vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
+  awsAccessKey=$(retry -t 5 -- vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
 
   echo "~~~ Exporting AWS secrets"
   export AWS_ACCESS_KEY_ID=$awsAccessKey