feat(gitlab): Job retry on agent.version and pipeline.changelog

.gitlab/binary_build/cluster_agent_cloudfoundry.yml

@@ -21,5 +21,5 @@ cluster_agent_cloudfoundry-build_amd64:
     - inv -e cluster-agent-cloudfoundry.build
     - cd $CI_PROJECT_DIR/$CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR
     - mkdir -p $OMNIBUS_PACKAGE_DIR
-    - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)
+    - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $?; export PACKAGE_VERSION
     - tar cf $OMNIBUS_PACKAGE_DIR/datadog-cluster-agent-cloudfoundry-$PACKAGE_VERSION-$ARCH.tar.xz datadog-cluster-agent-cloudfoundry

.gitlab/deploy_containers/deploy_containers_a7.yml

@@ -25,7 +25,7 @@ include:
   stage: deploy_containers
   dependencies: []
   before_script:
-    - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi
+    - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; export VERSION; fi
     - export IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
     - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64"
     - export IMG_WINDOWS_SOURCES="${IMG_BASE_SRC}-7${JMX}-win1809${FLAVOR}-amd64,${IMG_BASE_SRC}-7${JMX}-winltsc2022${FLAVOR}-amd64"
@@ -75,7 +75,7 @@ deploy_containers-dogstatsd:
     !reference [.manual_on_deploy_auto_on_rc]
   dependencies: []
   before_script:
-    - export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"
+    - VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; export VERSION
     - export IMG_SOURCES="${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64"
     - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}"
 

.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml

@@ -11,7 +11,7 @@ include:
   stage: deploy_cws_instrumentation
   dependencies: []
   before_script:
-    - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi
+    - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe)" || exit $?; export VERSION; fi
     - if [[ "$CWS_INSTRUMENTATION_REPOSITORY" == "" ]]; then export CWS_INSTRUMENTATION_REPOSITORY="cws-instrumentation"; fi
     - export IMG_BASE_SRC="${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
     - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64"

.gitlab/deploy_dca/deploy_dca.yml

@@ -15,7 +15,7 @@ include:
     - job: "docker_build_cluster_agent_arm64"
       artifacts: false
   before_script:
-    - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi
+    - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe)" || exit $?; export VERSION; fi
     - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi
     - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
     - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64"

.gitlab/deploy_packages/nix.yml

@@ -172,7 +172,7 @@ deploy_staging_dsd:
   needs: ["build_dogstatsd-binary_x64"]
   script:
     - $S3_CP_CMD $S3_ARTIFACTS_URI/dogstatsd/dogstatsd ./dogstatsd
-    - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)
+    - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $?; export PACKAGE_VERSION
     - $S3_CP_CMD ./dogstatsd $S3_DSD6_URI/linux/dogstatsd-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
 
 # deploy iot-agent x64 binary to staging bucket
@@ -185,7 +185,7 @@ deploy_staging_iot_agent:
   needs: ["build_iot_agent-binary_x64"]
   script:
     - $S3_CP_CMD $S3_ARTIFACTS_URI/iot/agent ./agent
-    - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)
+    - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $?; export PACKAGE_VERSION
     - $S3_CP_CMD ./agent $S3_DSD6_URI/linux/iot/agent-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
 
 # Datadog Installer

.gitlab/deploy_packages/oci.yml

@@ -15,7 +15,7 @@ include:
     - ls $OMNIBUS_PACKAGE_DIR
   script:
     - set +x
-    - export VERSION=$(inv agent.version --url-safe)-1
+    - VERSION=$(inv agent.version --url-safe)-1 || exit $?; export VERSION
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - go env -w GOPRIVATE="github.com/DataDog/*"
     - export PATH=$PATH:$(go env GOPATH)/bin

.gitlab/e2e/e2e.yml

@@ -343,7 +343,7 @@ new-e2e-installer-windows:
     # Must run before new_e2e_template changes the aws profile
     # Note: this is similar to the WINDOWS_AGENT_VERSION in new-e2e_windows_msi but this job is running cross platforms
     # Note 2: new_e2e_template does not define AGENT_MAJOR_VERSION, so define it as 7 below.
-    - export CURRENT_AGENT_VERSION=$(invoke agent.version --major-version 7)
+    - CURRENT_AGENT_VERSION=$(invoke agent.version --major-version 7) || exit $?; export CURRENT_AGENT_VERSION
     - export STABLE_AGENT_VERSION_PACKAGE=$(curl -sS https://hub.docker.com/v2/namespaces/datadog/repositories/agent-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1)
     - export STABLE_INSTALLER_VERSION_PACKAGE=$(curl -sS https://hub.docker.com/v2/namespaces/datadog/repositories/installer-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1)
     - !reference [.new_e2e_template, before_script]

.gitlab/e2e_install_packages/windows.yml

@@ -10,7 +10,7 @@
   before_script:
     # WINDOWS_AGENT_VERSION is used to verify the installed agent version
     # Must run before new_e2e_template changes the aws profile
-    - export WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION)
+    - WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) || exit $?; export WINDOWS_AGENT_VERSION
     - !reference [.new_e2e_template, before_script]
   script:
     # LAST_STABLE_VERSION is used for upgrade test
@@ -24,7 +24,7 @@
   before_script:
     # WINDOWS_AGENT_VERSION is used to verify the installed agent version
     # Must run before new_e2e_template changes the aws profile
-    - export WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION)
+    - WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) || exit $?; export WINDOWS_AGENT_VERSION
     - !reference [.new_e2e_template, before_script]
   script:
     # LAST_STABLE_VERSION is used for upgrade test

.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml

@@ -68,4 +68,4 @@ notify-slack:
   script:
     - export SDM_JWT=$(vault read -field=token identity/oidc/token/sdm)
     - python3 -m pip install -r tasks/requirements.txt
-    - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA}
+    - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} || exit $?

.gitlab/package_build/installer.yml

@@ -5,7 +5,8 @@
 .common_build_oci:
   script:
     - echo "About to build for $RELEASE_VERSION"
-    - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1
+    - AGENT_VERSION=$(inv agent.version -u)-1 || exit $?
+    - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/$AGENT_VERSION
     - !reference [.retrieve_linux_go_deps]
     - !reference [.cache_omnibus_ruby_deps, setup]
     # remove artifacts from previous pipelines that may come from the cache
@@ -143,15 +144,17 @@ installer-amd64-oci:
   variables:
     DESTINATION_FILE: "datadog-updater_7-amd64-oci.tar.xz"
   before_script:
-    - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - AGENT_VERSION=$(inv agent.version -u)-1 || exit $?
+    - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/$AGENT_VERSION
     - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"
 
 installer-arm64-oci:
   extends: installer-arm64
   variables:
     DESTINATION_FILE: "datadog-updater_7-arm64-oci.tar.xz"
   before_script:
-    - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - AGENT_VERSION=$(inv agent.version -u)-1 || exit $?
+    - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/$AGENT_VERSION
     - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"
 
 windows-installer-amd64:

.gitlab/packaging/oci.yml

@@ -6,7 +6,7 @@
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   before_script:
-    - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)-1
+    - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)-1 || exit $?; export PACKAGE_VERSION
     - export INSTALL_DIR=/opt/datadog-packages/${OCI_PRODUCT}/${PACKAGE_VERSION}
   variables:
     KUBERNETES_CPU_REQUEST: 16

.gitlab/setup/setup.yml

@@ -4,7 +4,7 @@ setup_agent_version:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   script:
-    - inv -e agent.version --cache-version
+    - inv -e agent.version --cache-version || exit $?
     - $S3_CP_CMD $CI_PROJECT_DIR/agent-version.cache $S3_ARTIFACTS_URI/agent-version.cache
   needs: []
 

.gitlab/trigger_release/trigger_release.yml

@@ -18,7 +18,7 @@
   script:
     # agent-release-management creates pipeline for both Agent 6 and Agent 7
     # when triggered with major version 7
-    - export RELEASE_VERSION=$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1
+    - RELEASE_VERSION=$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1 || exit $?; export RELEASE_VERSION
     - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN
     - 'inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-release-management" --git-ref "main"
       --variable ACTION

tasks/libs/releasing/version.py

@@ -265,10 +265,12 @@ def get_version(
         agent_version_cache_file_exist = os.path.exists(AGENT_VERSION_CACHE_NAME)
         if not agent_version_cache_file_exist:
             if pipeline_id and pipeline_id.isdigit() and project_name == REPO_NAME:
-                ctx.run(
+                result = ctx.run(
                     f"aws s3 cp s3://dd-ci-artefacts-build-stable/datadog-agent/{pipeline_id}/{AGENT_VERSION_CACHE_NAME} .",
                     hide="stdout",
                 )
+                if "unable to locate credentials" in result.stderr.casefold():
+                    raise Exit("Permanent error: unable to locate credentials, retry the job", 42)
                 agent_version_cache_file_exist = True
 
         if agent_version_cache_file_exist:
@@ -327,10 +329,12 @@ def get_version_numeric_only(ctx, major_version='7'):
     if pipeline_id and pipeline_id.isdigit() and project_name == REPO_NAME:
         try:
             if not os.path.exists(AGENT_VERSION_CACHE_NAME):
-                ctx.run(
+                result = ctx.run(
                     f"aws s3 cp s3://dd-ci-artefacts-build-stable/datadog-agent/{pipeline_id}/{AGENT_VERSION_CACHE_NAME} .",
                     hide="stdout",
                 )
+                if "unable to locate credentials" in result.stderr.casefold():
+                    raise Exit("Permanent error: unable to locate credentials, retry the job", 42)
 
             with open(AGENT_VERSION_CACHE_NAME) as file:
                 cache_data = json.load(file)

tasks/pipeline.py

@@ -593,11 +593,13 @@ def changelog(ctx, new_commit_sha):
     print(f"Posting message to slack: \n {slack_message}")
     send_slack_message("system-probe-ops", slack_message)
     print(f"Writing new commit sha: {new_commit_sha} to SSM")
-    ctx.run(
+    res = ctx.run(
         f"aws ssm put-parameter --name ci.datadog-agent.gitlab_changelog_commit_sha --value {new_commit_sha} "
         "--type \"SecureString\" --region us-east-1 --overwrite",
         hide=True,
     )
+    if "unable to locate credentials" in res.stderr.casefold():
+        raise Exit("Permanent error: unable to locate credentials, retry the job", code=42)
 
 
 @task