Release Cluster Agent as part of Agent release (#12943)

DataDog · Aug 8, 2022 · 5050a5e · 5050a5e
1 parent e665d8d
commit 5050a5e
Show file tree

Hide file tree

Showing 10 changed files with 101 additions and 96 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -56,6 +56,8 @@
 /.gitlab/deploy_7/windows_cloudfoundry.yml           @DataDog/integrations-tools-and-libraries @DataDog/agent-platform
 /.gitlab/deploy_7/winget.yml                         @DataDog/agent-platform @DataDog/windows-agent
 
+/.gitlab/deploy_dca.yml                             @DataDog/container-integrations @DataDog/agent-platform
+
 # @DataDog/windows-agent owns the vcpkg job
 /.gitlab/deps_build.yml                              @DataDog/agent-network @DataDog/agent-platform @DataDog/windows-agent
 /.gitlab/windows_python_build.yml                    @DataDog/windows-agent

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -18,6 +18,7 @@ include:
   - /.gitlab/dev_container_deploy.yml
   - /.gitlab/deploy_6.yml
   - /.gitlab/deploy_7.yml
+  - /.gitlab/deploy_dca.yml
   - /.gitlab/choco_build.yml
   - /.gitlab/choco_deploy.yml
   - /.gitlab/deploy_invalidate.yml
@@ -46,7 +47,7 @@ default:
 
 workflow:
   rules:
-    - if: $CI_COMMIT_TAG =~ /^dca-/ || $CI_COMMIT_TAG == null || $DEPLOY_AGENT == "true"
+    - if: $CI_COMMIT_TAG == null || $DEPLOY_AGENT == "true"
 
 stages:
   - maintenance_jobs
@@ -67,6 +68,7 @@ stages:
   - dev_container_deploy
   - deploy6
   - deploy7
+  - deploy_dca
   - choco_build
   - choco_deploy
   - deploy_invalidate

diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS
@@ -40,11 +40,12 @@ check_already_deployed_version_*     @DataDog/do-not-notify
 # dev_nightly*                         @DataDog/container-integrations
 # docker_trigger_internal*             @DataDog/container-integrations
 
-dca_dev_branch*                      @DataDog/do-not-notify
-dev_branch*                          @DataDog/do-not-notify
-dev_master*                          @DataDog/do-not-notify
-dev_nightly*                         @DataDog/do-not-notify
-docker_trigger_internal*             @DataDog/do-not-notify
+dca_dev_branch*                        @DataDog/do-not-notify
+dev_branch*                            @DataDog/do-not-notify
+dev_master*                            @DataDog/do-not-notify
+dev_nightly*                           @DataDog/do-not-notify
+docker_trigger_internal*               @DataDog/do-not-notify
+docker_trigger_cluster_agent_internal* @DataDog/do-not-notify
 
 # Deploy
 deploy_containers*                   @DataDog/container-integrations

diff --git a/.gitlab/binary_build/cluster_agent.yml b/.gitlab/binary_build/cluster_agent.yml
@@ -3,9 +3,7 @@
   stage: binary_build
   needs: ["go_mod_tidy_check"]
   script:
-    # Assuming that by default, the RELEASE_VERSION_7 entry will get used (ie. nightly-a7)
-    - if [[ "$CI_COMMIT_TAG" =~ ^dca- ]]; then export RELEASE_VERSION="$CI_COMMIT_TAG"; else export RELEASE_VERSION="$RELEASE_VERSION_7"; fi
-    - inv -e cluster-agent.build --release-version "$RELEASE_VERSION"
+    - inv -e cluster-agent.build --release-version "$RELEASE_VERSION_7"
     - $S3_CP_CMD $SRC_PATH/$CLUSTER_AGENT_BINARIES_DIR/datadog-cluster-agent $S3_ARTIFACTS_URI/datadog-cluster-agent.$ARCH
     - $S3_CP_CMD $SRC_PATH/Dockerfiles/cluster-agent/datadog-cluster.yaml $S3_ARTIFACTS_URI/datadog-cluster.yaml
     - $S3_CP_CMD $SRC_PATH/Dockerfiles/cluster-agent/security-agent-policies $S3_ARTIFACTS_URI/security-agent-policies --recursive

diff --git a/.gitlab/deploy_dca.yml b/.gitlab/deploy_dca.yml
@@ -0,0 +1,42 @@
+---
+# FIXME: our current Gitlab version doesn't support importing a file more than once
+# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
+# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
+# include:
+#   - /.gitlab/docker_common/publish_job_templates.yml
+
+#
+# DCA image tagging & manifest publication
+#
+
+.deploy_containers-dca-base:
+  extends: .docker_publish_job_definition
+  stage: deploy_dca
+  dependencies: []
+  before_script:
+    - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv -e agent.version --major-version 7 --url-safe)"; fi
+    - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
+    - export IMG_SOURCES=""${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64"
+    - export IMG_DESTINATIONS="cluster-agent:${VERSION}"
+
+
+deploy_containers-dca:
+  extends: .deploy_containers-dca-base
+  rules:
+    !reference [.on_deploy_a7_manual_auto_on_rc]
+
+
+deploy_containers-dca-rc:
+  extends: .deploy_containers-dca-base
+  rules:
+    !reference [.on_deploy_a7_rc]
+  variables:
+    VERSION: rc
+
+
+deploy_containers-dca-latest:
+  extends: .deploy_containers-dca-base
+  rules:
+    !reference [.on_deploy_a7_manual_final]
+  variables:
+    VERSION: latest
diff --git a/.gitlab/internal_image_deploy.yml b/.gitlab/internal_image_deploy.yml
@@ -26,3 +26,29 @@ docker_trigger_internal:
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_AGENT_SRC_REPO="${TMPL_AGENT_SRC_REPO}-release"; fi
     - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_AGENT_SRC_IMAGE,TMPL_AGENT_SRC_REPO,RELEASE_STAGING,RELEASE_PROD"
+
+
+docker_trigger_cluster_agent_internal:
+  stage: internal_image_deploy
+  rules:
+    !reference [.on_deploy_a7]
+  needs:
+    - job: docker_build_cluster_agent_amd64
+      artifacts: false
+    - job: docker_build_cluster_agent_arm64
+      artifacts: false
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["runner:main"]
+  variables:
+    IMAGE_VERSION: tmpl-v1
+    IMAGE_NAME: datadog-cluster-agent
+    RELEASE_TAG: ${CI_COMMIT_REF_SLUG}
+    BUILD_TAG: ${CI_COMMIT_REF_SLUG}
+    TMPL_AGENT_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+    TMPL_AGENT_SRC_REPO: ci/datadog-agent/cluster-agent
+    RELEASE_STAGING: "true"
+    RELEASE_PROD: "true"
+  script:
+    - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
+    - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_AGENT_SRC_REPO="${TMPL_AGENT_SRC_REPO}-release"; fi
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_AGENT_SRC_IMAGE,TMPL_AGENT_SRC_REPO,RELEASE_STAGING,RELEASE_PROD"
diff --git a/.gitlab/maintenance_jobs/docker.yml b/.gitlab/maintenance_jobs/docker.yml
@@ -46,6 +46,8 @@ revert_latest_7:
         IMG_DESTINATIONS: agent:7-servercore-jmx,agent:latest-servercore-jmx
       - IMG_SOURCES: datadog/dogstatsd:${NEW_LATEST_RELEASE_7}
         IMG_DESTINATIONS: dogstatsd:7,dogstatsd:latest
+      - IMG_SOURCES: datadog/cluster-agent:${NEW_LATEST_RELEASE_7}
+        IMG_DESTINATIONS: cluster-agent:latest
 
 #
 # Use this step to delete a tag of a given image

diff --git a/releasenotes-dca/config.yaml b/releasenotes-dca/config.yaml
@@ -2,7 +2,7 @@
 default_branch: main
 collapse_pre_releases: true
 no_show_source: true
-release_tag_re: '^dca-([\d.-]|rc|devel)+$'
+release_tag_re: '^7\.([\d.-]|rc|devel)+$'
 pre_release_tag_re: '(?P<pre_release>-rc\.\d+)$'
 template: |
           # Each section from every release note are combined when the

diff --git a/releasenotes-dca/notes/align-cluster-agent-release-4982c92b9a9c596f.yaml b/releasenotes-dca/notes/align-cluster-agent-release-4982c92b9a9c596f.yaml
@@ -0,0 +1,4 @@
+---
+other:
+  - |
+    Align Cluster Agent version to Agent version. Cluster Agent will now be released with 7.x.y tags
diff --git a/tasks/release.py b/tasks/release.py
@@ -57,11 +57,11 @@ def add_prelude(ctx, version):
 
 
 @task
-def add_dca_prelude(ctx, version, agent7_version, agent6_version=""):
+def add_dca_prelude(ctx, agent7_version, agent6_version=""):
     """
     Release of the Cluster Agent should be pinned to a version of the Agent.
     """
-    res = ctx.run(f"reno --rel-notes-dir releasenotes-dca new prelude-release-{version}")
+    res = ctx.run(f"reno --rel-notes-dir releasenotes-dca new prelude-release-{agent7_version}")
     new_releasenote = res.stdout.split(' ')[-1].strip()  # get the new releasenote file path
 
     if agent6_version != "":
@@ -79,7 +79,7 @@ def add_dca_prelude(ctx, version, agent7_version, agent6_version=""):
 
     ctx.run(f"git add {new_releasenote}")
     print("\nCommit this with:")
-    print(f"git commit -m \"Add prelude for {version} release\"")
+    print(f"git commit -m \"Add prelude for {agent7_version} release\"")
 
 
 @task
@@ -100,92 +100,20 @@ def add_installscript_prelude(ctx, version):
 
 
 @task
-def update_dca_changelog(ctx, new_version, agent_version):
+def update_dca_changelog(ctx, new_version):
     """
     Quick task to generate the new CHANGELOG-DCA using reno when releasing a minor
     version (linux/macOS only).
     """
-    new_version_int = list(map(int, new_version.split(".")))
-
-    if len(new_version_int) != 3:
-        print(f"Error: invalid version: {new_version_int}")
-        raise Exit(1)
-
-    agent_version_int = list(map(int, agent_version.split(".")))
-
-    if len(agent_version_int) != 3:
-        print(f"Error: invalid version: {agent_version_int}")
-        raise Exit(1)
-
-    # let's avoid losing uncommitted change with 'git reset --hard'
-    try:
-        ctx.run("git diff --exit-code HEAD", hide="both")
-    except Failure:
-        print("Error: You have uncommitted changes, please commit or stash before using update-dca-changelog")
-        return
-
-    # make sure we are up to date
-    ctx.run("git fetch")
-
-    # let's check that the tag for the new version is present (needed by reno)
-    try:
-        ctx.run(f"git tag --list | grep dca-{new_version}")
-    except Failure:
-        print(f"Missing 'dca-{new_version}' git tag: mandatory to use 'reno'")
-        raise
-
-    # Cluster agent minor releases are in sync with the agent's, bugfixes are not necessarily.
-    # We rely on the agent's devel tag to enforce the sync between both releases.
-    branching_point_agent = f"{agent_version_int[0]}.{agent_version_int[1]}.0-devel"
-    previous_minor_branchoff = f"dca-{new_version_int[0]}.{new_version_int[1] - 1}.X"
-    log_result = ctx.run(
-        f"git log {branching_point_agent}...remotes/origin/{previous_minor_branchoff} --name-only --oneline | grep releasenotes-dca/notes/ || true"
-    )
-    log_result = log_result.stdout.replace('\n', ' ').strip()
-
-    # Do not include release notes that were added in the previous minor release branch (previous_minor_branchoff)
-    # and the branch-off points for the current release (pined by the agent's devel tag)
-    if len(log_result) > 0:
-        ctx.run(f"git rm --ignore-unmatch {log_result}")
-
-    current_branchoff = f"dca-{new_version_int[0]}.{new_version_int[1]}.X"
-    # generate the new changelog. Specifying branch in case this is run outside the release branch that contains the tag.
-    ctx.run(
-        f"reno --rel-notes-dir releasenotes-dca report --ignore-cache --branch {current_branchoff} --version dca-{new_version} --no-show-source > /tmp/new_changelog-dca.rst"
-    )
-
-    # reseting git
-    ctx.run("git reset --hard HEAD")
-
-    # mac's `sed` has a different syntax for the "-i" paramter
-    # GNU sed has a `--version` parameter while BSD sed does not, using that to do proper detection.
-    if ctx.run("sed --version", hide='both'):
-        sed_i_arg = "-i"
-    else:
-        sed_i_arg = "-i ''"
-    # remove the old header from the existing changelog
-    ctx.run(f"sed {sed_i_arg} -e '1,4d' CHANGELOG-DCA.rst")
-
-    if sys.platform != 'darwin':
-        # sed on darwin doesn't support `-z`. On mac, you will need to manually update the following.
-        ctx.run(
-            "sed -z {0} -e 's/dca-{1}\\n===={2}/{1}\\n{2}/' /tmp/new_changelog-dca.rst".format(  # noqa: FS002
-                sed_i_arg, new_version, '=' * len(new_version)
-            )
-        )
-
-    # merging to CHANGELOG.rst
-    ctx.run("cat CHANGELOG-DCA.rst >> /tmp/new_changelog-dca.rst && mv /tmp/new_changelog-dca.rst CHANGELOG-DCA.rst")
-
-    # commit new CHANGELOG
-    ctx.run("git add CHANGELOG-DCA.rst")
-
-    print("\nCommit this with:")
-    print(f"git commit -m \"[DCA] Update CHANGELOG for {new_version}\"")
+    return update_changelog_generic(ctx, new_version, "releasenotes-dca", "CHANGELOG-DCA.rst")
 
 
 @task
 def update_changelog(ctx, new_version):
+    return update_changelog_generic(ctx, new_version, "releasenotes", "CHANGELOG.rst")
+
+
+def update_changelog_generic(ctx, new_version, changelog_dir, changelog_file):
     """
     Quick task to generate the new CHANGELOG using reno when releasing a minor
     version (linux/macOS only).
@@ -219,7 +147,7 @@ def update_changelog(ctx, new_version):
     if previous_minor == "7.15":
         previous_minor = "6.15"  # 7.15 is the first release in the 7.x series
     log_result = ctx.run(
-        f"git log {branching_point}...remotes/origin/{previous_minor}.x --name-only --oneline | grep releasenotes/notes/ || true"
+        f"git log {branching_point}...remotes/origin/{previous_minor}.x --name-only --oneline | grep {changelog_dir}/notes/ || true"
     )
     log_result = log_result.stdout.replace('\n', ' ').strip()
     if len(log_result) > 0:
@@ -246,13 +174,13 @@ def update_changelog(ctx, new_version):
         if v6_tag:
             ctx.run(f"sed {sed_i_arg} -E 's#^{new_version}#{new_version} / {v6_tag}#' /tmp/new_changelog.rst")
     # remove the old header from the existing changelog
-    ctx.run(f"sed {sed_i_arg} -e '1,4d' CHANGELOG.rst")
+    ctx.run(f"sed {sed_i_arg} -e '1,4d' {changelog_file}")
 
-    # merging to CHANGELOG.rst
-    ctx.run("cat CHANGELOG.rst >> /tmp/new_changelog.rst && mv /tmp/new_changelog.rst CHANGELOG.rst")
+    # merging to <changelog_file>
+    ctx.run(f"cat {changelog_file} >> /tmp/new_changelog.rst && mv /tmp/new_changelog.rst {changelog_file}")
 
     # commit new CHANGELOG
-    ctx.run("git add CHANGELOG.rst")
+    ctx.run(f"git add {changelog_file}.rst")
 
     print("\nCommit this with:")
     print(f"git commit -m \"[DCA] Update CHANGELOG for {new_version}\"")