Merge branch 'main' into nicolas.guerguadj/add-fips-mode

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v44534774-f5cc3e24
+      - image: gcr.io/datadoghq/agent-circleci-runner:v44808106-d8c4f8af
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.github/CODEOWNERS

@@ -98,7 +98,6 @@
 /.gitlab/common/test_infra_version.yml               @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx-loops
-/.gitlab/e2e_k8s/e2e_k8s.yml                         @DataDog/container-integrations @DataDog/agent-devx-loops
 /.gitlab/e2e/install_packages                        @DataDog/agent-delivery
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
@@ -565,10 +564,6 @@
 /test/                                  @DataDog/agent-devx-loops
 /test/benchmarks/                       @DataDog/agent-metrics-logs
 /test/benchmarks/kubernetes_state/      @DataDog/container-integrations
-/test/e2e/                              @DataDog/container-integrations @DataDog/agent-security
-/test/e2e/cws-tests/                    @DataDog/agent-security
-/test/e2e/argo-workflows/otlp-workflow.yaml @DataDog/opentelemetry
-/test/e2e/containers/otlp_sender/        @DataDog/opentelemetry
 /test/integration/                      @DataDog/container-integrations
 /test/integration/serverless            @DataDog/serverless @Datadog/serverless-aws
 /test/integration/serverless_perf       @DataDog/serverless @Datadog/serverless-aws

.gitlab-ci.yml

@@ -1,7 +1,6 @@
 ---
 include:
   - .gitlab/.pre/cancel-prev-pipelines.yml
-  - .gitlab/.pre/linters.yml
   - .gitlab/.pre/test_gitlab_configuration.yml
   - .gitlab/benchmarks/include.yml
   - .gitlab/binary_build/include.yml
@@ -23,7 +22,6 @@ include:
   - .gitlab/deps_fetch/deps_fetch.yml
   - .gitlab/dev_container_deploy/include.yml
   - .gitlab/e2e/e2e.yml
-  - .gitlab/e2e_k8s/e2e_k8s.yml
   - .gitlab/e2e_install_packages/include.yml
   - .gitlab/e2e_pre_test/e2e_pre_test.yml
   - .gitlab/fips_compliance/fips_compliance_e2e.yml
@@ -37,6 +35,7 @@ include:
   - .gitlab/kitchen_cleanup/include.yml
   - .gitlab/kitchen_deploy/kitchen_deploy.yml
   - .gitlab/kitchen_testing/include.yml
+  - .gitlab/lint/include.yml
   - .gitlab/maintenance_jobs/include.yml
   - .gitlab/notify/notify.yml
   - .gitlab/package_build/include.yml
@@ -67,6 +66,7 @@ stages:
   - maintenance_jobs
   - deps_build
   - deps_fetch
+  - lint
   - source_test
   - source_test_stats
   - software_composition_analysis
@@ -796,6 +796,7 @@ workflow:
       paths:
         - test/new-e2e/pkg/**/*
         - test/new-e2e/go.mod
+        - flakes.yaml
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_e2e_or_windows_installer_changes:
@@ -1034,6 +1035,15 @@ workflow:
   - when: manual
     allow_failure: true
 
+.on_cspm_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/security/**/*
+        - test/new-e2e/tests/cspm/**/* #TODO: Add other paths that should trigger the execution of CSPM e2e tests
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
 .on_windows_systemprobe_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:

.gitlab/e2e/e2e.yml

@@ -463,6 +463,19 @@ new-e2e-package-signing-suse-a7-x86_64:
     - .new-e2e_package_signing
   rules: !reference [.on_default_new_e2e_tests]
 
+new-e2e-cspm:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_cspm_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - qa_agent
+    - qa_dca
+  variables:
+    TARGETS: ./tests/cspm
+    TEAM: cspm
+
 generate-flakes-finder-pipeline:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: e2e

.gitlab/functional_test/regression_detector.yml

@@ -14,7 +14,9 @@ single-machine-performance-regression_detector:
       - submission_metadata # for provenance, debugging
       - ${CI_COMMIT_SHA}-baseline_sha # for provenance, debugging
       - outputs/report.md # for debugging, also on S3
-      - outputs/report.html # for debugging, also on S3
+      - outputs/regression_signal.json # for debugging, also on S3
+      - outputs/bounds_check_signal.json # for debugging, also on S3
+      - outputs/junit.xml # for debugging, also on S3
     when: always
   variables:
     SMP_VERSION: 0.16.0
@@ -33,7 +35,6 @@ single-machine-performance-regression_detector:
     # Ensure output files exist for artifact downloads step
     - mkdir outputs # Also needed for smp job sync step
     - touch outputs/report.md # Will be emitted by smp job sync
-    - touch outputs/report.html # Will be emitted by smp job sync
     # Compute merge base of current commit and `main`
     - git fetch origin
     - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch)
@@ -95,6 +96,12 @@ single-machine-performance-regression_detector:
     - !reference [.install_pr_commenter]
     # Post HTML report to GitHub
     - cat outputs/report.md | /usr/local/bin/pr-commenter --for-pr="$CI_COMMIT_REF_NAME" --header="Regression Detector"
+    # Upload JUnit XML outside of Agent CI's tooling because the `junit_upload`
+    # invoke task has additional logic that does not seem to apply well to SMP's
+    # JUnit XML. Agent CI seems to use `datadog-agent` as the service name when
+    # uploading JUnit XML, so the upload command below respects that convention.
+    - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" || exit $?; export DATADOG_API_KEY
+    - datadog-ci junit upload --service datadog-agent outputs/junit.xml
     # Finally, exit 1 if the job signals a regression else 0.
     - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE}
       job result

.gitlab/kernel_matrix_testing/common.yml

@@ -142,6 +142,7 @@
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
     VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
+    EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
   before_script:
     # XXX: system-probe images does not run `entrypoint.sh` which runs `source /root/.bashrc`.
     # Since the path of Go binary is added into PATH in the bashrc, we need to run it at some point.
@@ -151,6 +152,7 @@
     - !reference [.retrieve_linux_go_deps]
     - !reference [.kmt_new_profile]
     - !reference [.write_ssh_key_file]
+    - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH || true
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
@@ -193,6 +195,9 @@
       - $CI_PROJECT_DIR/stack.output
       - $CI_PROJECT_DIR/libvirt
       - $VMCONFIG_FILE
+    reports:
+      annotations:
+        - $EXTERNAL_LINKS_PATH
 
 .kmt_cleanup:
   stage: kernel_matrix_testing_cleanup

.gitlab/lint/include.yml

@@ -0,0 +1,6 @@
+# liont stage
+# Include job that run linters on the Agent code.
+
+include:
+  - .gitlab/lint/technical_linters.yml
+

.gitlab/.pre/linters.yml → .gitlab/lint/technical_linters.yml

@@ -1,14 +1,17 @@
 
 .lint:
-  stage: .pre
+  stage: lint
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
 
 lint_licenses:
   extends: .lint
   script:
+    - !reference [.retrieve_linux_go_deps]
+    - !reference [.retrieve_linux_go_tools_deps]
     - inv -e install-tools
     - inv -e lint-licenses
+  needs: ["go_tools_deps", "go_deps"]
 
 lint_shell:
   extends: .lint
@@ -39,3 +42,23 @@ lint_components:
   extends: .lint
   script:
     - inv -e lint-components lint-fxutil-oneshot-test
+
+
+lint_python:
+  extends: .lint
+  needs: []
+  script:
+    - inv -e linter.python
+
+lint_update_go:
+  extends: .lint
+  needs: []
+  script:
+    - inv -e linter.update-go
+
+validate_modules:
+  extends: .lint
+  needs: []
+  script:
+    - inv -e modules.validate
+    - inv -e modules.validate-used-by-otel

.gitlab/package_build/linux.yml

@@ -1,25 +1,28 @@
+.agent_build_script:
+  - echo "About to build for $RELEASE_VERSION"
+  - !reference [.retrieve_linux_go_deps]
+  - !reference [.cache_omnibus_ruby_deps, setup]
+  # remove artifacts from previous pipelines that may come from the cache
+  - rm -rf $OMNIBUS_PACKAGE_DIR/*
+  # Artifacts and cache must live within project directory but we run omnibus in a neutral directory.
+  # Thus, we move the artifacts at the end in a gitlab-friendly dir.
+  - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
+  - mkdir -p /tmp/system-probe
+  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf
+  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf
+  - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz
+  - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
+  - inv -e omnibus.build --fips-mode --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" --config-directory "$CONFIG_DIR" --install-directory "$INSTALL_DIR"
+  - ls -la $OMNIBUS_PACKAGE_DIR
+  - !reference [.upload_sbom_artifacts]
+
 .agent_build_common:
   rules:
     - !reference [.except_mergequeue]
     - when: on_success
   stage: package_build
   script:
-    - echo "About to build for $RELEASE_VERSION"
-    - !reference [.retrieve_linux_go_deps]
-    - !reference [.cache_omnibus_ruby_deps, setup]
-    # remove artifacts from previous pipelines that may come from the cache
-    - rm -rf $OMNIBUS_PACKAGE_DIR/*
-    # Artifacts and cache must live within project directory but we run omnibus in a neutral directory.
-    # Thus, we move the artifacts at the end in a gitlab-friendly dir.
-    - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
-    - mkdir -p /tmp/system-probe
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf
-    - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz
-    - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
-    - inv -e omnibus.build --fips-mode --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR"
-    - ls -la $OMNIBUS_PACKAGE_DIR
-    - !reference [.upload_sbom_artifacts]
+    - !reference [.agent_build_script]
   variables:
     KUBERNETES_CPU_REQUEST: 16
     KUBERNETES_MEMORY_REQUEST: "32Gi"
@@ -81,7 +84,37 @@
   before_script:
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
-# build Agent 6 binaries for x86_64
+# Temporary custom agent build test to prevent regression
+# This test will be removed when custom path are used to build macos agent
+# with in-house macos runner builds.
+datadog-agent-7-x64-custom-path-test:
+  extends: [.agent_build_x86, .agent_7_build]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  stage: package_build
+  script:
+    - mkdir /custom
+    - export CONFIG_DIR="/custom"
+    - export INSTALL_DIR="/custom/datadog-agent"
+    - !reference [.agent_build_script]
+    - ls -la $OMNIBUS_PACKAGE_DIR
+    - ls -la $INSTALL_DIR
+    - ls -la /custom/etc
+    - (ls -la /opt/datadog-agent 2>/dev/null && exit 1) || echo "/opt/datadog-agent has correctly not been generated"
+    - (ls -la /etc/datadog-agent 2>/dev/null && exit 1) || echo "/etc/datadog-agent has correctly not been generated"
+  variables:
+    KUBERNETES_CPU_REQUEST: 16
+    KUBERNETES_MEMORY_REQUEST: "32Gi"
+    KUBERNETES_MEMORY_LIMIT: "32Gi"
+  artifacts:
+    expire_in: 2 weeks
+    paths:
+      - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
+
+  # build Agent 6 binaries for x86_64
 datadog-agent-6-x64:
   extends: [.agent_build_common, .agent_build_x86, .agent_6_build]
 

.gitlab/source_test/include.yml

@@ -12,5 +12,4 @@ include:
   - .gitlab/source_test/slack.yml
   - .gitlab/source_test/golang_deps_diff.yml
   - .gitlab/source_test/notify.yml
-  - .gitlab/source_test/technical_linters.yml
   - .gitlab/source_test/tooling_unit_tests.yml