Merge branch 'master' into vandonr/vertx

Author: amarziali

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 0509dbd094c9cbf15f58db96f62276a0adff7efa
+default_system_tests_commit: &default_system_tests_commit 590162e1f5223cbbcd2876fdb820837cb13a85e1
 
 parameters:
   nightly:
@@ -696,6 +696,8 @@ jobs:
     <<: *tests
 
     resource_class: medium
+    environment:
+      - CI_AGENT_HOST=localhost
 
     docker:
       - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8
@@ -889,6 +891,7 @@ jobs:
             for log_dir in logs*; do
               tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
             done
+          when: always
 
       - store_artifacts:
           path: artifacts
@@ -978,6 +981,7 @@ jobs:
             for dir in system-tests/logs*/; do
                 cp -r "$dir" logs_debugger
             done
+          when: always
 
       - store_test_results:
           path: logs_debugger

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout "cloudfoundry" branch
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           ref: cloudfoundry
       - name: Get release version
@@ -43,7 +43,7 @@ jobs:
         run: |
           echo "${{ steps.get-release-version.outputs.VERSION }}: ${{ steps.get-release-url.outputs.URL }}" >> index.yml
       - name: Commit and push changes
-        uses: planetscale/ghcommit-action@b68767a2e130a71926b365322e62b583404a5e09 # v0.1.43
+        uses: planetscale/ghcommit-action@9400254a26464337cbe5af17c5f25075134e0089 # v0.2.7
         with:
           commit_message: "chore: Add version ${{ steps.get-release-version.outputs.VERSION }} to Cloud Foundry"
           repo: ${{ github.repository }}

.github/workflows/analyze-changes.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
       # Run the static analysis on the staging environment to benefit from the new features not yet released
       - name: Check code meets quality standards (staging)
         id: datadog-static-analysis-staging
-        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
+        uses: DataDog/datadog-static-analyzer-github-action@1297a546e6bb268e2ac5bc98a1477d22be335822 # v1
         with:
           dd_app_key: ${{ secrets.DATADOG_APP_KEY_STAGING }}
           dd_api_key: ${{ secrets.DATADOG_API_KEY_STAGING }}
@@ -44,12 +44,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.gradle/caches
@@ -59,7 +59,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/init@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -76,7 +76,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/analyze@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
 
       # For now, CodeQL SARIF results are not supported by Datadog CI
       # - name: Upload results to Datadog CI Static Analysis
@@ -106,12 +106,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.gradle/caches
@@ -144,7 +144,7 @@ jobs:
           ls -laR "./workspace/.trivy"
 
       - name: Run Trivy security scanner
-        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
         with:
           scan-type: rootfs
           scan-ref: './workspace/.trivy/'
@@ -157,7 +157,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/prune-github-container-registry.yaml

@@ -20,8 +20,6 @@ jobs:
           container: dd-trace-java/dd-lib-java-init
           keep-younger-than: 7 # days
           keep-last: 10
-          keep-tags: |
-            latest_snapshot
           prune-tags-regexes: |
-            ^[a-z0-9]{40}(-arm64|-amd64)?$
+            .+
           prune-untagged: true

.github/workflows/prune-old-pull-requests.yaml

@@ -13,7 +13,7 @@ jobs:
       pull-requests: write
     steps:
     - name: Prune old pull requests
-      uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+      uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
       with:
         days-before-stale: -1 # Disable general stale bot
         days-before-pr-stale: 90 # Only enable stale bot for PRs with no activity for 90 days

.github/workflows/update-docker-build-image.yaml

@@ -19,7 +19,7 @@ jobs:
       pull-requests: write # Required to create a pull request
     steps:
       - name: Checkout the repository
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Download ghcommit CLI
         run: |
           curl https://github.com/planetscale/ghcommit/releases/download/v0.1.48/ghcommit_linux_amd64 -o /usr/local/bin/ghcommit -L

.github/workflows/update-gradle-dependencies.yaml

@@ -13,7 +13,7 @@ jobs:
       pull-requests: write # Required to create a pull request
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
       - name: Download ghcommit CLI

.gitlab-ci.yml

@@ -25,6 +25,9 @@ variables:
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
   JAVA_BUILD_IMAGE_VERSION: "v25.01"
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
+  PROFILE_TESTS:
+    description: "Enable profiling of tests"
+    value: "false"
 
 default:
   tags: [ "arch:amd64" ]
@@ -35,6 +38,17 @@ default:
   script:
     - echo "done"
 
+.set_datadog_api_keys: &set_datadog_api_keys
+  - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
+  - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
+
+# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs
+.normalize_node_index: &normalize_node_index
+  - echo "CI_NODE_TOTAL=$CI_NODE_TOTAL , CI_NODE_INDEX=$CI_NODE_INDEX"
+  - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1}
+  - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1))
+  - echo "NORMALIZED_NODE_TOTAL=$NORMALIZED_NODE_TOTAL , NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX"
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
   stage: build
@@ -61,6 +75,7 @@ default:
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
     - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
+    - *normalize_node_index
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
@@ -117,10 +132,6 @@ test_published_artifacts:
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests
   stage: tests
   needs: [ build ]
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - when: on_success
   variables:
     BUILD_CACHE_TYPE: lib
   script:
@@ -140,26 +151,64 @@ test_published_artifacts:
     paths:
       - ./check_reports
 
+.check_job:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: lib
+  script:
+    - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh --destination ./check_reports --move
+  artifacts:
+    when: always
+    paths:
+      - ./check_reports
+      - '.gradle/daemon/*/*.out.log'
+
+check_base:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":baseCheck"
+
+check_inst:
+  extends: .check_job
+  parallel: 4
+  variables:
+    GRADLE_TARGET: ":instrumentationCheck"
+
+check_smoke:
+  extends: .check_job
+  parallel: 4
+  variables:
+    GRADLE_TARGET: ":smokeCheck"
+
+check_profiling:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":profilingCheck"
+
+check_debugger:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":debuggerCheck"
+
 muzzle:
   extends: .gradle_build
   needs: [ build ]
   stage: tests
   parallel: 8
   rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - when: on_success
+    - when: never
   variables:
     BUILD_CACHE_TYPE: test
   script:
     - export SKIP_BUILDSCAN="true"
-    - echo CI_NODE_INDEX=$CI_NODE_INDEX
-    - echo CI_NODE_TOTAL=CI_NODE_TOTAL
     - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
     - sort workspace/build/muzzleTasks > sortedMuzzleTasks
-    - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
-    - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1))
-    - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS
+    - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
+    - ./gradlew `cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs` $GRADLE_ARGS
   after_script:
     - .circleci/collect_reports.sh
   artifacts:
@@ -185,12 +234,90 @@ muzzle-dep-report:
       - ./reports
       - '.gradle/daemon/*/*.out.log'
 
+# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
+# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
+.prepare_test_env: &prepare_test_env
+  - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
+  - '[ ! -e pretest.env ] || rm pretest.env'
+  - |
+    for VARIABLE in "${gitlabVariables[@]}"
+    do
+      echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
+      unset "$VARIABLE"
+    done
+
+.restore_pretest_env: &restore_pretest_env
+  - source pretest.env
+
+.test_job:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: lib
+    GRADLE_PARAMS: ""
+    CONTINUE_ON_FAILURE: "false"
+  script:
+    - > 
+      if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
+      then
+      export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
+      fi
+    - *prepare_test_env
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
+  after_script:
+    - *restore_pretest_env
+    - *set_datadog_api_keys
+    - .circleci/collect_reports.sh
+    - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
+    - .circleci/collect_results.sh
+    - .circleci/upload_ciapp.sh tests $testJvm
+# TODO Get APM Test Agent Trace Check Results
+  artifacts:
+    when: always
+    paths:
+      - ./reports.tar
+      - ./profiles.tar
+      - ./results
+      - '.gradle/daemon/*/*.out.log'
+
+agent_integration_tests:
+  extends: .test_job
+  variables:
+    testJvm: "8"
+    CI_AGENT_HOST: local-agent
+    GRADLE_TARGET: "traceAgentTest"
+  services:
+    - name: datadog/agent:7.34.0
+      alias: local-agent
+      variables:
+        DD_APM_ENABLED: "true"
+        DD_BIND_HOST: "0.0.0.0"
+        DD_API_KEY: "invalid_key_but_this_is_fine"
+
 required:
   extends: .fan_in
   needs:
-    - spotless
-    - muzzle
-    - test_published_artifacts
+    - job: spotless
+      optional: true
+    - job: muzzle
+      optional: true
+    - job: test_published_artifacts
+      optional: true
+    - job: agent_integration_tests
+      optional: true
+    - job: check_base
+      optional: true
+    - job: check_inst
+      optional: true
+    - job: check_smoke
+      optional: true
+    - job: check_profiling
+      optional: true
+    - job: check_debugger
+      optional: true
 
 deploy_to_profiling_backend:
   stage: publish

dd-java-agent/agent-bootstrap/gradle.lockfile

@@ -39,7 +39,7 @@ commons-fileupload:commons-fileupload:1.5=jmhRuntimeClasspath,testCompileClasspa
 commons-io:commons-io:2.11.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 de.thetaphi:forbiddenapis:3.8=compileClasspath,jmhCompileClasspath
 info.picocli:picocli:4.6.3=jmhRuntimeClasspath,testRuntimeClasspath
-io.sqreen:libsqreen:11.2.0=jmhRuntimeClasspath,testRuntimeClasspath
+io.sqreen:libsqreen:12.0.0=jmhRuntimeClasspath,testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 jaxen:jaxen:1.2.0=spotbugs
 jline:jline:2.14.6=jmhRuntimeClasspath,testRuntimeClasspath