Merge branch 'master' into nayeem-kamal/RC-experimental-tracing-enabled-tmp

Author: nayeem-kamal

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit a0064c4012f19fbec5b9f0583efc80ab5f37c35e
+default_system_tests_commit: &default_system_tests_commit 454b41b414ba9e4b534893c2fe77477428d3bdca
 
 parameters:
   nightly:
@@ -113,7 +113,7 @@ commands:
       version of Testcontainers.
     steps:
       - setup_remote_docker:
-          version: 20.10.18
+          version: docker24
           # DLC shares Docker layers across jobs (at an extra cost).
           # But its time to setup (~1min) exceeds the time required to prefetch all images we use.
           docker_layer_caching: false
@@ -242,7 +242,7 @@ commands:
           paths:
             # Gradle version specific cache for incremental builds. Needs to match version in
             # gradle/wrapper/gradle-wrapper.properties
-            - ~/.gradle/caches/8.3
+            - ~/.gradle/caches/8.4
             # Workspace
             - ~/dd-trace-java/.gradle
             - ~/dd-trace-java/workspace
@@ -287,10 +287,16 @@ jobs:
       collectLibs:
         type: boolean
         default: false
+      triggeredBy:
+        type: string
+        default: ".*"
 
     steps:
       - setup_code
 
+      - skip_unless_matching_files_changed:
+          pattern: << parameters.triggeredBy >>
+
       - restore_dependency_cache:
           cacheType: << parameters.cacheType >>
 
@@ -374,13 +380,20 @@ jobs:
         type: string
       cacheType:
         type: string
+      triggeredBy:
+        type: string
+        default: ".*"
 
     resource_class: medium+
 
     parallelism: << parameters.parallelism >>
 
     steps:
       - setup_code
+
+      - skip_unless_matching_files_changed:
+          pattern: << parameters.triggeredBy >>
+
       - restore_dependency_cache:
           cacheType: << parameters.cacheType >>
       - restore_build_cache:
@@ -947,6 +960,7 @@ build_test_jobs: &build_test_jobs
       name: build_inst
       gradleTarget: :instrumentationTest
       cacheType: inst
+      triggeredBy: *instrumentation_modules
   - build:
       name: build_latestdep
       gradleTarget: :instrumentationLatestDepTest
@@ -986,6 +1000,7 @@ build_test_jobs: &build_test_jobs
       parallelism: 4
       gradleTarget: ":instrumentationCheck"
       cacheType: inst
+      triggeredBy: *instrumentation_modules
 
   - check:
       requires:
@@ -1319,6 +1334,7 @@ build_test_jobs: &build_test_jobs
 {% endfor %}
         - profiling
         - debugger
+        - system-tests
       name: required
       stage: required
 

.github/CODEOWNERS

@@ -47,3 +47,7 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java
 **/Iast*.groovy                        @DataDog/asm-java
+
+# @DataDog/data-jobs-monitoring
+dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
+dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring

.github/workflows/README.md

@@ -4,16 +4,6 @@ This lists and describes the repository GitHub actions.
 
 ## Release Management
 
-### add-assets-to-release [🔗](add-assets-to-release.yaml)
-
-_Trigger:_ When a release is published.
-
-_Actions:_
-* Ensure the release name is properly formatted (using `x.y.z` format),
-* Download `dd-java-agent`, `dd-trace-api` and `dd-trace-ot` artifacts from Sonatype (aka _Maven Central_ and upload them to the release (`dd-java-agent` will also be uploaded without version number).
-
-_Recovery:_ Download artifacts and upload them manually to the release.
-
 ### add-milestone-to-pull-requests [🔗](add-milestone-to-pull-requests.yaml)
 
 _Trigger:_ When a PR to `master` is closed.

.github/workflows/codeql-analysis.yml

@@ -33,7 +33,7 @@ jobs:
           # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
       - name: Build dd-trace-java for creating the CodeQL database
-        run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8
+        run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@1a927e9307bc11970b2c679922ebc4d03a5bd980 # 1.0.31

.github/workflows/draft-release-notes-on-tag.yaml

@@ -16,12 +16,13 @@ jobs:
           result-encoding: string
           script: |
             // Get the milestone title ("X.Y.Z") from tag name ("vX.Y.Z(-rc)")
-            const match = '${{github.event.ref}}'.match(/v(\d+\.\d+\.\d+)(?:-rc\d+)?/i)
+            const match = '${{github.event.ref}}'.match(/v(\d+\.\d+\.\d+)(-rc\d+)?/i)
             if (!match) {
               core.setFailed('Failed to parse tag name into milestone name: ${{github.event.ref}}')
               return
             }
             const milestoneTitle = match[1]
+            const isReleaseCandidate = match[2] !== undefined
 
             // Look for the milestone
             const milestone = (await github.paginate('GET /repos/{owner}/{repo}/milestones', {
@@ -135,8 +136,10 @@ jobs:
             }
 
             var changelog = ''
-            if ('${{github.event.ref}}'.match(/rc/i)) {
-              changelog += ':warning:  This is a RELEASE CANDIDATE and is NOT intended for use in production. Please contact Datadog support regarding any problems in this RC.\n'
+            if (isReleaseCandidate) {
+              changelog += '> [!WARNING]\n' +
+                '> This is a **release candidate** and is **not** intended for use in production.  \n' +
+                'Please [open an issue](https://github.com/DataDog/dd-trace-java/issues/new) regarding any problems in this release candidate.\n\n'
             }
             if (prByComponents.size > 0) {
               changelog += '# Components\n\n';

.github/workflows/trivy-analysis.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Build and publish artifacts locally
         run: |
-          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4
+          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Copy published artifacts
         run: |

.gitlab-ci.yml

@@ -91,6 +91,13 @@ package-arm:
   script:
     - ../.gitlab/build_java_package.sh
 
+package-oci:
+  stage: package
+  extends: .package-oci
+  when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
+  script:
+    - ../.gitlab/build_java_package_oci.sh
+
 .release-package:
   stage: deploy
   variables:
@@ -139,27 +146,6 @@ deploy_to_profiling_backend:
     UPSTREAM_TAG: $CI_COMMIT_TAG
     FORCE_TRIGGER: $FORCE_TRIGGER
 
-deploy_to_di_backend:automatic:
-  stage: deploy
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-      when: on_success
-    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
-      when: on_success
-  trigger:
-    project: DataDog/debugger-demos
-    branch: main
-  variables:
-    UPSTREAM_PACKAGE_JOB: build
-    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
-    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
-    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
-    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
-    UPSTREAM_TAG: $CI_COMMIT_TAG
-    UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
-
 deploy_to_di_backend:manual:
   stage: deploy
   rules:

.gitlab/build_java_package_oci.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [ -z "$CI_COMMIT_TAG" ] ; then
+  source ../upstream.env
+  VERSION=$UPSTREAM_TRACER_VERSION$CI_VERSION_SUFFIX
+else
+  VERSION=${CI_COMMIT_TAG##v}
+fi
+
+echo -n "$VERSION" > auto_inject-java.version
+
+mkdir -p sources
+cp ../workspace/dd-java-agent/build/libs/*.jar sources/dd-java-agent.jar
+cp auto_inject-java.version sources/version
+
+datadog-package create \
+    --version="$VERSION" \
+    --package="datadog-apm-library-java" \
+    --archive=true \
+    --archive-path="datadog-apm-library-java-$VERSION.tar" \
+    ./sources