Merge branch 'master' into mhlidd/update_trace_header_tags

Author: mhlidd

.github/workflows/README.md

@@ -85,13 +85,6 @@ _Action:_
 
 _Recovery:_ Check at the milestone for the related issues and update them manually.
 
-### prune-github-container-registry [🔗](prune-github-container-registry.yaml)
-
-_Trigger:_ Every day or manually.
-
-_Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
-
-_Recovery:_ Manually trigger the action again.
 
 ### prune-old-pull-requests [🔗](prune-old-pull-requests.yaml)
 

.github/workflows/analyze-changes.yaml

@@ -40,7 +40,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -57,7 +57,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
 
   trivy:
     name: Analyze changes with Trivy
@@ -122,7 +122,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/prune-github-container-registry.yaml

@@ -119,6 +119,18 @@ default:
   - .gitlab/cgroup-info.sh
   - gitlab_section_end "cgroup-info"
 
+.gitlab_base_ref_params: &gitlab_base_ref_params
+  - |
+    # FIXME: Disabled until we find a way to not hit GitHub API rate limit
+    if false && [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then
+      export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
+      if [[ -n "$GIT_BASE_REF" ]]; then
+        export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
+      else
+        echo "Failed to find base ref for PR" >&2
+      fi
+    fi
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
   stage: build
@@ -223,7 +235,8 @@ build_tests:
         MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on
 
   script:
-    - ./gradlew clean $GRADLE_TARGET -PskipTests $GRADLE_ARGS
+    - *gitlab_base_ref_params
+    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS
 
 populate_dep_cache:
   extends: build_tests
@@ -327,7 +340,8 @@ test_published_artifacts:
   variables:
     CACHE_TYPE: lib
   script:
-    - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+    - *gitlab_base_ref_params
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
   after_script:
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
@@ -460,6 +474,7 @@ muzzle-dep-report:
     - if: $CI_COMMIT_BRANCH == "master"
       when: on_success
   script:
+    - *gitlab_base_ref_params
     - >
       if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
       then
@@ -506,7 +521,7 @@ muzzle-dep-report:
     CI_USE_TEST_AGENT: "true"
     CI_AGENT_HOST: local-agent
   services:
-    - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+    - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.24.1
       alias: local-agent
       variables:
         LOG_LEVEL: "DEBUG"
@@ -729,6 +744,7 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
     UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
 
+# If the deploy_to_sonatype job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
 deploy_to_sonatype:
   extends: .gradle_build
   stage: publish
@@ -746,8 +762,8 @@ deploy_to_sonatype:
     - when: manual
       allow_failure: true
   script:
-    - export SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_username --with-decryption --query "Parameter.Value" --out text)
-    - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text)
+    - export SONATYPE_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
+    - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
     - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
@@ -765,7 +781,7 @@ deploy_artifacts_to_github:
       when: never
     - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
       when: on_success
-  # Requires the deploy_to_sonatype job to have run first the UP-TO-DATE gradle check across jobs is broken
+  # Requires the deploy_to_sonatype job to have run first (the UP-TO-DATE gradle check across jobs is broken)
   # This will deploy the artifacts built from the publishToSonatype task to the GitHub release
   needs:
     - job: deploy_to_sonatype

.gitlab-ci.yml

@@ -27,9 +27,6 @@
     UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
 
-    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-java
-    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
-
 benchmarks-startup:
   extends: .benchmarks
   script:
@@ -56,6 +53,7 @@ benchmarks-dacapo:
 
 benchmarks-post-results:
   extends: .benchmarks
+  tags: ["arch:amd64"]
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/upload-results-to-s3.sh

.gitlab/benchmarks.yml

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Determines the base branch for the current PR (if we are running in a PR).
+set -euo pipefail
+
+CURRENT_HEAD_SHA="$(git rev-parse HEAD)"
+if [[ -z "${CURRENT_HEAD_SHA:-}" ]]; then
+  echo "Failed to determine current HEAD SHA" >&2
+  exit 1
+fi
+
+# 'workspace' is declared in the ci pipeline cache
+CACHE_PATH=workspace/find-gh-base-ref.cache
+save_cache() {
+  local base_ref="$1"
+  local head_sha="$2"
+  mkdir -p workspace
+  echo "CACHED_BASE_REF=${base_ref}" > "$CACHE_PATH"
+  echo "CACHED_HEAD_SHA=${head_sha}" >> "$CACHE_PATH"
+}
+
+# Get cached result (if HEAD commit matches)
+if [[ -f $CACHE_PATH ]]; then
+  set -a
+  source "$CACHE_PATH"
+  set +a
+  if [[ "$CURRENT_HEAD_SHA" == "${CACHED_HEAD_SHA:-}" && -n "${CACHED_BASE_REF:-}" ]]; then
+    echo "Cache hit on $CACHE_PATH" >&2
+    echo "$CACHED_BASE_REF"
+    exit 0
+  else
+    echo "Cache miss on $CACHE_PATH" >&2
+  fi
+fi
+
+# Happy path: if we're just one commit away from master, base ref is master.
+if [[ $(git log --pretty=oneline origin/master..HEAD | wc -l) -eq 1 ]]; then
+  echo "We are just one commit away from master, base ref is master" >&2
+  save_cache "master" "$CURRENT_HEAD_SHA"
+  echo "master"
+  exit 0
+fi
+
+# In GitLab: we have no reference to the base branch or even the PR number.
+# We have to find it from the current branch name, which is defined in
+# CI_COMMIT_REF_NAME.
+if [[ -z "${CI_COMMIT_REF_NAME}" ]]; then
+  echo "CI_COMMIT_REF_NAME is not set, not running in GitLab CI?" >&2
+  exit 1
+fi
+
+# In GitLab, CI_PROJECT_NAME is set, otherwise, set it for testing.
+export CI_PROJECT_NAME="${CI_PROJECT_NAME:-dd-trace-java}"
+
+if [[ -z "${GITHUB_TOKEN:-}" ]]; then
+  echo "GITHUB_TOKEN is not set, fetching from AWS SSM" >&2
+  if ! command -v aws >/dev/null 2>&1; then
+    echo "aws is not installed, please install it" >&2
+    exit 1
+  fi
+  set +e
+  GITHUB_TOKEN=$(aws ssm get-parameter --name "ci.$CI_PROJECT_NAME.gh_release_token" --with-decryption --query "Parameter.Value" --output text)
+  set -e
+  if [[ -z "${GITHUB_TOKEN:-}" ]]; then
+    echo "Failed to fetch GITHUB_TOKEN from AWS SSM" >&2
+    exit 1
+  fi
+  export GITHUB_TOKEN
+fi
+
+if ! command -v curl >/dev/null 2>&1; then
+  echo "curl is not installed, please install it" >&2
+  exit 1
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "jq is not installed, please install it" >&2
+  exit 1
+fi
+
+while true; do
+  set +e
+  PR_DATA=$(curl \
+    -XGET \
+    --silent \
+    --include \
+    --fail-with-body \
+    -H 'Accept: application/vnd.github+json' \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    "https://api.github.com/repos/datadog/dd-trace-java/pulls?head=DataDog:${CI_COMMIT_REF_NAME}&sort=updated&direction=desc")
+  exit_code=$?
+  set -e
+  if [[ ${exit_code} -eq 0 ]]; then
+    PR_NUMBER=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].number')
+    PR_BASE_REF=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].base.ref')
+    if [[ -n "${PR_BASE_REF:-}" ]]; then
+      echo "PR is https://github.com/datadog/dd-trace-java/pull/${PR_NUMBER} and base ref is ${PR_BASE_REF}">&2
+      save_cache "${PR_BASE_REF}" "$CURRENT_HEAD_SHA"
+      echo "${PR_BASE_REF}"
+      exit 0
+    fi
+  fi
+  if echo "$PR_DATA" | grep -q "^x-ratelimit-reset:"; then
+    reset_timestamp=$(echo -n "$PR_DATA" | grep "^x-ratelimit-reset:" | sed -e 's/^x-ratelimit-reset: //' -e 's/\r//')
+    now=$(date +%s)
+    sleep_time=$((reset_timestamp - now + 1))
+    echo "GitHub rate limit exceeded, sleeping for ${sleep_time} seconds" >&2
+    sleep "${sleep_time}"
+    continue
+  fi
+  echo -e "GitHub request failed for an unknown reason:\n$(echo "$PR_DATA" | sed '/^$/q')" >&2
+  exit 1
+done

.gitlab/find-gh-base-ref.sh

@@ -22,8 +22,6 @@
       - platform/artifacts/
     expire_in: 3 months
   variables:
-    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
-
     K6_OPTIONS_WARMUP_RATE: 2000
     K6_OPTIONS_WARMUP_DURATION: 5m
     K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s

.gitlab/macrobenchmarks.yml

@@ -21,7 +21,7 @@ plugins {
   id 'de.thetaphi.forbiddenapis' version '3.8'
 
   id 'pl.allegro.tech.build.axion-release' version '1.14.4'
-  id 'io.github.gradle-nexus.publish-plugin' version '1.3.0'
+  id 'io.github.gradle-nexus.publish-plugin' version '2.0.0'
 
   id 'com.gradleup.shadow' version '8.3.6' apply false
   id 'me.champeau.jmh' version '0.7.0' apply false
@@ -35,13 +35,17 @@ def isCI = System.getenv("CI") != null
 
 apply from: "$rootDir/gradle/repositories.gradle"
 apply from: "$rootDir/gradle/scm.gradle"
+
 spotless {
   // only resolve the spotless dependencies once in the build
   predeclareDeps()
 }
+
 spotlessPredeclare {
   // these need to align with the types and versions in gradle/spotless.gradle
   java {
+    removeUnusedImports()
+
     // This is the last Google Java Format version that supports Java 8
     googleJavaFormat('1.7')
   }
@@ -88,7 +92,7 @@ nexusPublishing {
     def forceLocal = project.hasProperty('forceLocal') && forceLocal
     if (forceLocal && !isCI) {
       local {
-        // For testing use with https://hub.docker.com/r/sonatype/nexus
+        // For testing, use with https://hub.docker.com/r/sonatype/nexus
         // docker run --rm -d -p 8081:8081 --name nexus sonatype/nexus:oss
         // ./gradlew publishToLocal
         // Doesn't work for testing releases though... (due to staging)
@@ -99,7 +103,13 @@ nexusPublishing {
         allowInsecureProtocol = true
       }
     } else {
+      // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central
+      // For official documentation:
+      // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration
+      // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods
       sonatype {
+        nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/"))
+        snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/"))
         username = System.getenv("SONATYPE_USERNAME")
         password = System.getenv("SONATYPE_PASSWORD")
       }