Merge branch 'master' into nicholas.hulston/dynamodb-span-pointers

Author: nhulston

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 6980534f333b3f7a35d83df2230f00f4e26642f5
+default_system_tests_commit: &default_system_tests_commit 590162e1f5223cbbcd2876fdb820837cb13a85e1
 
 parameters:
   nightly:
@@ -891,6 +891,7 @@ jobs:
             for log_dir in logs*; do
               tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
             done
+          when: always
 
       - store_artifacts:
           path: artifacts
@@ -980,6 +981,7 @@ jobs:
             for dir in system-tests/logs*/; do
                 cp -r "$dir" logs_debugger
             done
+          when: always
 
       - store_test_results:
           path: logs_debugger

.gitlab-ci.yml

@@ -42,6 +42,13 @@ default:
   - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
   - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
 
+# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs
+.normalize_node_index: &normalize_node_index
+  - echo "CI_NODE_TOTAL=$CI_NODE_TOTAL , CI_NODE_INDEX=$CI_NODE_INDEX"
+  - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1}
+  - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1))
+  - echo "NORMALIZED_NODE_TOTAL=$NORMALIZED_NODE_TOTAL , NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX"
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
   stage: build
@@ -68,6 +75,7 @@ default:
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
     - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
+    - *normalize_node_index
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
@@ -143,6 +151,49 @@ test_published_artifacts:
     paths:
       - ./check_reports
 
+.check_job:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: lib
+  script:
+    - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh --destination ./check_reports --move
+  artifacts:
+    when: always
+    paths:
+      - ./check_reports
+      - '.gradle/daemon/*/*.out.log'
+
+check_base:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":baseCheck"
+
+check_inst:
+  extends: .check_job
+  parallel: 4
+  variables:
+    GRADLE_TARGET: ":instrumentationCheck"
+
+check_smoke:
+  extends: .check_job
+  parallel: 4
+  variables:
+    GRADLE_TARGET: ":smokeCheck"
+
+check_profiling:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":profilingCheck"
+
+check_debugger:
+  extends: .check_job
+  variables:
+    GRADLE_TARGET: ":debuggerCheck"
+
 muzzle:
   extends: .gradle_build
   needs: [ build ]
@@ -154,13 +205,10 @@ muzzle:
     BUILD_CACHE_TYPE: test
   script:
     - export SKIP_BUILDSCAN="true"
-    - echo CI_NODE_INDEX=$CI_NODE_INDEX
-    - echo CI_NODE_TOTAL=CI_NODE_TOTAL
     - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
     - sort workspace/build/muzzleTasks > sortedMuzzleTasks
-    - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
-    - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1))
-    - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS
+    - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
+    - ./gradlew `cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs` $GRADLE_ARGS
   after_script:
     - .circleci/collect_reports.sh
   artifacts:
@@ -260,6 +308,16 @@ required:
       optional: true
     - job: agent_integration_tests
       optional: true
+    - job: check_base
+      optional: true
+    - job: check_inst
+      optional: true
+    - job: check_smoke
+      optional: true
+    - job: check_profiling
+      optional: true
+    - job: check_debugger
+      optional: true
 
 deploy_to_profiling_backend:
   stage: publish

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -96,9 +96,9 @@ private enum AgentFeature {
     CIVISIBILITY_AGENTLESS(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED, false),
     USM(UsmConfig.USM_ENABLED, false),
     TELEMETRY(GeneralConfig.TELEMETRY_ENABLED, true),
-    DYNAMIC_INSTRUMENTATION(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED, false),
-    EXCEPTION_REPLAY(DebuggerConfig.EXCEPTION_REPLAY_ENABLED, false),
-    CODE_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false),
+    DEBUGGER(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED, false),
+    EXCEPTION_DEBUGGING(DebuggerConfig.EXCEPTION_REPLAY_ENABLED, false),
+    SPAN_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false),
     DATA_JOBS(GeneralConfig.DATA_JOBS_ENABLED, false),
     AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false);
 
@@ -149,10 +149,9 @@ public boolean isEnabledByDefault() {
   private static boolean ciVisibilityEnabled = false;
   private static boolean usmEnabled = false;
   private static boolean telemetryEnabled = true;
-  private static boolean dynamicInstrumentationEnabled = false;
-  private static boolean exceptionReplayEnabled = false;
-  private static boolean codeOriginEnabled = false;
-  private static boolean distributedDebuggerEnabled = false;
+  private static boolean debuggerEnabled = false;
+  private static boolean exceptionDebuggingEnabled = false;
+  private static boolean spanOriginEnabled = false;
   private static boolean agentlessLogSubmissionEnabled = false;
 
   /**
@@ -262,9 +261,9 @@ public static void start(
             || isFeatureEnabled(AgentFeature.DEPRECATED_REMOTE_CONFIG);
     cwsEnabled = isFeatureEnabled(AgentFeature.CWS);
     telemetryEnabled = isFeatureEnabled(AgentFeature.TELEMETRY);
-    dynamicInstrumentationEnabled = isFeatureEnabled(AgentFeature.DYNAMIC_INSTRUMENTATION);
-    exceptionReplayEnabled = isFeatureEnabled(AgentFeature.EXCEPTION_REPLAY);
-    codeOriginEnabled = isFeatureEnabled(AgentFeature.CODE_ORIGIN);
+    debuggerEnabled = isFeatureEnabled(AgentFeature.DEBUGGER);
+    exceptionDebuggingEnabled = isFeatureEnabled(AgentFeature.EXCEPTION_DEBUGGING);
+    spanOriginEnabled = isFeatureEnabled(AgentFeature.SPAN_ORIGIN);
     agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION);
 
     if (profilingEnabled) {
@@ -1115,10 +1114,7 @@ private static void shutdownProfilingAgent(final boolean sync) {
   }
 
   private static void maybeStartDebugger(Instrumentation inst, Class<?> scoClass, Object sco) {
-    if (isExplicitlyDisabled(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED)
-        && isExplicitlyDisabled(DebuggerConfig.EXCEPTION_REPLAY_ENABLED)
-        && isExplicitlyDisabled(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED)
-        && isExplicitlyDisabled(DebuggerConfig.DISTRIBUTED_DEBUGGER_ENABLED)) {
+    if (!debuggerEnabled && !exceptionDebuggingEnabled && !spanOriginEnabled) {
       return;
     }
     if (!remoteConfigEnabled) {
@@ -1128,11 +1124,6 @@ && isExplicitlyDisabled(DebuggerConfig.DISTRIBUTED_DEBUGGER_ENABLED)) {
     startDebuggerAgent(inst, scoClass, sco);
   }
 
-  private static boolean isExplicitlyDisabled(String booleanKey) {
-    return Config.get().configProvider().isSet(booleanKey)
-        && !Config.get().configProvider().getBoolean(booleanKey);
-  }
-
   private static synchronized void startDebuggerAgent(
       Instrumentation inst, Class<?> scoClass, Object sco) {
     StaticEventLogger.begin("Debugger");

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -5,6 +5,7 @@
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
+import java.util.Collection;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -47,7 +48,8 @@ TestSuiteImpl testSuiteStart(
   SkipReason skipReason(TestIdentifier test);
 
   @Nonnull
-  TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource);
+  TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags);
 
   /**
    * Returns the priority of the test execution that can be used for ordering tests. The higher the

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -29,7 +29,6 @@
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.api.gateway.RequestContextSlot;
-import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
@@ -230,23 +229,22 @@ public void setSkipReason(String skipReason) {
   public void end(@Nullable Long endTime) {
     closeOutstandingSpans();
 
-    final AgentScope scope = AgentTracer.activeScope();
-    if (scope == null) {
+    final AgentSpan activeSpan = AgentTracer.activeSpan();
+    if (activeSpan == null) {
       throw new IllegalStateException(
-          "No active scope present, it is possible that end() was called multiple times");
+          "No active span present, it is possible that end() was called multiple times");
     }
 
-    AgentSpan scopeSpan = scope.span();
-    if (scopeSpan != span) {
+    if (activeSpan != this.span) {
       throw new IllegalStateException(
-          "Active scope does not correspond to the finished test, "
+          "Active span does not correspond to the finished test, "
               + "it is possible that end() was called multiple times "
               + "or an operation that was started by the test is still in progress; "
-              + "active scope span is: "
-              + scopeSpan
+              + "active span is: "
+              + activeSpan
               + "; "
               + "expected span is: "
-              + span);
+              + this.span);
     }
 
     InstrumentationTestBridge.fireBeforeTestEnd(context);
@@ -263,7 +261,7 @@ public void end(@Nullable Long endTime) {
       }
     }
 
-    scope.close();
+    AgentTracer.closeActive();
 
     onSpanFinish.accept(span);
 
@@ -310,18 +308,17 @@ public void end(@Nullable Long endTime) {
    * spans stack until it encounters a CI Visibility span or the stack is empty.
    */
   private void closeOutstandingSpans() {
-    AgentScope scope;
-    while ((scope = AgentTracer.activeScope()) != null) {
-      AgentSpan span = scope.span();
+    AgentSpan activeSpan;
+    while ((activeSpan = AgentTracer.activeSpan()) != null) {
 
-      if (span == this.span || span.getTag(Tags.TEST_SESSION_ID) != null) {
+      if (activeSpan == this.span || activeSpan.getTag(Tags.TEST_SESSION_ID) != null) {
         // encountered this span or another CI Visibility span (test, suite, module, session)
         break;
       }
 
-      log.debug("Closing outstanding span: {}", span);
-      scope.close();
-      span.finish();
+      log.debug("Closing outstanding span: {}", activeSpan);
+      AgentTracer.closeActive();
+      activeSpan.finish();
     }
   }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java

@@ -11,7 +11,6 @@
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
-import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
@@ -193,26 +192,25 @@ public void setSkipReason(String skipReason) {
   @Override
   public void end(@Nullable Long endTime) {
     if (!parallelized) {
-      final AgentScope scope = AgentTracer.activeScope();
-      if (scope == null) {
+      final AgentSpan activeSpan = AgentTracer.activeSpan();
+      if (activeSpan == null) {
         throw new IllegalStateException(
-            "No active scope present, it is possible that end() was called multiple times");
+            "No active span present, it is possible that end() was called multiple times");
       }
 
-      AgentSpan scopeSpan = scope.span();
-      if (scopeSpan != span) {
+      if (activeSpan != this.span) {
         throw new IllegalStateException(
-            "Active scope does not correspond to the finished suite, "
+            "Active span does not correspond to the finished suite, "
                 + "it is possible that end() was called multiple times "
                 + "or an operation that was started by the suite is still in progress; "
-                + "active scope span is: "
-                + scopeSpan
+                + "active span is: "
+                + activeSpan
                 + "; "
                 + "expected span is: "
-                + span);
+                + this.span);
       }
 
-      scope.close();
+      AgentTracer.closeActive();
     }
 
     onSpanFinish.accept(span);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -127,8 +127,9 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @Override
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
-    return executionStrategy.executionPolicy(test, testSource);
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
+    return executionStrategy.executionPolicy(test, testSource, testTags);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -27,6 +27,7 @@
 import datadog.trace.civisibility.test.ExecutionResults;
 import datadog.trace.civisibility.test.ExecutionStrategy;
 import datadog.trace.civisibility.utils.SpanUtils;
+import java.util.Collection;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -112,8 +113,9 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @Override
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
-    return executionStrategy.executionPolicy(test, testSource);
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
+    return executionStrategy.executionPolicy(test, testSource, testTags);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/NoOpTestEventsHandler.java

@@ -100,7 +100,8 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @NotNull
   @Override
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData source) {
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData source, Collection<String> testTags) {
     return Regular.INSTANCE;
   }