Merge branch 'master' into smola/unique-onrootspanfinished

Author: smola

.gitlab/benchmarks.yml

@@ -10,6 +10,7 @@ variables:
   image: $BASE_CI_IMAGE
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
+    - export CIRCLE_CI_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.circleci_token --with-decryption --query "Parameter.Value" --out text)
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java

@@ -86,7 +86,10 @@ public static synchronized void run(
     if (config.isDebuggerExceptionEnabled()) {
       defaultExceptionDebugger =
           new DefaultExceptionDebugger(
-              configurationUpdater, classNameFiltering, EXCEPTION_CAPTURE_INTERVAL);
+              configurationUpdater,
+              classNameFiltering,
+              EXCEPTION_CAPTURE_INTERVAL,
+              config.getDebuggerMaxExceptionPerSecond());
       DebuggerContext.initExceptionDebugger(defaultExceptionDebugger);
     }
     if (config.isDebuggerInstrumentTheWorld()) {

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java

@@ -7,6 +7,7 @@
 import com.datadog.debugger.agent.DebuggerAgent;
 import com.datadog.debugger.exception.ExceptionProbeManager.ThrowableState;
 import com.datadog.debugger.sink.Snapshot;
+import com.datadog.debugger.util.CircuitBreaker;
 import com.datadog.debugger.util.ClassNameFiltering;
 import com.datadog.debugger.util.ExceptionHelper;
 import datadog.trace.bootstrap.debugger.DebuggerContext;
@@ -31,24 +32,29 @@ public class DefaultExceptionDebugger implements DebuggerContext.ExceptionDebugg
   private final ExceptionProbeManager exceptionProbeManager;
   private final ConfigurationUpdater configurationUpdater;
   private final ClassNameFiltering classNameFiltering;
+  private final CircuitBreaker circuitBreaker;
 
   public DefaultExceptionDebugger(
       ConfigurationUpdater configurationUpdater,
       ClassNameFiltering classNameFiltering,
-      Duration captureInterval) {
+      Duration captureInterval,
+      int maxExceptionPerSecond) {
     this(
         new ExceptionProbeManager(classNameFiltering, captureInterval),
         configurationUpdater,
-        classNameFiltering);
+        classNameFiltering,
+        maxExceptionPerSecond);
   }
 
   DefaultExceptionDebugger(
       ExceptionProbeManager exceptionProbeManager,
       ConfigurationUpdater configurationUpdater,
-      ClassNameFiltering classNameFiltering) {
+      ClassNameFiltering classNameFiltering,
+      int maxExceptionPerSecond) {
     this.exceptionProbeManager = exceptionProbeManager;
     this.configurationUpdater = configurationUpdater;
     this.classNameFiltering = classNameFiltering;
+    this.circuitBreaker = new CircuitBreaker(maxExceptionPerSecond, Duration.ofSeconds(1));
   }
 
   @Override
@@ -59,6 +65,9 @@ public void handleException(Throwable t, AgentSpan span) {
       }
       return;
     }
+    if (!circuitBreaker.trip()) {
+      return;
+    }
     String fingerprint = Fingerprinter.fingerprint(t, classNameFiltering);
     if (fingerprint == null) {
       LOGGER.debug("Unable to fingerprint exception", t);

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/CircuitBreaker.java

@@ -0,0 +1,41 @@
+package com.datadog.debugger.util;
+
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// CircuitBreaker is a simple circuit breaker implementation that allows a certain number of trips
+// within a time window. If the number of trips exceeds the limit, the circuit breaker will trip and
+// return false until the time window has passed.
+public class CircuitBreaker {
+  private static final Logger LOGGER = LoggerFactory.getLogger(CircuitBreaker.class);
+
+  private final int maxTrips;
+  private final Duration timeWindow;
+  private AtomicInteger count = new AtomicInteger(0);
+  private volatile long lastResetTime = System.currentTimeMillis();
+  private volatile long lastLoggingTime = System.currentTimeMillis();
+
+  public CircuitBreaker(int maxTrips, Duration timeWindow) {
+    this.maxTrips = maxTrips;
+    this.timeWindow = timeWindow;
+  }
+
+  public boolean trip() {
+    int localCount = count.incrementAndGet();
+    if (localCount > maxTrips) {
+      long currentTime = System.currentTimeMillis();
+      if (currentTime - lastLoggingTime > Duration.ofMinutes(1).toMillis()) {
+        lastLoggingTime = currentTime;
+        LOGGER.debug("Circuit breaker opened");
+      }
+      if (currentTime - lastResetTime > timeWindow.toMillis()) {
+        lastResetTime = currentTime;
+        localCount = 1;
+        count.set(localCount);
+      }
+    }
+    return localCount <= maxTrips;
+  }
+}

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java

@@ -60,7 +60,8 @@ public void setUp() {
     configurationUpdater = mock(ConfigurationUpdater.class);
     classNameFiltering = new ClassNameFiltering(emptySet());
     exceptionDebugger =
-        new DefaultExceptionDebugger(configurationUpdater, classNameFiltering, Duration.ofHours(1));
+        new DefaultExceptionDebugger(
+            configurationUpdater, classNameFiltering, Duration.ofHours(1), 100);
     listener = new TestSnapshotListener(createConfig(), mock(ProbeStatusSink.class));
     DebuggerAgentHelper.injectSink(listener);
   }
@@ -203,7 +204,7 @@ public void doubleNestedException() {
   public void filteringOutErrors() {
     ExceptionProbeManager manager = mock(ExceptionProbeManager.class);
     exceptionDebugger =
-        new DefaultExceptionDebugger(manager, configurationUpdater, classNameFiltering);
+        new DefaultExceptionDebugger(manager, configurationUpdater, classNameFiltering, 100);
     exceptionDebugger.handleException(new AssertionError("test"), mock(AgentSpan.class));
     verify(manager, times(0)).isAlreadyInstrumented(any());
   }

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeInstrumentationTest.java

@@ -326,7 +326,7 @@ private TestSnapshotListener setupExceptionDebugging(
     DebuggerContext.initValueSerializer(new JsonSnapshotSerializer());
     DefaultExceptionDebugger exceptionDebugger =
         new DefaultExceptionDebugger(
-            exceptionProbeManager, configurationUpdater, classNameFiltering);
+            exceptionProbeManager, configurationUpdater, classNameFiltering, 100);
     DebuggerContext.initExceptionDebugger(exceptionDebugger);
     configurationUpdater.accept(REMOTE_CONFIG, null);
     return listener;

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/util/CircuitBreakerTest.java

@@ -0,0 +1,37 @@
+package com.datadog.debugger.util;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.util.concurrent.locks.LockSupport;
+import org.junit.jupiter.api.Test;
+
+class CircuitBreakerTest {
+
+  @Test
+  void noBreaker() {
+    CircuitBreaker cb = new CircuitBreaker(3, Duration.ofMillis(10));
+    for (int i = 0; i < 10; i++) {
+      assertTrue(cb.trip());
+      LockSupport.parkNanos(Duration.ofMillis(50).toNanos());
+    }
+  }
+
+  @Test
+  void breaker() {
+    CircuitBreaker cb = new CircuitBreaker(3, Duration.ofMillis(200));
+    for (int i = 0; i < 3; i++) {
+      assertTrue(cb.trip());
+    }
+    for (int i = 0; i < 100; i++) {
+      assertFalse(cb.trip());
+    }
+    LockSupport.parkNanos(Duration.ofMillis(250).toNanos());
+    for (int i = 0; i < 3; i++) {
+      assertTrue(cb.trip());
+    }
+    for (int i = 0; i < 100; i++) {
+      assertFalse(cb.trip());
+    }
+  }
+}

dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/AgentTestRunner.groovy

@@ -546,6 +546,18 @@ abstract class AgentTestRunner extends DDSpecification implements AgentBuilder.L
       return delegate.getDataTop(key)
     }
 
+    @Override
+    void setMetaStructTop(String key, Object value) {
+      check()
+      delegate.setMetaStructTop(key, value)
+    }
+
+    @Override
+    void setMetaStructCurrent(String key, Object value) {
+      check()
+      delegate.setMetaStructCurrent(key, value)
+    }
+
     @Override
     void effectivelyBlocked() {
       check()

dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

@@ -174,6 +174,7 @@ public final class ConfigDefaults {
   static final boolean DEFAULT_DEBUGGER_SYMBOL_FORCE_UPLOAD = false;
   static final int DEFAULT_DEBUGGER_SYMBOL_FLUSH_THRESHOLD = 100; // nb of classes
   static final boolean DEFAULT_DEBUGGER_EXCEPTION_ENABLED = false;
+  static final int DEFAULT_DEBUGGER_MAX_EXCEPTION_PER_SECOND = 100;
   static final boolean DEFAULT_DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT = true;
 
   static final boolean DEFAULT_TRACE_REPORT_HOSTNAME = false;

dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java

@@ -31,6 +31,8 @@ public final class DebuggerConfig {
   public static final String DEBUGGER_SYMBOL_FLUSH_THRESHOLD = "symbol.database.flush.threshold";
   public static final String DEBUGGER_EXCEPTION_ENABLED = "exception.debugging.enabled";
   public static final String EXCEPTION_REPLAY_ENABLED = "exception.replay.enabled";
+  public static final String DEBUGGER_MAX_EXCEPTION_PER_SECOND =
+      "exception.replay.max.exception.analysis.limit";
   public static final String DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT =
       "internal.exception.replay.only.local.root";
   public static final String THIRD_PARTY_INCLUDES = "third.party.includes";