Add block_failure telemetry for Tomcat blocking enforcement failures

dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/latestDepTest/groovy/test/SetupSpecHelper.groovy

@@ -31,23 +31,34 @@ class SetupSpecHelper {
     ss.registerCallback(uriEvent, new TriFunction<RequestContext, String, URIDataAdapter, Flow<Void>>() {
         @Override
         Flow<Void> apply(RequestContext requestContext, String s, URIDataAdapter uriDataAdapter) {
-          requestContext.setBlockResponseFunction(TestSpringBlockResponseFunction.INSTANCE)
+          BlockResponseFunction original = requestContext.getBlockResponseFunction()
+          requestContext.setBlockResponseFunction(new TestSpringBlockResponseFunction(original))
           origUriCallback.apply(requestContext, s, uriDataAdapter)
         }
       })
   }
 
-  enum TestSpringBlockResponseFunction implements BlockResponseFunction {
-    INSTANCE
+  /**
+   * Commits the blocking response through Spring's request attributes when they are available,
+   * falling back to the block response function the server instrumentation had already registered
+   * (e.g. Tomcat's) for block points that happen before Spring populates RequestContextHolder.
+   */
+  static class TestSpringBlockResponseFunction implements BlockResponseFunction {
+    private final BlockResponseFunction original
+
+    TestSpringBlockResponseFunction(BlockResponseFunction original) {
+      this.original = original
+    }
 
     @Override
     boolean tryCommitBlockingResponse(TraceSegment segment, int statusCode, BlockingContentType templateType, Map<String, String> extraHeaders, String securityResponseId) {
       ServletRequestAttributes attributes = RequestContextHolder.requestAttributes
       if (attributes) {
         ServletBlockingHelper
           .commitBlockingResponse(segment, attributes.request, attributes.response, statusCode, templateType, extraHeaders, securityResponseId)
+        return true
       }
-      true
+      original != null && original.tryCommitBlockingResponse(segment, statusCode, templateType, extraHeaders, securityResponseId)
     }
   }
 }

dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/SetupSpecHelper.groovy

@@ -31,23 +31,34 @@ class SetupSpecHelper {
     ss.registerCallback(uriEvent, new TriFunction<RequestContext, String, URIDataAdapter, Flow<Void>>() {
         @Override
         Flow<Void> apply(RequestContext requestContext, String s, URIDataAdapter uriDataAdapter) {
-          requestContext.setBlockResponseFunction(TestSpringBlockResponseFunction.INSTANCE)
+          BlockResponseFunction original = requestContext.getBlockResponseFunction()
+          requestContext.setBlockResponseFunction(new TestSpringBlockResponseFunction(original))
           origUriCallback.apply(requestContext, s, uriDataAdapter)
         }
       })
   }
 
-  enum TestSpringBlockResponseFunction implements BlockResponseFunction {
-    INSTANCE
+  /**
+   * Commits the blocking response through Spring's request attributes when they are available,
+   * falling back to the block response function the server instrumentation had already registered
+   * (e.g. Tomcat's) for block points that happen before Spring populates RequestContextHolder.
+   */
+  static class TestSpringBlockResponseFunction implements BlockResponseFunction {
+    private final BlockResponseFunction original
+
+    TestSpringBlockResponseFunction(BlockResponseFunction original) {
+      this.original = original
+    }
 
     @Override
     boolean tryCommitBlockingResponse(TraceSegment segment, int statusCode, BlockingContentType templateType, Map<String, String> extraHeaders, String securityResponseId) {
       ServletRequestAttributes attributes = RequestContextHolder.requestAttributes
       if (attributes) {
         ServletBlockingHelper
           .commitBlockingResponse(segment, attributes.request, attributes.response, statusCode, templateType, extraHeaders, securityResponseId)
+        return true
       }
-      true
+      original != null && original.tryCommitBlockingResponse(segment, statusCode, templateType, extraHeaders, securityResponseId)
     }
   }
 }

dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-6.0/src/test/groovy/datadog/trace/instrumentation/springweb6/SetupSpecHelper.groovy

@@ -31,23 +31,34 @@ class SetupSpecHelper {
     ss.registerCallback(uriEvent, new TriFunction<RequestContext, String, URIDataAdapter, Flow<Void>>() {
         @Override
         Flow<Void> apply(RequestContext requestContext, String s, URIDataAdapter uriDataAdapter) {
-          requestContext.setBlockResponseFunction(TestSpringBlockResponseFunction.INSTANCE)
+          BlockResponseFunction original = requestContext.getBlockResponseFunction()
+          requestContext.setBlockResponseFunction(new TestSpringBlockResponseFunction(original))
           origUriCallback.apply(requestContext, s, uriDataAdapter)
         }
       })
   }
 
-  enum TestSpringBlockResponseFunction implements BlockResponseFunction {
-    INSTANCE
+  /**
+   * Commits the blocking response through Spring's request attributes when they are available,
+   * falling back to the block response function the server instrumentation had already registered
+   * (e.g. Tomcat's) for block points that happen before Spring populates RequestContextHolder.
+   */
+  static class TestSpringBlockResponseFunction implements BlockResponseFunction {
+    private final BlockResponseFunction original
+
+    TestSpringBlockResponseFunction(BlockResponseFunction original) {
+      this.original = original
+    }
 
     @Override
     boolean tryCommitBlockingResponse(TraceSegment segment, int statusCode, BlockingContentType templateType, Map<String, String> extraHeaders, String securityResponseId) {
       ServletRequestAttributes attributes = RequestContextHolder.requestAttributes
       if (attributes) {
         JakartaServletBlockingHelper
           .commitBlockingResponse(segment, attributes.request, attributes.response, statusCode, templateType, extraHeaders, securityResponseId)
+        return true
       }
-      true
+      original != null && original.tryCommitBlockingResponse(segment, statusCode, templateType, extraHeaders, securityResponseId)
     }
   }
 }

dd-java-agent/instrumentation/tomcat/tomcat-5.5/src/main/java/datadog/trace/instrumentation/tomcat/RequestInstrumentation.java

@@ -54,6 +54,7 @@ public String[] helperClassNames() {
       packageName + ".TomcatDecorator",
       packageName + ".TomcatDecorator$TomcatBlockResponseFunction",
       packageName + ".TomcatBlockingHelper",
+      packageName + ".BlockFailureReporter",
       packageName + ".RequestURIDataAdapter",
     };
   }

dd-java-agent/instrumentation/tomcat/tomcat-5.5/src/main/java/datadog/trace/instrumentation/tomcat/ResponseInstrumentation.java

@@ -38,6 +38,7 @@ public String[] helperClassNames() {
       packageName + ".TomcatDecorator",
       packageName + ".TomcatDecorator$TomcatBlockResponseFunction",
       packageName + ".TomcatBlockingHelper",
+      packageName + ".BlockFailureReporter",
       packageName + ".RequestURIDataAdapter",
     };
   }

dd-java-agent/instrumentation/tomcat/tomcat-5.5/src/main/java/datadog/trace/instrumentation/tomcat/TomcatServerInstrumentation.java

@@ -56,6 +56,7 @@ public String[] helperClassNames() {
       packageName + ".TomcatDecorator$TomcatBlockResponseFunction",
       packageName + ".RequestURIDataAdapter",
       packageName + ".TomcatBlockingHelper",
+      packageName + ".BlockFailureReporter",
     };
   }
 
@@ -186,9 +187,7 @@ public static class PostParseAdvice {
 
     @Advice.OnMethodExit(suppress = Throwable.class)
     public static void afterParse(
-        @Advice.Argument(1) Request req,
-        @Advice.Argument(3) Response resp,
-        @Advice.Return(readOnly = false) Boolean ret) {
+        @Advice.Argument(1) Request req, @Advice.Return(readOnly = false) Boolean ret) {
       Object contextObj = req.getAttribute(DD_CONTEXT_ATTRIBUTE);
       if (contextObj instanceof Context) {
         Context context = (Context) contextObj;
@@ -202,8 +201,7 @@ public static void afterParse(
           DECORATE.onRequest(span, req, req, parentContext);
           Flow.Action.RequestBlockingAction rba = span.getRequestBlockingAction();
           if (rba != null) {
-            TomcatBlockingHelper.commitBlockingResponse(
-                span.getRequestContext().getTraceSegment(), req, resp, rba);
+            BlockFailureReporter.tryCommitAndReport(span.getRequestContext(), rba);
             ret = false; // skip pipeline
           }
         }

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-5.5/src/main/java/datadog/trace/instrumentation/tomcat55/CommitActionInstrumentation.java

@@ -14,6 +14,7 @@
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.decorator.HttpServerDecorator;
+import datadog.trace.instrumentation.tomcat.BlockFailureReporter;
 import datadog.trace.instrumentation.tomcat.ExtractAdapter;
 import datadog.trace.instrumentation.tomcat.TomcatDecorator;
 import net.bytebuddy.asm.Advice;
@@ -75,6 +76,7 @@ public String[] helperClassNames() {
       pkg + ".TomcatDecorator",
       pkg + ".TomcatDecorator$TomcatBlockResponseFunction",
       pkg + ".TomcatBlockingHelper",
+      pkg + ".BlockFailureReporter",
       pkg + ".RequestURIDataAdapter",
     };
   }
@@ -114,7 +116,7 @@ static class ProcessCommitActionAdvice {
         Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action;
         BlockResponseFunction brf = requestContext.getBlockResponseFunction();
         if (brf != null) {
-          brf.tryCommitBlockingResponse(requestContext.getTraceSegment(), rba);
+          BlockFailureReporter.tryCommitAndReport(requestContext, rba);
           thiz.action(ActionCode.ACTION_CLOSE, null);
           return true;
         }

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-5.5/src/main/java/datadog/trace/instrumentation/tomcat55/ParsedBodyParametersInstrumentation.java

@@ -20,6 +20,7 @@
 import datadog.trace.api.gateway.RequestContextSlot;
 import datadog.trace.bootstrap.CallDepthThreadLocalMap;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
+import datadog.trace.instrumentation.tomcat.BlockFailureReporter;
 import java.util.Hashtable;
 import java.util.function.BiFunction;
 import net.bytebuddy.asm.Advice;
@@ -65,6 +66,11 @@ public Reference[] additionalMuzzleReferences() {
     return new Reference[] {PARAM_HASH_STRING_ARRAY_REFERENCE};
   }
 
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {"datadog.trace.instrumentation.tomcat.BlockFailureReporter"};
+  }
+
   @Override
   public void methodAdvice(MethodTransformer transformer) {
     transformer.applyAdvice(
@@ -143,7 +149,7 @@ static void after(
           Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action;
           BlockResponseFunction blockResponseFunction = reqCtx.getBlockResponseFunction();
           if (blockResponseFunction != null) {
-            blockResponseFunction.tryCommitBlockingResponse(reqCtx.getTraceSegment(), rba);
+            BlockFailureReporter.tryCommitAndReport(reqCtx, rba);
             if (t == null) {
               t = new BlockingException("Blocked request (for processParameters)");
             }

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-6.0/build.gradle

@@ -25,6 +25,7 @@ muzzle {
 
 dependencies {
   compileOnly group: 'org.apache.tomcat', name: 'coyote', version: '6.0.53'
+  implementation project(':dd-java-agent:instrumentation:tomcat:tomcat-common')
 }
 
 // testing happens in tomcat-5.5 module

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-6.0/src/main/java/datadog/trace/instrumentation/tomcat6/ParsedBodyParametersInstrumentation.java

@@ -12,13 +12,13 @@
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.agent.tooling.muzzle.Reference;
-import datadog.trace.api.gateway.BlockResponseFunction;
 import datadog.trace.api.gateway.CallbackProvider;
 import datadog.trace.api.gateway.Flow;
 import datadog.trace.api.gateway.RequestContext;
 import datadog.trace.api.gateway.RequestContextSlot;
 import datadog.trace.bootstrap.CallDepthThreadLocalMap;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
+import datadog.trace.instrumentation.tomcat.BlockFailureReporter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
@@ -53,6 +53,11 @@ public Reference[] additionalMuzzleReferences() {
     return new Reference[] {PARAM_HASH_VALUES_MAP_REFERENCE};
   }
 
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {"datadog.trace.instrumentation.tomcat.BlockFailureReporter"};
+  }
+
   @Override
   public void methodAdvice(MethodTransformer transformer) {
     transformer.applyAdvice(
@@ -129,15 +134,8 @@ static void after(
         Flow.Action action = flow.getAction();
         if (action instanceof Flow.Action.RequestBlockingAction) {
           Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action;
-          BlockResponseFunction blockResponseFunction = reqCtx.getBlockResponseFunction();
-          if (blockResponseFunction != null) {
-            boolean committedBlockingResponse =
-                blockResponseFunction.tryCommitBlockingResponse(reqCtx.getTraceSegment(), rba);
-            if (committedBlockingResponse) {
-              if (t == null) {
-                t = new BlockingException("Blocked request (for processParameters)");
-              }
-            }
+          if (BlockFailureReporter.tryCommitAndReport(reqCtx, rba) && t == null) {
+            t = new BlockingException("Blocked request (for processParameters)");
           }
         }
       } finally {

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-7.0/src/main/java/datadog/trace/instrumentation/tomcat7/CommitActionInstrumentation.java

@@ -15,6 +15,7 @@
 import datadog.trace.api.gateway.RequestContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.decorator.HttpServerDecorator;
+import datadog.trace.instrumentation.tomcat.BlockFailureReporter;
 import datadog.trace.instrumentation.tomcat.ExtractAdapter;
 import datadog.trace.instrumentation.tomcat.TomcatDecorator;
 import net.bytebuddy.asm.Advice;
@@ -76,6 +77,7 @@ public String[] helperClassNames() {
       pkg + ".TomcatDecorator",
       pkg + ".TomcatDecorator$TomcatBlockResponseFunction",
       pkg + ".TomcatBlockingHelper",
+      pkg + ".BlockFailureReporter",
       pkg + ".RequestURIDataAdapter",
     };
   }
@@ -121,7 +123,7 @@ static class ProcessCommitActionAdvice {
         Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action;
         BlockResponseFunction brf = requestContext.getBlockResponseFunction();
         if (brf != null) {
-          brf.tryCommitBlockingResponse(requestContext.getTraceSegment(), rba);
+          BlockFailureReporter.tryCommitAndReport(requestContext, rba);
           thiz.action(ActionCode.CLOSE, null);
           return true;
         }

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-7.0/src/main/java/datadog/trace/instrumentation/tomcat7/GlassFishBlockingHelper.java

@@ -7,6 +7,7 @@
 import datadog.trace.api.gateway.RequestContext;
 import datadog.trace.api.http.MultipartContentDecoder;
 import datadog.trace.bootstrap.blocking.BlockingActionHelper;
+import datadog.trace.instrumentation.tomcat.BlockFailureReporter;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -16,9 +17,13 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.Part;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public final class GlassFishBlockingHelper {
 
+  private static final Logger log = LoggerFactory.getLogger(GlassFishBlockingHelper.class);
+
   public static final int MAX_FILE_CONTENT_COUNT = Config.get().getAppSecMaxFileContentCount();
   public static final int MAX_FILE_CONTENT_BYTES = Config.get().getAppSecMaxFileContentBytes();
 
@@ -38,12 +43,21 @@ public static boolean tryBlock(
     try {
       BlockResponseFunction brf = reqCtx.getBlockResponseFunction();
       if (brf != null) {
-        brf.tryCommitBlockingResponse(reqCtx.getTraceSegment(), rba);
+        // tryCommitAndReport already reports the block failure when the commit fails
+        if (!BlockFailureReporter.tryCommitAndReport(reqCtx, rba)) {
+          return false;
+        }
       } else if (!commitBlocking(fallbackReq, fallbackResp, rba)) {
+        if (fallbackResp != null) {
+          // a commit was genuinely attempted (there was a response to write to) and failed
+          BlockFailureReporter.reportBlockFailure(reqCtx);
+        }
         return false;
       }
-    } catch (Exception ignored) {
-      // commit failed — response not sent, cannot block this request
+    } catch (Exception e) {
+      // commit failed - response not sent, cannot block this request
+      log.debug("Error committing blocking response", e);
+      BlockFailureReporter.reportBlockFailure(reqCtx);
       return false;
     }
     // Response was committed — mark as blocked on a best-effort basis.

dd-java-agent/instrumentation/tomcat/tomcat-appsec/tomcat-appsec-7.0/src/main/java/datadog/trace/instrumentation/tomcat7/GlassFishMultipartInstrumentation.java

@@ -54,6 +54,15 @@ public void methodAdvice(MethodTransformer transformer) {
         getClass().getName() + "$GetPartsAdvice");
   }
 
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {
+      // referenced by GlassFishBlockingHelper.tryBlock() to commit the blocking response
+      "datadog.trace.instrumentation.tomcat.BlockFailureReporter",
+      "datadog.trace.instrumentation.tomcat7.GlassFishBlockingHelper",
+    };
+  }
+
   public static class GetPartsAdvice {
 
     @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)