Collect parsed request body if RASP event

improve truncation

wip

wip - not working

wip - fix

Author: jandro996

dd-java-agent/appsec/src/main/java/com/datadog/appsec/ddwaf/WAFModule.java

@@ -480,6 +480,7 @@ public void onDataAvailable(
         }
 
         if (gwCtx.isRasp) {
+          reqCtx.setRaspMatched(true);
           WafMetricCollector.get().raspRuleMatch(gwCtx.raspRuleType);
         }
 

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java

@@ -129,6 +129,9 @@ public class AppSecRequestContext implements DataBundle, Closeable {
   private volatile int wafTimeouts;
   private volatile int raspTimeouts;
 
+  private volatile Object processedRequestBody;
+  private volatile boolean raspMatched;
+
   // keep a reference to the last published usr.id
   private volatile String userId;
   // keep a reference to the last published usr.login
@@ -675,4 +678,20 @@ public boolean isWafContextClosed() {
   void setRequestEndCalled() {
     requestEndCalled = true;
   }
+
+  public void setProcessedRequestBody(Object processedRequestBody) {
+    this.processedRequestBody = processedRequestBody;
+  }
+
+  public Object getProcessedRequestBody() {
+    return processedRequestBody;
+  }
+
+  public boolean isRaspMatched() {
+    return raspMatched;
+  }
+
+  public void setRaspMatched(boolean raspMatched) {
+    this.raspMatched = raspMatched;
+  }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java

@@ -77,6 +77,7 @@ public class GatewayBridge {
   private static final String USER_COLLECTION_MODE_TAG = "_dd.appsec.user.collection_mode";
 
   private static final Map<LoginEvent, Address<?>> EVENT_MAPPINGS = new EnumMap<>(LoginEvent.class);
+  private static final String METASTRUCT_REQUEST_BODY = "http.request.body";
 
   static {
     EVENT_MAPPINGS.put(LoginEvent.LOGIN_SUCCESS, KnownAddresses.LOGIN_SUCCESS);
@@ -472,7 +473,7 @@ private Flow<Void> onGrpcServerRequestMessage(RequestContext ctx_, Object obj) {
       if (subInfo == null || subInfo.isEmpty()) {
         return NoopFlow.INSTANCE;
       }
-      Object convObj = ObjectIntrospection.convert(obj, ctx);
+      Object convObj = ObjectIntrospection.convert(obj, ctx).getValue();
       DataBundle bundle =
           new SingletonDataBundle<>(KnownAddresses.GRPC_SERVER_REQUEST_MESSAGE, convObj);
       try {
@@ -572,9 +573,16 @@ private Flow<Void> onRequestBodyProcessed(RequestContext ctx_, Object obj) {
       if (subInfo == null || subInfo.isEmpty()) {
         return NoopFlow.INSTANCE;
       }
+      ObjectIntrospection.ConversionResult<Object> converted =
+          ObjectIntrospection.convert(obj, ctx);
+      if (Config.get().isAppSecRaspCollectRequestBody()) {
+        ctx.setProcessedRequestBody(converted.getValue());
+        if (converted.isAnyTruncated()) {
+          ctx_.getTraceSegment().setTagTop("_dd.appsec.rasp.request_body_size.exceeded", true);
+        }
+      }
       DataBundle bundle =
-          new SingletonDataBundle<>(
-              KnownAddresses.REQUEST_BODY_OBJECT, ObjectIntrospection.convert(obj, ctx));
+          new SingletonDataBundle<>(KnownAddresses.REQUEST_BODY_OBJECT, converted.getValue());
       try {
         GatewayContext gwCtx = new GatewayContext(false);
         return producerService.publishDataEvent(subInfo, ctx, bundle, gwCtx);
@@ -723,6 +731,12 @@ private NoopFlow onRequestEnded(RequestContext ctx_, IGSpanInfo spanInfo) {
           StackUtils.addStacktraceEventsToMetaStruct(ctx_, METASTRUCT_EXPLOIT, stackTraces);
         }
 
+        // Report collected parsed request body if there is a RASP event
+        if (ctx.isRaspMatched() && ctx.getProcessedRequestBody() != null) {
+          ctx_.getOrCreateMetaStructTop(
+              METASTRUCT_REQUEST_BODY, k -> ctx.getProcessedRequestBody());
+        }
+
       } else if (hasUserInfo(traceSeg)) {
         // Report all collected request headers on user tracking event
         writeRequestHeaders(traceSeg, REQUEST_HEADERS_ALLOW_LIST, ctx.getRequestHeaders(), false);

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/event/data/ObjectIntrospectionSpecification.groovy

@@ -28,12 +28,12 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
   void 'null is preserved'() {
     expect:
-    convert(null, ctx) == null
+    convert(null, ctx).getValue() == null
   }
 
   void 'type #type is preserved'() {
     when:
-    def result = convert(input, ctx)
+    def result = convert(input, ctx).getValue()
 
     then:
     input.getClass() == type
@@ -56,7 +56,7 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
   void 'type #type is converted to string'() {
     when:
-    def result = convert(input, ctx)
+    def result = convert(input, ctx).getValue()
 
     then:
     type.isAssignableFrom(input.getClass())
@@ -83,9 +83,9 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     }
 
     expect:
-    convert(iter, ctx) instanceof List
-    convert(iter, ctx) == ['a', 'b']
-    convert(['a', 'b'], ctx) == ['a', 'b']
+    convert(iter, ctx).getValue() instanceof List
+    convert(iter, ctx) .getValue()== ['a', 'b']
+    convert(['a', 'b'], ctx).getValue() == ['a', 'b']
   }
 
   void 'maps are converted to hash maps'() {
@@ -95,21 +95,21 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     }
 
     expect:
-    convert(map, ctx) instanceof HashMap
-    convert(map, ctx) == [a: 'b']
-    convert([(6): 'b'], ctx) == ['6': 'b']
-    convert([(null): 'b'], ctx) == ['null': 'b']
-    convert([(true): 'b'], ctx) == ['true': 'b']
-    convert([('a' as Character): 'b'], ctx) == ['a': 'b']
-    convert([(createCharBuffer('a')): 'b'], ctx) == ['a': 'b']
+    convert(map, ctx).getValue() instanceof HashMap
+    convert(map, ctx).getValue() == [a: 'b']
+    convert([(6): 'b'], ctx).getValue() == ['6': 'b']
+    convert([(null): 'b'], ctx).getValue() == ['null': 'b']
+    convert([(true): 'b'], ctx).getValue() == ['true': 'b']
+    convert([('a' as Character): 'b'], ctx).getValue() == ['a': 'b']
+    convert([(createCharBuffer('a')): 'b'], ctx).getValue() == ['a': 'b']
   }
 
   void 'arrays are converted into lists'() {
     expect:
-    convert([6, 'b'] as Object[], ctx) == [6, 'b']
-    convert([null, null] as Object[], ctx) == [null, null]
-    convert([1, 2] as int[], ctx) == [1 as int, 2 as int]
-    convert([1, 2] as byte[], ctx) == [1 as byte, 2 as byte]
+    convert([6, 'b'] as Object[], ctx).getValue() == [6, 'b']
+    convert([null, null] as Object[], ctx).getValue() == [null, null]
+    convert([1, 2] as int[], ctx).getValue() == [1 as int, 2 as int]
+    convert([1, 2] as byte[], ctx).getValue() == [1 as byte, 2 as byte]
   }
 
   @SuppressWarnings('UnusedPrivateField')
@@ -127,8 +127,8 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
   void 'other objects are converted into hash maps'() {
     expect:
-    convert(new ClassToBeConverted(), ctx) instanceof HashMap
-    convert(new ClassToBeConvertedExt(), ctx) == [c: 'd', a: 'b', l: [1, 2]]
+    convert(new ClassToBeConverted(), ctx).getValue() instanceof HashMap
+    convert(new ClassToBeConvertedExt(), ctx) .getValue()== [c: 'd', a: 'b', l: [1, 2]]
   }
 
   class ProtobufLikeClass {
@@ -139,15 +139,15 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
   void 'some field names are ignored'() {
     expect:
-    convert(new ProtobufLikeClass(), ctx) instanceof HashMap
-    convert(new ProtobufLikeClass(), ctx) == [c: 'd']
+    convert(new ProtobufLikeClass(), ctx).getValue() instanceof HashMap
+    convert(new ProtobufLikeClass(), ctx).getValue() == [c: 'd']
   }
 
   void 'invalid keys are converted to special strings'() {
     expect:
-    convert(Collections.singletonMap(new ClassToBeConverted(), 'a'), ctx) == ['invalid_key:1': 'a']
-    convert([new ClassToBeConverted(): 'a', new ClassToBeConverted(): 'b'], ctx) == ['invalid_key:1': 'a', 'invalid_key:2': 'b']
-    convert(Collections.singletonMap([1, 2], 'a'), ctx) == ['invalid_key:1': 'a']
+    convert(Collections.singletonMap(new ClassToBeConverted(), 'a'), ctx).getValue() == ['invalid_key:1': 'a']
+    convert([new ClassToBeConverted(): 'a', new ClassToBeConverted(): 'b'], ctx).getValue() == ['invalid_key:1': 'a', 'invalid_key:2': 'b']
+    convert(Collections.singletonMap([1, 2], 'a'), ctx).getValue() == ['invalid_key:1': 'a']
   }
 
   void 'max number of elements is honored'() {
@@ -156,16 +156,28 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     128.times { m[it] = 'b' }
 
     when:
-    def result1 = convert([['a'] * 255], ctx)[0]
-    def result2 = convert([['a'] * 255 as String[]], ctx)[0]
+    def result1 = convert([['a'] * 255], ctx)
+    def result2 = convert([['a'] * 255 as String[]], ctx)
     def result3 = convert(m, ctx)
 
     then:
-    result1.size() == 254 // +2 for the lists
-    result2.size() == 254 // +2 for the lists
-    result3.size() == 127  // +1 for the map, 2 for each entry (key and value)
+    result1.getValue()[0].size() == 254 // +2 for the lists
+    result1.isAnyTruncated()
+    result1.isCollectionTruncated()
+    !result1.isDepthTruncated()
+    !result1.isStringTruncated()
+    result2.getValue()[0].size() == 254 // +2 for the lists
+    result2.isAnyTruncated()
+    result2.isCollectionTruncated()
+    !result2.isDepthTruncated()
+    !result2.isStringTruncated()
+    result3.getValue().size() == 127// +1 for the map, 2 for each entry (key and value)
+    !result3.isAnyTruncated()
+    !result3.isCollectionTruncated()
+    !result3.isDepthTruncated()
+    !result3.isStringTruncated()
     2 * ctx.setWafTruncated()
-    2 * wafMetricCollector.wafInputTruncated(false, true, false)
+    //2 * wafMetricCollector.wafInputTruncated(false, true, false)
 
   }
 
@@ -179,18 +191,23 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     when:
     // Invoke conversion with context
     def result = convert(objArray, ctx)
+    def converted = result.getValue()
 
     then:
     // Traverse converted arrays to count actual depth
     int depth = 0
-    for (p = result; p != null; p = p[0]) {
+    for (p = converted; p != null; p = p[0]) {
       depth++
     }
     depth == 21 // after max depth we have nulls
 
     // Should record a truncation due to depth
     1 * ctx.setWafTruncated()
-    1 * wafMetricCollector.wafInputTruncated(false, false, true)
+    result.isDepthTruncated()
+    !result.isCollectionTruncated()
+    result.isAnyTruncated()
+    !result.isStringTruncated()
+    //1 * wafMetricCollector.wafInputTruncated(false, false, true)
   }
 
   void 'max depth is honored — list version'() {
@@ -203,18 +220,23 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     when:
     // Invoke conversion with context
     def result = convert(list, ctx)
+    def converted = result.getValue()
 
     then:
     // Traverse converted lists to count actual depth
     int depth = 0
-    for (p = result; p != null; p = p[0]) {
+    for (p = converted; p != null; p = p[0]) {
       depth++
     }
     depth == 21 // after max depth we have nulls
 
     // Should record a truncation due to depth
     1 * ctx.setWafTruncated()
-    1 * wafMetricCollector.wafInputTruncated(false, false, true)
+    result.isDepthTruncated()
+    !result.isCollectionTruncated()
+    result.isAnyTruncated()
+    !result.isStringTruncated()
+    //1 * wafMetricCollector.wafInputTruncated(false, false, true)
   }
 
   def 'max depth is honored — map version'() {
@@ -227,26 +249,32 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     when:
     // Invoke conversion with context
     def result = convert(map, ctx)
+    def converted = result.getValue()
 
     then:
     // Traverse converted maps to count actual depth
     int depth = 0
-    for (p = result; p != null; p = p['a']) {
+    for (p = converted; p != null; p = p['a']) {
       depth++
     }
     depth == 21 // after max depth we have nulls
 
     // Should record a truncation due to depth
     1 * ctx.setWafTruncated()
-    1 * wafMetricCollector.wafInputTruncated(false, false, true)
+    result.isDepthTruncated()
+    !result.isCollectionTruncated()
+    result.isAnyTruncated()
+    !result.isStringTruncated()
+    //1 * wafMetricCollector.wafInputTruncated(false, false, true)
   }
 
   void 'truncate long #typeName to 4096 chars and set truncation flag'() {
     setup:
     def longInput = rawInput
 
     when:
-    def converted   = convert(longInput, ctx)
+    def result = convert(longInput, ctx)
+    def converted   = result.getValue()
 
     then:
     // Should always produce a String of exactly 4096 chars
@@ -255,7 +283,11 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
     // Should record a truncation due to string length
     1 * ctx.setWafTruncated()
-    1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    //1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    result.isStringTruncated()
+    !result.isDepthTruncated()
+    !result.isCollectionTruncated()
+    result.isAnyTruncated()
 
     where:
     typeName        | rawInput
@@ -271,7 +303,8 @@ class ObjectIntrospectionSpecification extends DDSpecification {
 
     when:
     // convert returns Pair<convertedObject, wasTruncated>
-    def converted   = convert(inputMap, ctx)
+    def result = convert(inputMap, ctx)
+    def converted   = result.getValue()
 
     then:
     // Extract the single truncated key
@@ -281,7 +314,11 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     truncatedKey.length() == 4096
 
     1 * ctx.setWafTruncated()
-    1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    //1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    result.isStringTruncated()
+    !result.isDepthTruncated()
+    !result.isCollectionTruncated()
+    result.isAnyTruncated()
 
 
     where:
@@ -302,6 +339,7 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     }
 
     expect:
-    convert([cs], ctx) == ['error:my exception']
+    convert([cs], ctx).getValue() == ['error:my exception']
   }
 }
+