[AAP] API10 initial implementation

tracer/missing-nullability-files.csv

@@ -223,6 +223,8 @@ src/Datadog.Trace/Agent/Transports/HttpStreamRequest.cs
 src/Datadog.Trace/Agent/Transports/HttpStreamRequestFactory.cs
 src/Datadog.Trace/Agent/Transports/MimeTypes.cs
 src/Datadog.Trace/Agent/Transports/SocketHandlerRequestFactory.cs
+src/Datadog.Trace/AppSec/Rasp/DownstreamSampler.cs
+src/Datadog.Trace/AppSec/Rasp/IDownstreamSampler.cs
 src/Datadog.Trace/AppSec/Waf/WafConstants.cs
 src/Datadog.Trace/AppSec/Waf/WafReturnCode.cs
 src/Datadog.Trace/ClrProfiler/Helpers/Interception.cs

tracer/src/Datadog.Trace/AppSec/AddressesConstants.cs

@@ -14,7 +14,6 @@ internal static class AddressesConstants
         public const string FileAccess = "server.io.fs.file";
         public const string DBStatement = "server.db.statement";
         public const string DBSystem = "server.db.system";
-        public const string UrlAccess = "server.io.net.url";
         public const string ShellInjection = "server.sys.shell.cmd";
         public const string CommandInjection = "server.sys.exec.cmd";
         public const string RequestMethod = "server.request.method";
@@ -32,6 +31,14 @@ internal static class AddressesConstants
         public const string ResponseBody = "server.response.body";
         public const string ResponseHeaderNoCookies = "server.response.headers.no_cookies";
 
+        public const string DownstreamUrl = "server.io.net.url";
+        public const string DownstreamRequestHeaders = "server.io.net.request.headers";
+        public const string DownstreamRequestMethod = "server.io.net.request.method";
+        public const string DownstreamRequestBody = "server.io.net.request.body";
+        public const string DownstreamResponseStatus = "server.io.net.response.status";
+        public const string DownstreamResponseHeaders = "server.io.net.response.headers";
+        public const string DownstreamResponseBody = "server.io.net.response.body";
+
         public const string UserId = "usr.id";
         public const string UserLogin = "usr.login";
         public const string UserSessionId = "usr.session_id";

tracer/src/Datadog.Trace/AppSec/AppSecRequestContext.cs

@@ -1,16 +1,15 @@
-// <copyright file="AppSecRequestContext.cs" company="Datadog">
+// <copyright file="AppSecRequestContext.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
 #nullable enable
 
+using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Threading;
 using Datadog.Trace.AppSec.Rasp;
 using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.Logging;
-using Datadog.Trace.Tagging;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.AppSec;
@@ -25,6 +24,7 @@ internal sealed partial class AppSecRequestContext
     private readonly object _sync = new();
     private readonly RaspMetricsHelper? _raspMetricsHelper = Security.Instance.RaspEnabled ? new RaspMetricsHelper() : null;
     private readonly List<object> _wafSecurityEvents = new();
+    private readonly ConcurrentDictionary<ulong, bool> _sampledHttpClientRequests = new();
     private int _wafTimeout = 0;
     private int? _wafError = null;
     private int? _wafRaspError = null;
@@ -139,6 +139,26 @@ internal void AddStackTrace(string stackCategory, Dictionary<string, object> sta
             _raspStackTraces[stackCategory].Add(stackTrace);
         }
     }
+
+    public bool IsHttpClientRequestSampled(ulong id)
+    {
+        if (_sampledHttpClientRequests.TryGetValue(id, out bool value))
+        {
+            return value;
+        }
+
+        if (Security.Instance.SampleDownstreamRequest(this, id))
+        {
+            if (_sampledHttpClientRequests.Count < Security.Instance.ApiSecurityMaxDownstreamRequestBodyAnalysis)
+            {
+                _sampledHttpClientRequests[id] = true;
+                return true;
+            }
+        }
+
+        _sampledHttpClientRequests[id] = false;
+        return false;
+    }
 }
 
 internal partial class AppSecRequestContext

tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Core.cs

@@ -1,4 +1,4 @@
-// <copyright file="SecurityCoordinator.Core.cs" company="Datadog">
+// <copyright file="SecurityCoordinator.Core.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>

tracer/src/Datadog.Trace/AppSec/Rasp/BodyParser.cs

@@ -0,0 +1,126 @@
+// <copyright file="BodyParser.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Datadog.Trace.Vendors.Newtonsoft.Json;
+
+namespace Datadog.Trace.AppSec.Rasp;
+
+internal static class BodyParser
+{
+    private const int MaxElements = 1000;
+    private const int MaxDepth = 64;
+    private const int MaxStringSize = 1024;
+
+    public static object? Parse(string json)
+    {
+        if (string.IsNullOrEmpty(json))
+        {
+            return null;
+        }
+
+        using var stringReader = new StringReader(json);
+        using var jsonReader = new JsonTextReader(stringReader);
+
+        if (!jsonReader.Read())
+        {
+            return null;
+        }
+
+        State state = new();
+        return ReadValue(jsonReader, ref state, 0);
+    }
+
+    private static object? ReadValue(JsonTextReader r, ref State state, int depth)
+    {
+        if (depth >= MaxDepth)
+        {
+            state.ObjectTooDeep = true;
+            r.Skip();
+            return null;
+        }
+
+        if (state.ElemsLeft-- <= 0)
+        {
+            state.ListMapTooLarge = true;
+            r.Skip();
+            return null;
+        }
+
+        return r.TokenType switch
+        {
+            JsonToken.StartObject => ReadObject(r, ref state, depth),
+            JsonToken.StartArray => ReadArray(r, ref state, depth),
+            JsonToken.String => ReadString(r, ref state),
+            JsonToken.Integer => r.Value != null ? Convert.ToDouble(r.Value) : null,
+            JsonToken.Float => r.Value != null ? Convert.ToDouble(r.Value) : null,
+            JsonToken.Boolean => r.Value,
+            JsonToken.Null => null,
+            _ => null
+        };
+    }
+
+    private static string? ReadString(JsonTextReader r, ref State state)
+    {
+        string? val = r.Value?.ToString();
+        if (val != null && val.Length > MaxStringSize)
+        {
+            state.StringTooLong = true;
+            val = val.Substring(0, MaxStringSize);
+        }
+
+        return val;
+    }
+
+    private static Dictionary<string, object?> ReadObject(JsonTextReader r, ref State state, int depth)
+    {
+        var dict = new Dictionary<string, object?>();
+
+        while (r.Read() && r.TokenType != JsonToken.EndObject)
+        {
+            if (r.TokenType == JsonToken.PropertyName)
+            {
+                string propertyName = r.Value?.ToString() ?? string.Empty;
+
+                if (r.Read())
+                {
+                    object? val = ReadValue(r, ref state, depth + 1);
+                    if (!state.ListMapTooLarge)
+                    {
+                        dict[propertyName] = val;
+                    }
+                }
+            }
+        }
+
+        return dict;
+    }
+
+    private static List<object?> ReadArray(JsonTextReader r, ref State state, int depth)
+    {
+        var list = new List<object?>();
+
+        while (r.Read() && r.TokenType != JsonToken.EndArray)
+        {
+            object? val = ReadValue(r, ref state, depth + 1);
+            if (!state.ListMapTooLarge)
+            {
+                list.Add(val);
+            }
+        }
+
+        return list;
+    }
+
+    public record struct State(
+        int ElemsLeft = MaxElements,
+        bool ObjectTooDeep = false,
+        bool ListMapTooLarge = false,
+        bool StringTooLong = false);
+}

tracer/src/Datadog.Trace/AppSec/Rasp/DownstreamSampler.cs

@@ -0,0 +1,70 @@
+// <copyright file="DownstreamSampler.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Datadog.Trace.AppSec.Rasp;
+
+internal sealed class DownstreamSampler : IDownstreamSampler
+{
+    private const long KnuthFactor = 1111111111111111111L;
+    private readonly double _threshold;
+    private long _globalRequestCount = 0;
+
+    public DownstreamSampler(double rate)
+    {
+        double sanitizedRate = rate < 0.0 ? 0 : (rate > 1.0 ? 1 : rate);
+        _threshold = SamplingCutoff(sanitizedRate);
+    }
+
+    private static double SamplingCutoff(double rate)
+    {
+        const double max = 18446744073709551615.0;
+
+        if (rate < 0.5)
+        {
+            return (long)(rate * max) + long.MinValue;
+        }
+        else if (rate < 1.0)
+        {
+            return (long)((rate * max) + long.MinValue);
+        }
+
+        return long.MaxValue;
+    }
+
+    public bool SampleHttpClientRequest(AppSecRequestContext ctx, ulong requestId)
+    {
+        long counter = UpdateRequestCount();
+
+        unchecked
+        {
+            if ((counter * KnuthFactor) + long.MinValue > _threshold)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private long UpdateRequestCount()
+    {
+        long initial, computed;
+        do
+        {
+            initial = Interlocked.Read(ref _globalRequestCount);
+            computed = (initial == long.MaxValue) ? 0L : initial + 1L;
+        }
+        while (Interlocked.CompareExchange(ref _globalRequestCount, computed, initial) != initial);
+
+        return computed;
+    }
+}

tracer/src/Datadog.Trace/AppSec/Rasp/HttpClient/IHttpContent.cs

@@ -0,0 +1,24 @@
+// <copyright file="IHttpContent.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using System.Threading.Tasks;
+using Datadog.Trace.DuckTyping;
+
+namespace Datadog.Trace.AppSec.Rasp.HttpClient;
+
+/// <summary>
+/// HttpResponseMessage interface for ducktyping
+/// </summary>
+internal interface IHttpContent : IDuckType
+{
+    IHttpContentHeaders Headers { get; }
+
+    bool TryComputeLength(out long length);
+
+    Task LoadIntoBufferAsync();
+
+    Task<string> ReadAsStringAsync();
+}

tracer/src/Datadog.Trace/AppSec/Rasp/HttpClient/IHttpContentHeaders.cs

@@ -0,0 +1,17 @@
+// <copyright file="IHttpContentHeaders.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using Datadog.Trace.ClrProfiler.AutoInstrumentation.Http.HttpClient;
+
+namespace Datadog.Trace.AppSec.Rasp.HttpClient;
+
+/// <summary>
+/// HttpContentHeaders interface for ducktyping
+/// </summary>
+internal interface IHttpContentHeaders : IRequestHeaders
+{
+    IMediaTypeHeaderValue ContentType { get; }
+}

tracer/src/Datadog.Trace/AppSec/Rasp/HttpClient/IHttpHeaders.cs

@@ -0,0 +1,17 @@
+// <copyright file="IHttpHeaders.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using System.Collections.Generic;
+
+namespace Datadog.Trace.AppSec.Rasp.HttpClient;
+
+/// <summary>
+/// HttpHeaders interface for ducktyping
+/// </summary>
+internal interface IHttpHeaders
+{
+    IEnumerator<KeyValuePair<string, IEnumerable<string>>> GetEnumerator();
+}

tracer/src/Datadog.Trace/AppSec/Rasp/HttpClient/IHttpRequestMessage.cs

@@ -0,0 +1,25 @@
+// <copyright file="IHttpRequestMessage.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using System;
+using Datadog.Trace.ClrProfiler.AutoInstrumentation.Http.HttpClient;
+using Datadog.Trace.DuckTyping;
+
+namespace Datadog.Trace.AppSec.Rasp.HttpClient;
+
+/// <summary>
+/// HttpRequestMessage interface for ducktyping
+/// </summary>
+internal interface IHttpRequestMessage : IDuckType
+{
+    HttpMethodStruct Method { get; }
+
+    Uri RequestUri { get; }
+
+    IHttpHeaders Headers { get; }
+
+    IHttpContent Content { get; }
+}