feat(redaction): add configurable sensitive data redaction

Adds configurable sensitive data redaction support for captured requests and responses.

Author: georgidhristov

DebugProbe.AspNetCore.Tests/Configuration/DebugProbeOptionsTests.cs

@@ -17,6 +17,10 @@ public void Defaults_work_correctly()
         Assert.Null(options.AllowLocalCompareTargets);
         Assert.False(options.AllowUiInProduction);
         Assert.Empty(options.IgnorePaths);
+        Assert.Equal(["Authorization", "Cookie", "Set-Cookie"], options.RedactedHeaders);
+        Assert.Empty(options.RedactedQueryParameters);
+        Assert.Empty(options.RedactedJsonFields);
+        Assert.Equal("[REDACTED]", options.RedactionText);
     }
 
     [Fact]
@@ -30,6 +34,10 @@ public void Custom_options_are_registered_and_used()
             options.MaxBodyCaptureSizeKb = 4;
             options.AllowLocalCompareTargets = true;
             options.IgnorePaths = ["/health"];
+            options.RedactedHeaders = ["X-Api-Key"];
+            options.RedactedQueryParameters = ["token"];
+            options.RedactedJsonFields = ["password"];
+            options.RedactionText = "***";
         });
 
         using var provider = services.BuildServiceProvider();
@@ -40,6 +48,10 @@ public void Custom_options_are_registered_and_used()
         Assert.Equal(4, options.MaxBodyCaptureSizeKb);
         Assert.True(options.AllowLocalCompareTargets);
         Assert.Equal(["/health"], options.IgnorePaths);
+        Assert.Equal(["X-Api-Key"], options.RedactedHeaders);
+        Assert.Equal(["token"], options.RedactedQueryParameters);
+        Assert.Equal(["password"], options.RedactedJsonFields);
+        Assert.Equal("***", options.RedactionText);
         Assert.NotNull(store.Environment);
     }
 }

DebugProbe.AspNetCore.Tests/Handlers/DebugProbeHttpClientHandlerTests.cs

@@ -52,6 +52,49 @@ public async Task Captures_outgoing_http_call_on_active_trace()
         Assert.Contains("\"ok\": true", outgoing.ResponseBody);
     }
 
+    [Fact]
+    public async Task Redacts_configured_outgoing_url_headers_and_json_fields()
+    {
+        var entry = new DebugEntry();
+        var context = new DefaultHttpContext();
+        context.Items["DebugProbeEntry"] = entry;
+
+        using var handler = new DebugProbeHttpClientHandler(
+            new HttpContextAccessor { HttpContext = context },
+            new DebugProbeOptions
+            {
+                RedactedHeaders = ["Authorization", "X-Api-Key"],
+                RedactedQueryParameters = ["token"],
+                RedactedJsonFields = ["password", "refreshToken"]
+            })
+        {
+            InnerHandler = new StubHandler(_ =>
+            {
+                var response = new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new StringContent("{\"refreshToken\":\"response-token\"}", Encoding.UTF8, "application/json")
+                };
+                return response;
+            })
+        };
+
+        using var client = new HttpClient(handler);
+        using var request = new HttpRequestMessage(HttpMethod.Post, "https://api.example.test/orders?token=query-secret&safe=yes")
+        {
+            Content = new StringContent("{\"password\":\"body-secret\"}", Encoding.UTF8, "application/json")
+        };
+        request.Headers.Add("X-Api-Key", "header-secret");
+
+        await client.SendAsync(request);
+
+        var outgoing = Assert.Single(entry.OutgoingRequests);
+
+        Assert.Equal("https://api.example.test/orders?token=[REDACTED]&safe=yes", outgoing.Url);
+        Assert.Equal("[REDACTED]", outgoing.RequestHeaders["X-Api-Key"]);
+        Assert.Contains("\"password\": \"[REDACTED]\"", outgoing.RequestBody);
+        Assert.Contains("\"refreshToken\": \"[REDACTED]\"", outgoing.ResponseBody);
+    }
+
     private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> send) : HttpMessageHandler
     {
         protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)

DebugProbe.AspNetCore.Tests/Middleware/RedactionTests.cs

@@ -0,0 +1,75 @@
+using System.Text;
+using DebugProbe.AspNetCore.Options;
+using DebugProbe.AspNetCore.Tests.Infrastructure;
+
+namespace DebugProbe.AspNetCore.Tests.Middleware;
+
+public class RedactionTests
+{
+    [Fact]
+    public async Task Redacts_configured_headers_query_parameters_and_json_fields()
+    {
+        await using var app = await DebugProbeTestApp.CreateAsync(
+            endpoints => endpoints.MapPost("/orders", async context =>
+            {
+                context.Response.Headers["X-Session-Token"] = "response-secret";
+                context.Response.ContentType = "application/json";
+                await context.Response.WriteAsync("{\"ok\":true,\"refreshToken\":\"response-token\"}");
+            }),
+            ConfigureRedaction);
+
+        using var request = new HttpRequestMessage(HttpMethod.Post, "/orders?api_key=query-secret&safe=yes")
+        {
+            Content = new StringContent(
+                "{\"name\":\"Ada\",\"password\":\"secret\",\"profile\":{\"refreshToken\":\"nested-token\"}}",
+                Encoding.UTF8,
+                "application/json")
+        };
+        request.Headers.Add("X-Api-Key", "header-secret");
+
+        await app.Client.SendAsync(request);
+
+        var entry = app.SingleEntry;
+
+        Assert.Equal("[REDACTED]", entry.RequestHeaders["X-Api-Key"]);
+        Assert.Equal("[REDACTED]", entry.ResponseHeaders["X-Session-Token"]);
+        Assert.Equal("?api_key=[REDACTED]&safe=yes", entry.Query);
+        Assert.Equal("http://localhost/orders?api_key=[REDACTED]&safe=yes", entry.RequestUrl);
+        Assert.Contains("\"password\":\"[REDACTED]\"", entry.RequestBody);
+        Assert.Contains("\"refreshToken\":\"[REDACTED]\"", entry.RequestBody);
+        Assert.Contains("\"refreshToken\":\"[REDACTED]\"", entry.ResponseBody);
+        Assert.DoesNotContain("secret", entry.RequestBody);
+        Assert.DoesNotContain("response-token", entry.ResponseBody);
+    }
+
+    [Fact]
+    public async Task Leaves_invalid_json_body_unchanged_when_json_fields_are_configured()
+    {
+        await using var app = await DebugProbeTestApp.CreateAsync(
+            endpoints => endpoints.MapPost("/text", async context =>
+            {
+                context.Response.ContentType = "text/plain";
+                await context.Response.WriteAsync("ok");
+            }),
+            options => options.RedactedJsonFields = ["password"]);
+
+        await app.Client.PostAsync(
+            "/text",
+            new StringContent("password=secret", Encoding.UTF8, "text/plain"));
+
+        Assert.Equal("password=secret", app.SingleEntry.RequestBody);
+    }
+
+    private static void ConfigureRedaction(DebugProbeOptions options)
+    {
+        options.RedactedHeaders =
+        [
+            ..options.RedactedHeaders,
+            "X-Api-Key",
+            "X-Session-Token"
+        ];
+
+        options.RedactedQueryParameters = ["api_key"];
+        options.RedactedJsonFields = ["password", "refreshToken"];
+    }
+}

DebugProbe.AspNetCore/Handlers/DebugProbeHttpClientHandler.cs

@@ -71,7 +71,7 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
         {
             Method = request.Method.Method,
 
-            Url = request.RequestUri?.ToString() ?? string.Empty,
+            Url = RedactionUtils.RedactUrl(request.RequestUri?.ToString(), _options),
 
             StatusCode = response != null ? (int)response.StatusCode : null,
 
@@ -83,9 +83,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
 
             IsSuccessStatusCode = response?.IsSuccessStatusCode ?? false,
 
-            RequestHeaders = request.Headers.ToDictionary(x => x.Key, x => HeaderUtils.RedactIfSensitive(x.Key, string.Join(", ", x.Value))),
+            RequestHeaders = request.Headers.ToDictionary(x => x.Key, x => RedactionUtils.RedactHeader(x.Key, string.Join(", ", x.Value), _options)),
 
-            ResponseHeaders = response != null ? response.Headers.ToDictionary(x => x.Key, x => HeaderUtils.RedactIfSensitive(x.Key, string.Join(", ", x.Value))) : []
+            ResponseHeaders = response != null ? response.Headers.ToDictionary(x => x.Key, x => RedactionUtils.RedactHeader(x.Key, string.Join(", ", x.Value), _options)) : []
         };
 
         if (request.Content != null)
@@ -96,7 +96,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
             {
                 var body = await request.Content.ReadAsStringAsync();
 
-                outgoing.RequestBody = JsonUtils.Format(HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes));
+                outgoing.RequestBody = JsonUtils.Format(RedactionUtils.RedactJsonFields(
+                    HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes),
+                    _options));
             }
         }
 
@@ -108,7 +110,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
             {
                 var body = await response.Content.ReadAsStringAsync();
 
-                outgoing.ResponseBody = JsonUtils.Format(HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes));
+                outgoing.ResponseBody = JsonUtils.Format(RedactionUtils.RedactJsonFields(
+                    HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes),
+                    _options));
             }
         }
 

DebugProbe.AspNetCore/Internal/Utils/RedactionUtils.cs

@@ -0,0 +1,150 @@
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using DebugProbe.AspNetCore.Options;
+
+namespace DebugProbe.AspNetCore.Internal.Utils;
+
+internal static class RedactionUtils
+{
+    public static string RedactHeader(string name, string value, DebugProbeOptions options)
+    {
+        return IsMatch(name, options.RedactedHeaders) ? options.RedactionText : value;
+    }
+
+    public static string RedactQueryString(string? queryString, DebugProbeOptions options)
+    {
+        if (string.IsNullOrEmpty(queryString) || options.RedactedQueryParameters.Length == 0)
+        {
+            return queryString ?? string.Empty;
+        }
+
+        var prefix = queryString.StartsWith('?') ? "?" : string.Empty;
+        var query = prefix.Length == 0 ? queryString : queryString[1..];
+
+        return prefix + RedactQuery(query, options);
+    }
+
+    public static string RedactUrl(string? url, DebugProbeOptions options)
+    {
+        if (string.IsNullOrEmpty(url) || options.RedactedQueryParameters.Length == 0)
+        {
+            return url ?? string.Empty;
+        }
+
+        var fragmentIndex = url.IndexOf('#');
+        var fragment = fragmentIndex >= 0 ? url[fragmentIndex..] : string.Empty;
+        var withoutFragment = fragmentIndex >= 0 ? url[..fragmentIndex] : url;
+
+        var queryIndex = withoutFragment.IndexOf('?');
+        if (queryIndex < 0)
+        {
+            return url;
+        }
+
+        var beforeQuery = withoutFragment[..queryIndex];
+        var query = withoutFragment[(queryIndex + 1)..];
+
+        return $"{beforeQuery}?{RedactQuery(query, options)}{fragment}";
+    }
+
+    public static string RedactJsonFields(string? body, DebugProbeOptions options)
+    {
+        if (string.IsNullOrWhiteSpace(body) || options.RedactedJsonFields.Length == 0)
+        {
+            return body ?? string.Empty;
+        }
+
+        try
+        {
+            var node = JsonNode.Parse(body);
+            if (node is null)
+            {
+                return body;
+            }
+
+            RedactNode(node, options);
+
+            return JsonSerializer.Serialize(
+                node,
+                new JsonSerializerOptions
+                {
+                    Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+                });
+        }
+        catch
+        {
+            return body;
+        }
+    }
+
+    private static string RedactQuery(string query, DebugProbeOptions options)
+    {
+        if (query.Length == 0)
+        {
+            return query;
+        }
+
+        var parts = query.Split('&');
+
+        for (var i = 0; i < parts.Length; i++)
+        {
+            var part = parts[i];
+            var equalsIndex = part.IndexOf('=');
+            var name = equalsIndex >= 0 ? part[..equalsIndex] : part;
+
+            if (!IsMatch(DecodeQueryValue(name), options.RedactedQueryParameters))
+            {
+                continue;
+            }
+
+            parts[i] = equalsIndex >= 0? $"{name}={options.RedactionText}" : $"{name}={options.RedactionText}";
+        }
+
+        return string.Join("&", parts);
+    }
+
+    private static void RedactNode(JsonNode node, DebugProbeOptions options)
+    {
+        if (node is JsonObject jsonObject)
+        {
+            foreach (var property in jsonObject.ToList())
+            {
+                if (IsMatch(property.Key, options.RedactedJsonFields))
+                {
+                    jsonObject[property.Key] = options.RedactionText;
+                    continue;
+                }
+
+                if (property.Value is not null)
+                {
+                    RedactNode(property.Value, options);
+                }
+            }
+
+            return;
+        }
+
+        if (node is JsonArray jsonArray)
+        {
+            foreach (var item in jsonArray)
+            {
+                if (item is not null)
+                {
+                    RedactNode(item, options);
+                }
+            }
+        }
+    }
+
+    private static string DecodeQueryValue(string value)
+    {
+        return Uri.UnescapeDataString(value.Replace("+", " "));
+    }
+
+    private static bool IsMatch(string value, string[] candidates)
+    {
+        return candidates.Any(candidate =>
+            string.Equals(value, candidate, StringComparison.OrdinalIgnoreCase));
+    }
+}

DebugProbe.AspNetCore/Middleware/DebugProbeMiddleware.cs

@@ -118,7 +118,7 @@ public async Task Invoke(HttpContext context, DebugEntryStore store)
 
             entry.Path = context.Request.Path;
 
-            entry.Query = context.Request.QueryString.ToString();
+            entry.Query = RedactionUtils.RedactQueryString(context.Request.QueryString.ToString(), _options);
 
             entry.StatusCode = statusCode;
 
@@ -133,20 +133,26 @@ public async Task Invoke(HttpContext context, DebugEntryStore store)
             entry.RequestHeaders =
                 context.Request.Headers.ToDictionary(
                     x => x.Key,
-                    x => HeaderUtils.RedactIfSensitive(x.Key, x.Value.ToString()));
+                    x => RedactionUtils.RedactHeader(x.Key, x.Value.ToString(), _options));
 
             entry.RequestUrl =
-                $"{context.Request.Scheme}://{context.Request.Host}" +
-                $"{context.Request.Path}{context.Request.QueryString}";
+                RedactionUtils.RedactUrl(
+                    $"{context.Request.Scheme}://{context.Request.Host}" +
+                    $"{context.Request.Path}{context.Request.QueryString}",
+                    _options);
 
-            entry.RequestBody = HttpContentUtils.Trim(requestBody, maxBodySize);
+            entry.RequestBody = RedactionUtils.RedactJsonFields(
+                HttpContentUtils.Trim(requestBody, maxBodySize),
+                _options);
 
-            entry.ResponseBody = HttpContentUtils.Trim(responseBody, maxBodySize);
+            entry.ResponseBody = RedactionUtils.RedactJsonFields(
+                HttpContentUtils.Trim(responseBody, maxBodySize),
+                _options);
 
             entry.ResponseHeaders =
                 context.Response.Headers.ToDictionary(
                     x => x.Key,
-                    x => HeaderUtils.RedactIfSensitive(x.Key, x.Value.ToString()));
+                    x => RedactionUtils.RedactHeader(x.Key, x.Value.ToString(), _options));
 
             store.Add(entry);
         }