Adds UrlDiscoveryPlugin and preset

dev-proxy-plugins/Reporters/JsonReporter.cs

@@ -1,65 +1,100 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-using System.Text.Json;
-using Microsoft.DevProxy.Abstractions;
-using Microsoft.DevProxy.Plugins.RequestLogs;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-
-namespace Microsoft.DevProxy.Plugins.Reporters;
-
-public class JsonReporter(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet<UrlToWatch> urlsToWatch, IConfigurationSection? configSection = null) : BaseReporter(pluginEvents, context, logger, urlsToWatch, configSection)
-{
-    public override string Name => nameof(JsonReporter);
-    public override string FileExtension => ".json";
-
-    private readonly Dictionary<Type, Func<object, object>> _transformers = new()
-    {
-        { typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummary },
-        { typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummary },
-    };
-
-    protected override string GetReport(KeyValuePair<string, object> report)
-    {
-        Logger.LogDebug("Serializing report {reportKey}...", report.Key);
-
-        var reportData = report.Value;
-        var reportType = reportData.GetType();
-
-        if (_transformers.TryGetValue(reportType, out var transform))
-        {
-            Logger.LogDebug("Transforming {reportType} using {transform}...", reportType.Name, transform.Method.Name);
-            reportData = transform(reportData);
-        }
-        else
-        {
-            Logger.LogDebug("No transformer found for {reportType}", reportType.Name);
-        }
-
-        if (reportData is string strVal)
-        {
-            Logger.LogDebug("{reportKey} is a string. Checking if it's JSON...", report.Key);
-
-            try
-            {
-                JsonSerializer.Deserialize<object>(strVal);
-                Logger.LogDebug("{reportKey} is already JSON, ignore", report.Key);
-                // already JSON, ignore
-                return strVal;
-            }
-            catch
-            {
-                Logger.LogDebug("{reportKey} is not JSON, serializing...", report.Key);
-            }
-        }
-
-        return JsonSerializer.Serialize(reportData, ProxyUtils.JsonSerializerOptions);
-    }
-
-    private static object TransformExecutionSummary(object report)
-    {
-        var executionSummaryReport = (ExecutionSummaryPluginReportBase)report;
-        return executionSummaryReport.Data;
-    }
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Text;
+using System.Text.Json;
+using Microsoft.DevProxy.Abstractions;
+using Microsoft.DevProxy.Plugins.RequestLogs;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.DevProxy.Plugins.Reporters;
+
+public class JsonReporter(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet<UrlToWatch> urlsToWatch, IConfigurationSection? configSection = null) : BaseReporter(pluginEvents, context, logger, urlsToWatch, configSection)
+{
+    public override string Name => nameof(JsonReporter);
+    private string _fileExtension = ".json";
+    public override string FileExtension => _fileExtension;
+
+    private readonly Dictionary<Type, Func<object, object>> _transformers = new()
+    {
+        { typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummary },
+        { typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummary },
+        { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport }
+    };
+
+    protected override string GetReport(KeyValuePair<string, object> report)
+    {
+        Logger.LogDebug("Serializing report {reportKey}...", report.Key);
+
+        var reportData = report.Value;
+        var reportType = reportData.GetType();
+        _fileExtension = reportType.Name == nameof(UrlDiscoveryPluginReport) ? ".jsonc" : ".json";
+
+        if (_transformers.TryGetValue(reportType, out var transform))
+        {
+            Logger.LogDebug("Transforming {reportType} using {transform}...", reportType.Name, transform.Method.Name);
+            reportData = transform(reportData);
+        }
+        else
+        {
+            Logger.LogDebug("No transformer found for {reportType}", reportType.Name);
+        }
+
+        if (reportData is string strVal)
+        {
+            Logger.LogDebug("{reportKey} is a string. Checking if it's JSON...", report.Key);
+
+            try
+            {
+                JsonSerializer.Deserialize<object>(strVal, ProxyUtils.JsonSerializerOptions);
+                Logger.LogDebug("{reportKey} is already JSON, ignore", report.Key);
+                // already JSON, ignore
+                return strVal;
+            }
+            catch
+            {
+                Logger.LogDebug("{reportKey} is not JSON, serializing...", report.Key);
+            }
+        }
+
+        return JsonSerializer.Serialize(reportData, ProxyUtils.JsonSerializerOptions);
+    }
+
+    private static object TransformExecutionSummary(object report)
+    {
+        var executionSummaryReport = (ExecutionSummaryPluginReportBase)report;
+        return executionSummaryReport.Data;
+    }
+
+    private static object TransformUrlDiscoveryReport(object report)
+    {
+        var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report;
+
+        var sb = new StringBuilder();
+        sb.AppendLine("{");
+        sb.AppendLine("  // Wildcards");
+        sb.AppendLine("  // ");
+        sb.AppendLine("  // You can use wildcards to catch multiple URLs with the same pattern.");
+        sb.AppendLine("  // For example, you can use the following URL pattern to catch all API requests to");
+        sb.AppendLine("  // JSON Placeholder API:");
+        sb.AppendLine("  // ");
+        sb.AppendLine("  // https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("  // ");
+        sb.AppendLine("  // Excluding URLs");
+        sb.AppendLine("  // ");
+        sb.AppendLine("  // You can exclude URLs with ! to prevent them from being intercepted.");
+        sb.AppendLine("  // For example, you can exclude the URL https://jsonplaceholder.typicode.com/authors");
+        sb.AppendLine("  // by using the following URL pattern:");
+        sb.AppendLine("  // ");
+        sb.AppendLine("  // !https://jsonplaceholder.typicode.com/authors");
+        sb.AppendLine("  // https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("  \"urlsToWatch\": [");
+        sb.AppendJoin($",{Environment.NewLine}", urlDiscoveryPluginReport.Data.Select(u => $"    \"{u}\""));
+        sb.AppendLine("");
+        sb.AppendLine("  ]");
+        sb.AppendLine("}");
+
+        return sb.ToString();
+    }
 }

dev-proxy-plugins/Reporters/MarkdownReporter.cs

@@ -24,7 +24,8 @@ public class MarkdownReporter(IPluginEvents pluginEvents, IProxyContext context,
         { typeof(HttpFileGeneratorPlugin), TransformHttpFileGeneratorReport },
         { typeof(GraphMinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport },
         { typeof(GraphMinimalPermissionsPluginReport), TransformMinimalPermissionsReport },
-        { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }
+        { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport },
+        { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport }
     };
 
     private const string _requestsInterceptedMessage = "Requests intercepted";
@@ -483,6 +484,43 @@ void transformPermissionsInfo(GraphMinimalPermissionsInfo permissionsInfo, strin
         return sb.ToString();
     }
 
+    private static string? TransformUrlDiscoveryReport(object report)
+    {
+        var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report;
+
+        var sb = new StringBuilder();
+        sb.AppendLine("## Wildcards");
+        sb.AppendLine("");
+        sb.AppendLine("You can use wildcards to catch multiple URLs with the same pattern.");
+        sb.AppendLine("For example, you can use the following URL pattern to catch all API requests to");
+        sb.AppendLine("JSON Placeholder API:");
+        sb.AppendLine("");
+        sb.AppendLine("```text");
+        sb.AppendLine("https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("```");
+        sb.AppendLine("");
+        sb.AppendLine("## Excluding URLs");
+        sb.AppendLine("");
+        sb.AppendLine("You can exclude URLs with ! to prevent them from being intercepted.");
+        sb.AppendLine("For example, you can exclude the URL `https://jsonplaceholder.typicode.com/authors`");
+        sb.AppendLine("by using the following URL pattern:");
+        sb.AppendLine("");
+        sb.AppendLine("```text");
+        sb.AppendLine("!https://jsonplaceholder.typicode.com/authors");
+        sb.AppendLine("https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("```");
+        sb.AppendLine("");
+        sb.AppendLine("Intercepted URLs:");
+        sb.AppendLine();
+        sb.AppendLine("```text");
+
+        sb.AppendJoin(Environment.NewLine, urlDiscoveryPluginReport.Data);
+
+        sb.AppendLine("");
+        sb.AppendLine("```");
+        return sb.ToString();
+    }
+
     private static string? TransformHttpFileGeneratorReport(object report)
     {
         var httpFileGeneratorReport = (HttpFileGeneratorPluginReport)report;

dev-proxy-plugins/Reporters/PlainTextReporter.cs

@@ -24,7 +24,8 @@ public class PlainTextReporter(IPluginEvents pluginEvents, IProxyContext context
         { typeof(HttpFileGeneratorPluginReport), TransformHttpFileGeneratorReport },
         { typeof(GraphMinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport },
         { typeof(GraphMinimalPermissionsPluginReport), TransformMinimalPermissionsReport },
-        { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }
+        { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport },
+        { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport }
     };
 
     private const string _requestsInterceptedMessage = "Requests intercepted";
@@ -75,6 +76,36 @@ public class PlainTextReporter(IPluginEvents pluginEvents, IProxyContext context
         return sb.ToString();
     }
 
+    private static string? TransformUrlDiscoveryReport(object report)
+    {
+        var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report;
+
+        var sb = new StringBuilder();
+        sb.AppendLine("Wildcards");
+        sb.AppendLine("");
+        sb.AppendLine("You can use wildcards to catch multiple URLs with the same pattern.");
+        sb.AppendLine("For example, you can use the following URL pattern to catch all API requests to");
+        sb.AppendLine("JSON Placeholder API:");
+        sb.AppendLine("");
+        sb.AppendLine("https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("");
+        sb.AppendLine("Excluding URLs");
+        sb.AppendLine("");
+        sb.AppendLine("You can exclude URLs with ! to prevent them from being intercepted.");
+        sb.AppendLine("For example, you can exclude the URL https://jsonplaceholder.typicode.com/authors");
+        sb.AppendLine("by using the following URL pattern:");
+        sb.AppendLine("");
+        sb.AppendLine("!https://jsonplaceholder.typicode.com/authors");
+        sb.AppendLine("https://jsonplaceholder.typicode.com/*");
+        sb.AppendLine("");
+        sb.AppendLine("Intercepted URLs:");
+        sb.AppendLine();
+
+        sb.AppendJoin(Environment.NewLine, urlDiscoveryPluginReport.Data);
+
+        return sb.ToString();
+    }
+
     private static string? TransformExecutionSummaryByMessageType(object report)
     {
         var executionSummaryReport = (ExecutionSummaryPluginReportByMessageType)report;

dev-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs

@@ -79,6 +79,7 @@ private Task AfterRecordingStopAsync(object? sender, RecordingArgs e)
     {
         if (!e.RequestLogs.Any())
         {
+            Logger.LogRequest("No messages recorded", MessageType.Skipped);
             return Task.CompletedTask;
         }
 

dev-proxy-plugins/RequestLogs/UrlDiscoveryPlugin.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.DevProxy.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.DevProxy.Plugins.RequestLogs;
+
+public class UrlDiscoveryPluginReport
+{
+    public required List<string> Data { get; init; }
+}
+
+public class UrlDiscoveryPlugin(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet<UrlToWatch> urlsToWatch, IConfigurationSection? configSection = null) : BaseReportingPlugin(pluginEvents, context, logger, urlsToWatch, configSection)
+{
+    public override string Name => nameof(UrlDiscoveryPlugin);
+    private readonly ExecutionSummaryPluginConfiguration _configuration = new();
+
+    public override async Task RegisterAsync()
+    {
+        await base.RegisterAsync();
+
+        ConfigSection?.Bind(_configuration);
+
+        PluginEvents.AfterRecordingStop += AfterRecordingStopAsync;
+    }
+
+    private Task AfterRecordingStopAsync(object? sender, RecordingArgs e)
+    {
+        if (!e.RequestLogs.Any())
+        {
+            Logger.LogRequest("No messages recorded", MessageType.Skipped);
+            return Task.CompletedTask;
+        }
+
+        UrlDiscoveryPluginReport report = new()
+        {
+            Data = [.. e.RequestLogs.Select(log => log.Context?.Session.HttpClient.Request.RequestUri.ToString()).Distinct().Order()]
+        };
+
+        StoreReport(report, e);
+
+        return Task.CompletedTask;
+    }
+}

dev-proxy/presets/urls-to-watch.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://raw.githubusercontent.com/microsoft/dev-proxy/main/schemas/v0.24.0/rc.schema.json",
+  "plugins": [
+    {
+      "name": "UrlDiscoveryPlugin",
+      "enabled": true,
+      "pluginPath": "~appFolder/plugins/dev-proxy-plugins.dll"
+    },
+    {
+      "name": "PlainTextReporter",
+      "enabled": true,
+      "pluginPath": "~appFolder/plugins/dev-proxy-plugins.dll"
+    }
+  ],
+  "urlsToWatch": [
+    "https://*/*"
+  ],
+  "record": true
+}