Support filtering by area when running tool description evaluator

eng/tools/ToolDescriptionEvaluator/Program.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System.Diagnostics;
+using System.Linq;
 using System.Text.Json;
 using ToolSelection.Models;
 using ToolSelection.Services;
@@ -39,6 +40,7 @@ static async Task Main(string[] args)
             string? customToolsFile = null; // Optional custom tools file
             string? customPromptsFile = null; // Optional custom prompts file
             string? customOutputFileName = null; // Optional custom output file name
+            string? areaFilter = null; // Optional area filter for prompts
 
             for (int i = 0; i < args.Length; i++)
             {
@@ -65,6 +67,10 @@ static async Task Main(string[] args)
                 {
                     customOutputFileName = args[i + 1];
                 }
+                else if (args[i] == "--area" && i + 1 < args.Length)
+                {
+                    areaFilter = args[i + 1];
+                }
             }
 
             string exeDir = AppContext.BaseDirectory;
@@ -229,6 +235,13 @@ static async Task Main(string[] args)
                 Console.WriteLine("📝 Using default prompts (e2eTestPrompts.md)");
             }
 
+            if (!string.IsNullOrEmpty(areaFilter))
+            {
+                var areaCount = areaFilter.Split(',', StringSplitOptions.RemoveEmptyEntries).Length;
+                var areaLabel = areaCount > 1 ? "areas" : "area";
+                Console.WriteLine($"🎯 Filtering prompts to {areaLabel}: {areaFilter}");
+            }
+
             // Create or overwrite the output file
             using var writer = new StreamWriter(outputFilePath, false);
 
@@ -256,7 +269,7 @@ static async Task Main(string[] args)
                 // User specified a custom prompts file
                 if (customPromptsFileResolved.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
                 {
-                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(customPromptsFileResolved, isCiMode);
+                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(customPromptsFileResolved, isCiMode, areaFilter);
                 }
                 else if (customPromptsFileResolved.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
                 {
@@ -265,7 +278,7 @@ static async Task Main(string[] args)
                 else
                 {
                     // Try to infer format or default to markdown
-                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(customPromptsFileResolved, isCiMode) ??
+                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(customPromptsFileResolved, isCiMode, areaFilter) ??
                                         await LoadPromptsFromJsonAsync(customPromptsFileResolved, isCiMode);
                 }
             }
@@ -278,7 +291,7 @@ static async Task Main(string[] args)
                 if (File.Exists(defaultPromptsPath))
                 {
                     // Load from markdown and save a normalized JSON copy for future runs
-                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(defaultPromptsPath, isCiMode);
+                    toolNameAndPrompts = await LoadPromptsFromMarkdownAsync(defaultPromptsPath, isCiMode, areaFilter);
 
                     if (toolNameAndPrompts != null)
                     {
@@ -654,7 +667,7 @@ private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, stri
         }
     }
 
-    private static async Task<Dictionary<string, List<string>>?> LoadPromptsFromMarkdownAsync(string filePath, bool isCiMode = false)
+    private static async Task<Dictionary<string, List<string>>?> LoadPromptsFromMarkdownAsync(string filePath, bool isCiMode = false, string? areaFilter = null)
     {
         try
         {
@@ -678,10 +691,9 @@ private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, stri
             {
                 var trimmedLine = line.Trim();
 
-                // Skip table headers and separators
+                // Skip headers, separators, and non-table content
                 if (trimmedLine.StartsWith("| Tool Name") ||
                     trimmedLine.StartsWith("|:-------") ||
-                    trimmedLine.StartsWith("##") ||
                     trimmedLine.StartsWith("#") ||
                     string.IsNullOrWhiteSpace(trimmedLine))
                 {
@@ -701,6 +713,30 @@ private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, stri
                         if (string.IsNullOrWhiteSpace(toolName) || string.IsNullOrWhiteSpace(prompt))
                             continue;
 
+                        // Filter by tool name prefix(es) (e.g., azmcp_keyvault, azmcp_storage)
+                        if (!string.IsNullOrEmpty(areaFilter))
+                        {
+                            // Support multiple areas separated by commas
+                            var areas = areaFilter.Split(',', StringSplitOptions.RemoveEmptyEntries)
+                                                   .Select(a => a.Trim())
+                                                   .Where(a => !string.IsNullOrEmpty(a))
+                                                   .ToList();
+
+                            // Auto-add azmcp_ prefix if not already present for each area
+                            var prefixesToMatch = areas.Select(area =>
+                                area.StartsWith("azmcp_", StringComparison.OrdinalIgnoreCase)
+                                    ? area
+                                    : $"azmcp_{area}"
+                            ).ToList();
+
+                            // Check if tool name starts with any of the area filters
+                            if (!prefixesToMatch.Any(prefix => toolName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
+                            {
+                                // Skip this tool as it doesn't match any prefix
+                                continue;
+                            }
+                        }
+
                         if (!prompts.ContainsKey(toolName))
                         {
                             prompts[toolName] = new List<string>();
@@ -711,6 +747,15 @@ private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, stri
                 }
             }
 
+            // If area filter was specified but no prompts found, provide feedback
+            if (!string.IsNullOrEmpty(areaFilter) && prompts.Count == 0 && !isCiMode)
+            {
+                var actualPrefix = areaFilter.StartsWith("azmcp_", StringComparison.OrdinalIgnoreCase)
+                    ? areaFilter
+                    : $"azmcp_{areaFilter}";
+                Console.WriteLine($"⚠️  No prompts found for prefix '{actualPrefix}'. Use service names like 'keyvault', 'storage', 'functionapp', etc.");
+            }
+
             return prompts.Count > 0 ? prompts : null;
         }
         catch (Exception)
@@ -1156,6 +1201,7 @@ private static void ShowHelp()
         Console.WriteLine("  --ci                          Run in CI mode (graceful failures)");
         Console.WriteLine("  --tools-file <path>           Use a custom JSON file for tools instead of dynamic loading from docs .md");
         Console.WriteLine("  --prompts-file <path>         Use custom prompts file (supported formats: .md or .json)");
+        Console.WriteLine("  --area <area>                 Filter prompts by tool name prefix(es) (e.g., \"keyvault\", \"storage,functionapp\", \"azmcp_keyvault\")");
         Console.WriteLine("  --output-file-name <name>     Custom output file name (no extension)");
         Console.WriteLine("  --text-results                Output results in .txt format");
         Console.WriteLine("  --top <N>                     Number of results to display per test (default 5)");
@@ -1170,6 +1216,10 @@ private static void ShowHelp()
         Console.WriteLine("  ToolDescriptionEvaluator                                          # Use dynamic tool loading (default)");
         Console.WriteLine("  ToolDescriptionEvaluator --tools-file my-tools.json               # Use custom tools file");
         Console.WriteLine("  ToolDescriptionEvaluator --prompts-file my-prompts.md             # Use custom prompts file");
+        Console.WriteLine("  ToolDescriptionEvaluator --area \"keyvault\"                       # Test only Key Vault prompts (auto-prefixed to azmcp_keyvault)");
+        Console.WriteLine("  ToolDescriptionEvaluator --area \"storage\"                        # Test only Storage prompts (auto-prefixed to azmcp_storage)");
+        Console.WriteLine("  ToolDescriptionEvaluator --area \"keyvault,storage\"               # Test Key Vault and Storage prompts (multiple areas)");
+        Console.WriteLine("  ToolDescriptionEvaluator --area \"azmcp_functionapp\"              # Test only Function App prompts (explicit prefix)");
         Console.WriteLine("  ToolDescriptionEvaluator --output-file-name my-results            # Use custom output file name (don't include extension)");
         Console.WriteLine("  ToolDescriptionEvaluator --text-results                           # Output in text format");
         Console.WriteLine("  ToolDescriptionEvaluator --ci --tools-file tools.json             # CI mode with JSON file");

eng/tools/ToolDescriptionEvaluator/README.md

@@ -56,7 +56,27 @@ dotnet run -- --validate \
   --prompt "what storage accounts do I have"
 ```
 
-### 3. Custom Files Mode
+### 3. Tool Prefix Filtering Mode
+
+Filter prompts by tool name prefix to test specific Azure service tools. Service names are automatically prefixed with `azmcp_`:
+
+```bash
+# Filter by service name (auto-prefixed to azmcp_*)
+dotnet run -- --area "keyvault"      # Filters azmcp_keyvault_* tools
+dotnet run -- --area "storage"       # Filters azmcp_storage_* tools
+dotnet run -- --area "functionapp"   # Filters azmcp_functionapp_* tools
+dotnet run -- --area "sql"           # Filters azmcp_sql_* tools
+dotnet run -- --area "cosmos"        # Filters azmcp_cosmos_* tools
+
+# Filter multiple services at once (comma-separated)
+dotnet run -- --area "keyvault,storage"       # Filters both Key Vault and Storage tools
+dotnet run -- --area "sql,cosmos,storage"     # Filters SQL, Cosmos, and Storage tools
+
+# Or use explicit prefix (same result)
+dotnet run -- --area "azmcp_keyvault"
+```
+
+### 4. Custom Files Mode
 
 Use custom tools or prompts files:
 
@@ -67,8 +87,8 @@ dotnet run -- --tools-file my-tools.json
 # Use custom prompts file (supports .md or .json)
 dotnet run -- --prompts-file my-prompts.md
 
-# Use both custom files
-dotnet run -- --tools-file my-tools.json --prompts-file my-prompts.json
+# Use both custom files with prefix filtering
+dotnet run -- --tools-file my-tools.json --prompts-file my-prompts.json --area "storage"
 ```
 
 ## Input Data Sources
@@ -91,14 +111,30 @@ The tool can load data from multiple sources:
 You can call the build script in this directory:
 
 ```bash
+# Run with all areas
 ./Run-ToolDescriptionEvaluator.ps1
+
+# Run with specific tool prefix filtering
+./Run-ToolDescriptionEvaluator.ps1 -Area "storage"
+./Run-ToolDescriptionEvaluator.ps1 -Area "keyvault"
+./Run-ToolDescriptionEvaluator.ps1 -Area "functionapp"
+
+# Run with multiple areas (comma-separated)
+./Run-ToolDescriptionEvaluator.ps1 -Area "keyvault,storage"
+./Run-ToolDescriptionEvaluator.ps1 -Area "sql,cosmos,functionapp"
+
+# Build Azure MCP Server first, then run with prefix filtering
+./Run-ToolDescriptionEvaluator.ps1 -BuildAzureMcp -Area "sql"
 ```
 
 or run the following commands directly:
 
 ```bash
 dotnet build
 dotnet run
+
+# With tool prefix filtering
+dotnet run -- --area "storage"
 ```
 
 ## Setup
@@ -168,12 +204,34 @@ dotnet run -- --text
 - Includes confidence scores and success rates
 - Shows top matching tools for each prompt
 
-### Custom output file name
+### Command Line Options
 
-You can use a custom file name by using the option `--output-file-name`
+The tool supports several command line options for customization:
 
 ```bash
-dotnet run -- --output-file-name my-tests
+# Tool prefix filtering
+dotnet run -- --area "storage"                    # Filter to storage tools only (auto-prefixed to azmcp_storage)
+dotnet run -- --area "keyvault"                   # Filter to Key Vault tools only (auto-prefixed to azmcp_keyvault)
+dotnet run -- --area "functionapp"                # Filter to Function App tools only (auto-prefixed to azmcp_functionapp)
+dotnet run -- --area "keyvault,storage"           # Filter to multiple areas (comma-separated)
+dotnet run -- --area "sql,cosmos,functionapp"     # Filter to SQL, Cosmos, and Function App tools
+
+# File options
+dotnet run -- --tools-file my-tools.json          # Use custom tools file
+dotnet run -- --prompts-file my-prompts.md        # Use custom prompts file
+dotnet run -- --output-file-name my-tests         # Custom output filename
+
+# Output format
+dotnet run -- --text-results                      # Output in plain text format
+
+# Result limits
+dotnet run -- --top 10                           # Show top 10 results per test
+
+# CI mode
+dotnet run -- --ci                               # Run in CI mode (graceful failures)
+
+# Combined options
+dotnet run -- --area "keyvault" --text-results --top 3
 ```
 
 ### Analysis Metrics
@@ -202,18 +260,52 @@ The tool provides several key metrics:
 
 #### Markdown Format (Default)
 
-The tool reads from `../../../docs/e2eTestPrompts.md` which contains tables like:
+The tool reads from `../../../servers/Azure.Mcp.Server/docs/e2eTestPrompts.md` which contains tables organized by service:
 
 ```markdown
 ## Azure Storage
 
 | Tool Name | Test Prompt |
 |:----------|:------------|
-| azmcp-storage-account-get | List all storage accounts in my subscription |
-| azmcp-storage-account-get | Show me my storage accounts |
-| azmcp-storage-container-get | List containers in storage account <account-name> |
+| azmcp_storage_account_get | List all storage accounts in my subscription |
+| azmcp_storage_account_get | Show me my storage accounts |
+| azmcp_storage_container_get | List containers in storage account <account-name> |
+
+## Azure Key Vault
+
+| Tool Name | Test Prompt |
+|:----------|:------------|
+| azmcp_keyvault_secret_get | Get my secret from Key Vault |
+| azmcp_keyvault_key_list | List all keys in my Key Vault |
 ```
 
+#### Tool Prefix Filtering
+
+The tool supports filtering by tool name prefixes using the `--area` parameter. This allows you to test all tools for a specific Azure service by matching the tool name prefix.
+
+For example, `--area "keyvault"` (auto-prefixed to `azmcp_keyvault`) will match all tools starting with `azmcp_keyvault` including:
+- `azmcp_keyvault_certificate_create`
+- `azmcp_keyvault_certificate_get`  
+- `azmcp_keyvault_secret_get`
+- `azmcp_keyvault_key_list`
+- And all other Key Vault tools
+
+```bash
+# Filter by service name (automatically prefixed with azmcp_)
+dotnet run -- --area "keyvault"       # Matches all azmcp_keyvault_* tools
+dotnet run -- --area "storage"        # Matches all azmcp_storage_* tools
+dotnet run -- --area "functionapp"    # Matches all azmcp_functionapp_* tools
+
+# Filter multiple services at once (comma-separated)
+dotnet run -- --area "keyvault,storage"        # Matches Key Vault and Storage tools
+dotnet run -- --area "sql,cosmos,functionapp"  # Matches SQL, Cosmos, and Function App tools
+
+# Or use explicit prefix (same result)
+dotnet run -- --area "azmcp_keyvault"
+```
+
+Common service names (automatically prefixed with `azmcp_`) include: `foundry`, `search`, `appconfig`, `applens`, `appservice`, `applicationinsights`, `acr`, `cosmos`, `kusto`, `mysql`, `postgres`, `eventgrid`, `functionapp`, `keyvault`, `aks`, `loadtesting`, `monitor`, `quota`, `redis`, `storage`, `servicebus`, `sql`, `virtualdesktop`, `workbooks`, and more.
+
 #### JSON Format (Alternative)
 
 Prompts can be organized in JSON format:

eng/tools/ToolDescriptionEvaluator/Run-ToolDescriptionEvaluator.ps1

@@ -15,6 +15,9 @@
 .PARAMETER BuildAzureMcp
     Optionally build the root project in Debug mode to ensure tools can be loaded dynamically
 
+.PARAMETER Area
+    Filter prompts by tool name prefix. Service names are auto-prefixed with "azmcp_" (e.g., "keyvault" becomes "azmcp_keyvault")
+
 .EXAMPLE
     ./Run-ToolDescriptionEvaluator.ps1
     Builds and runs the Tool Description Evaluator application with default settings
@@ -23,11 +26,24 @@
     ./Run-ToolDescriptionEvaluator.ps1 -BuildAzureMcp
     Builds the Azure MCP Server project in Debug mode, then builds and runs the Tool Description Evaluator application
     with default settings
+
+.EXAMPLE
+    ./Run-ToolDescriptionEvaluator.ps1 -Area "storage"
+    Runs the Tool Description Evaluator filtering prompts to only tools with the azmcp_storage prefix
+
+.EXAMPLE
+    ./Run-ToolDescriptionEvaluator.ps1 -Area "keyvault"
+    Runs the Tool Description Evaluator filtering prompts to only tools with the azmcp_keyvault prefix
+
+.EXAMPLE
+    ./Run-ToolDescriptionEvaluator.ps1 -Area "functionapp" -BuildAzureMcp
+    Builds the Azure MCP Server project, then runs the Tool Description Evaluator filtering prompts to only Function App tools
 #>
 
 [CmdletBinding()]
 param(
-    [switch]$BuildAzureMcp
+    [switch]$BuildAzureMcp,
+    [string]$Area
 )
 
 Set-StrictMode -Version 3.0
@@ -104,10 +120,29 @@ try {
     }
 
     Write-Host "Build completed successfully!" -ForegroundColor Green
-    Write-Host "Running with: dotnet run" -ForegroundColor Cyan
+
+    # Build the command arguments
+    $runArgs = @()
+
+    if ($Area) {
+        $runArgs += "--area"
+        $runArgs += $Area
+        Write-Host "Running with area filter: $Area" -ForegroundColor Cyan
+    }
+
+    $runCommand = "dotnet run"
+    if ($runArgs.Count -gt 0) {
+        $runCommand += " -- " + ($runArgs -join " ")
+    }
+
+    Write-Host "Running with: $runCommand" -ForegroundColor Cyan
     Push-Location $toolDir
 
-    & dotnet run
+    if ($runArgs.Count -gt 0) {
+        & dotnet run -- @runArgs
+    } else {
+        & dotnet run
+    }
 
     Pop-Location
 }