Fix misleading code coverage measurement (#630)

scripts/build-cli.ps1

@@ -257,7 +257,13 @@ try
     # Step 5: Run tests (unless skipped)
     if (-not $SkipTests) {
         Write-Host "[TEST] Running tests..." -ForegroundColor Blue
-        dotnet run --project $CliTestsProjectPath -c Release --no-build --results-directory $CliSolutionDir\TestResults --report-trx --coverage --coverage-output-format cobertura
+        # Measure coverage against hand-written product source only. coverage.runsettings
+        # excludes generated interop (CsWin32/COM/Regex generators in obj\**) from the
+        # denominator; without it the reported number is diluted from a meaningful ~49% to a
+        # meaningless ~18%. Hand-written services (incl. the hardware/COM/GPU interop) are NOT
+        # excluded -- they are covered by real tests. See issue #630.
+        $CoverageSettings = (Resolve-Path "$CliSolutionDir\coverage.runsettings").Path
+        dotnet run --project $CliTestsProjectPath -c Release --no-build --results-directory $CliSolutionDir\TestResults --report-trx --coverage --coverage-settings $CoverageSettings --coverage-output-format cobertura
         $TestExitCode = $LASTEXITCODE
 
         # Copy test results to artifacts BEFORE checking for failure - find all TRX files

scripts/coverage-report.ps1

@@ -0,0 +1,220 @@
+#!/usr/bin/env pwsh
+<#
+.SYNOPSIS
+    Measure and report *meaningful* code coverage for the winapp CLI test suite.
+
+.DESCRIPTION
+    Runs the WinApp.Cli.Tests suite with Microsoft code coverage, applying
+    src/winapp-CLI/coverage.runsettings so that auto-generated interop code
+    (CsWin32 P/Invoke thunks, ComInterfaceGenerator COM shims, RegexGenerator
+    state machines under obj\**) is excluded from the denominator.
+
+    Without that exclusion the raw number is dominated by generated code and
+    reports ~18% instead of the real ~49% over hand-written source. See issue #630.
+
+    The script parses the produced Cobertura report, de-duplicates line hits across
+    partial classes / build configs, and prints:
+      * overall line coverage over hand-written product source,
+      * a per-directory breakdown,
+      * the top uncovered files (biggest gaps),
+      * optionally a per-area (single directory) view.
+
+    With -Threshold it fails (exit 1) when overall coverage is below the target,
+    so it can be used as a CI gate.
+
+.PARAMETER Configuration
+    Build configuration to test. Default: Release.
+
+.PARAMETER Filter
+    Optional MTP test filter (e.g. "FullyQualifiedName~MsixService"). Note: a filtered
+    run only instruments the code that actually loads, so the denominator will be smaller
+    than a full run. Use a full run for the authoritative number.
+
+.PARAMETER Area
+    Optional product sub-directory to focus the per-file report on (e.g. Services,
+    Commands, Helpers). The overall/per-directory numbers still reflect the whole suite.
+
+.PARAMETER Threshold
+    Optional overall line-coverage percentage (0-100). If set and coverage is below it,
+    the script exits with code 1.
+
+.PARAMETER Top
+    Number of top uncovered files to list. Default: 40.
+
+.PARAMETER SkipBuild
+    Skip building the test project (assumes it is already built for -Configuration).
+
+.PARAMETER CoberturaPath
+    Use an existing Cobertura XML report instead of running the tests.
+
+.PARAMETER CsvOut
+    Optional path to write the full per-file report as CSV.
+
+.EXAMPLE
+    ./scripts/coverage-report.ps1
+
+.EXAMPLE
+    ./scripts/coverage-report.ps1 -Area Services -Top 60
+
+.EXAMPLE
+    ./scripts/coverage-report.ps1 -Filter "FullyQualifiedName~MsixService" -SkipBuild
+
+.EXAMPLE
+    ./scripts/coverage-report.ps1 -Threshold 95
+#>
+[CmdletBinding()]
+param(
+    [string]$Configuration = "Release",
+    [string]$Filter,
+    [string]$Area,
+    [ValidateRange(0, 100)][double]$Threshold = -1,
+    [int]$Top = 40,
+    [switch]$SkipBuild,
+    [string]$CoberturaPath,
+    [string]$CsvOut
+)
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = Split-Path -Parent $PSScriptRoot
+$CliSolutionDir = Join-Path $RepoRoot "src\winapp-CLI"
+$TestsProject = Join-Path $CliSolutionDir "WinApp.Cli.Tests\WinApp.Cli.Tests.csproj"
+$Settings = Join-Path $CliSolutionDir "coverage.runsettings"
+$ResultsDir = Join-Path $CliSolutionDir "TestResults\coverage-report"
+$TestExitCode = 0
+
+function Write-Section($text) {
+    Write-Host ""
+    Write-Host $text -ForegroundColor Cyan
+    Write-Host ("-" * $text.Length) -ForegroundColor DarkGray
+}
+
+if (-not $CoberturaPath) {
+    if (-not (Test-Path $Settings)) {
+        throw "Coverage settings not found: $Settings"
+    }
+
+    if (-not $SkipBuild) {
+        Write-Section "Building test project ($Configuration)"
+        dotnet build $TestsProject -c $Configuration --nologo -v quiet
+        if ($LASTEXITCODE -ne 0) { throw "Test project build failed." }
+    }
+
+    if (Test-Path $ResultsDir) { Remove-Item $ResultsDir -Recurse -Force }
+    New-Item -ItemType Directory -Path $ResultsDir -Force | Out-Null
+
+    Write-Section "Running tests with coverage"
+    $runArgs = @(
+        "run", "--project", $TestsProject, "-c", $Configuration, "--no-build",
+        "--results-directory", $ResultsDir,
+        "--coverage", "--coverage-settings", $Settings,
+        "--coverage-output-format", "cobertura", "--coverage-output", "coverage.cobertura.xml"
+    )
+    if ($Filter) { $runArgs += @("--filter", $Filter) }
+    dotnet @runArgs
+    $TestExitCode = $LASTEXITCODE
+    # MTP returns non-zero when tests fail. We still parse and print coverage below (a failed
+    # run usually still emits a report), then propagate the failure via the exit code at the end
+    # so this script can't green-light a run whose tests actually failed.
+    if ($TestExitCode -ne 0) {
+        Write-Host "WARNING: test run exited with code $TestExitCode (test failures). Coverage is still reported below." -ForegroundColor Yellow
+    }
+
+    $CoberturaPath = Get-ChildItem -Path $ResultsDir -Filter "*.cobertura.xml" -Recurse -File |
+        Sort-Object LastWriteTime | Select-Object -Last 1 -ExpandProperty FullName
+    if (-not $CoberturaPath) { throw "No Cobertura report was produced under $ResultsDir." }
+}
+
+Write-Section "Parsing coverage report"
+Write-Host $CoberturaPath -ForegroundColor DarkGray
+[xml]$xml = Get-Content $CoberturaPath
+
+# De-duplicate line hits per source file (partial classes and multiple build configs
+# produce several <class> entries for the same file); keep the max hit count per line.
+$byFile = @{}
+foreach ($cls in $xml.coverage.packages.package.classes.class) {
+    $fn = $cls.filename
+    if (-not $fn) { continue }
+    if ($fn -match '\\obj\\') { continue }              # defensive: settings already exclude these
+    if ($fn -notmatch '\\WinApp\.Cli\\') { continue }   # product source only
+    if (-not $cls.lines.line) { continue }
+    if (-not $byFile.ContainsKey($fn)) { $byFile[$fn] = @{} }
+    foreach ($ln in @($cls.lines.line)) {
+        $num = [int]$ln.number
+        $hits = [int]$ln.hits
+        if (-not $byFile[$fn].ContainsKey($num) -or $byFile[$fn][$num] -lt $hits) {
+            $byFile[$fn][$num] = $hits
+        }
+    }
+}
+
+if ($byFile.Count -eq 0) {
+    throw "No hand-written product source found in the coverage report. Was the suite run against WinApp.Cli?"
+}
+
+$rows = foreach ($fn in $byFile.Keys) {
+    $valid = $byFile[$fn].Count
+    $covered = @($byFile[$fn].Values | Where-Object { $_ -gt 0 }).Count
+    $rel = $fn -replace '.*\\WinApp\.Cli\\', ''
+    [pscustomobject]@{
+        File      = $rel
+        Dir       = ($rel -split '\\')[0]
+        Valid     = $valid
+        Covered   = $covered
+        Uncovered = $valid - $covered
+        Pct       = [math]::Round($covered / $valid * 100, 1)
+    }
+}
+
+$totalValid = ($rows | Measure-Object Valid -Sum).Sum
+$totalCovered = ($rows | Measure-Object Covered -Sum).Sum
+$overall = [math]::Round($totalCovered / $totalValid * 100, 2)
+
+if ($CsvOut) {
+    $rows | Sort-Object Uncovered -Descending | Export-Csv $CsvOut -NoTypeInformation
+    Write-Host "Full per-file report written to $CsvOut" -ForegroundColor DarkGray
+}
+
+Write-Section "Coverage by directory"
+$rows | Group-Object Dir | ForEach-Object {
+    $v = ($_.Group | Measure-Object Valid -Sum).Sum
+    $c = ($_.Group | Measure-Object Covered -Sum).Sum
+    [pscustomobject]@{
+        Dir       = $_.Name
+        Files     = $_.Count
+        Valid     = $v
+        Covered   = $c
+        Pct       = [math]::Round($c / $v * 100, 1)
+        Uncovered = $v - $c
+    }
+} | Sort-Object Uncovered -Descending | Format-Table -AutoSize | Out-String | Write-Host
+
+$reportRows = $rows
+if ($Area) {
+    $reportRows = $rows | Where-Object { $_.Dir -ieq $Area -or $_.File -ilike "$Area\*" }
+    Write-Section "Top $Top uncovered files in '$Area'"
+} else {
+    Write-Section "Top $Top uncovered files"
+}
+$reportRows | Sort-Object Uncovered -Descending | Select-Object -First $Top |
+    Format-Table @{n = 'File'; e = { $_.File }; w = 60 }, Valid, Covered, Uncovered, Pct -AutoSize |
+    Out-String | Write-Host
+
+Write-Section "Overall (hand-written product source)"
+Write-Host ("  Files:     {0}" -f $rows.Count)
+Write-Host ("  Lines:     {0} covered / {1} valid" -f $totalCovered, $totalValid)
+$color = if ($overall -ge 95) { "Green" } elseif ($overall -ge 75) { "Yellow" } else { "Red" }
+Write-Host ("  Coverage:  {0}%" -f $overall) -ForegroundColor $color
+
+if ($Threshold -ge 0) {
+    if ($overall -lt $Threshold) {
+        Write-Host ("FAIL: coverage {0}% is below threshold {1}%." -f $overall, $Threshold) -ForegroundColor Red
+        exit 1
+    }
+    Write-Host ("PASS: coverage {0}% meets threshold {1}%." -f $overall, $Threshold) -ForegroundColor Green
+}
+
+if ($TestExitCode -ne 0) {
+    Write-Host ("FAIL: the test run reported failures (exit code {0}); failing despite the coverage report above." -f $TestExitCode) -ForegroundColor Red
+    exit $TestExitCode
+}

src/winapp-CLI/WinApp.Cli.Tests/README.md

@@ -103,6 +103,58 @@ Current test coverage includes comprehensive testing across multiple areas:
 
 The E2E tests provide comprehensive coverage of real-world scenarios, ensuring the CLI works correctly for typical developer workflows.
 
+## Code coverage
+
+We gate coverage on the **testable surface** of the CLI, not the raw line count. The raw
+`--coverage` denominator is dominated by generated interop (CsWin32 `NativeMethods.g.cs`,
+`ComInterfaceGenerator` COM shims for D3D11/UI Automation, `RegexGenerator`) emitted into
+`obj\`, which made the reported number misleading (~18% vs. ~49% real). See issue #630.
+
+### Measuring
+
+```powershell
+# Whole CLI, per-directory + top uncovered files, overall %:
+pwsh scripts\coverage-report.ps1
+
+# Focus one area and fail under a threshold (what sub-agents use):
+pwsh scripts\coverage-report.ps1 -Area Services -Filter "FullyQualifiedName~MsixService" -Threshold 95
+```
+
+`build-cli.ps1` also runs the suite with `src\winapp-CLI\coverage.runsettings` so CI numbers
+match local ones.
+
+### What's excluded (and why)
+
+**Only generated code is excluded** — via `coverage.runsettings` (`obj\**`, `*.g.cs`,
+`*.Designer.cs`, plus the `[GeneratedCode]` attribute). This is the CsWin32 P/Invoke thunks,
+`ComInterfaceGenerator` COM shims, and `RegexGenerator` state machines. It isn't hand-written,
+so it doesn't belong in the denominator. That single change is what moves the reported number
+from the misleading ~18% to the real figure.
+
+**Hardware / COM / GPU code is _not_ excluded.** `UiAutomationService`, `WgcCapture`, and the
+keyboard/mouse input helpers are real product code and stay in the denominator. This foundation
+PR sets up the measurement; the tests that cover them land in follow-up PRs, two ways:
+
+1. **Unit tests** for their pure-logic seams (selector parsing, element tree → JSON,
+   property/value formatting, foreground classification in `ForegroundGuard`, gesture
+   targeting) — these need no live desktop.
+2. **A real, in-process UI test** that launches the WinUI sample app and drives the genuine
+   `UiAutomationService` — inspect / search / invoke / set-value / wait-for / screenshot, plus
+   real type/click through the input helpers. Because it runs in-process in the MSTest host,
+   `--coverage` instruments those COM/input paths automatically — no separate collector or
+   coverage-merge step. It will be gated to **skip** when no interactive desktop is available,
+   so it never blocks a run; its coverage counts when the environment can host it.
+
+> `coverage.runsettings` intentionally contains **no XML comments** — the `--coverage-settings`
+> parser rejects them. Document rationale here, not in that file.
+
+### Policy
+
+- **Don't exclude services (or any logic) to grow the number.** The only exclusion is generated
+  code. Everything hand-written — including the hardware/COM/GPU interop — stays in the
+  denominator and is covered by real tests.
+- Write meaningful use-case tests first, then unit tests to close the remaining gaps.
+
 ## Framework Used
 
 - **MSTest** - Microsoft's testing framework for .NET

src/winapp-CLI/coverage.runsettings

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0">
+        <Configuration>
+          <CodeCoverage>
+            <CollectFromChildProcesses>True</CollectFromChildProcesses>
+            <ModulePaths>
+              <Exclude>
+                <ModulePath>.*Tests\.dll$</ModulePath>
+                <ModulePath>.*Microsoft\.VisualStudio\.TestPlatform.*</ModulePath>
+                <ModulePath>.*Microsoft\.TestPlatform.*</ModulePath>
+                <ModulePath>.*testhost.*</ModulePath>
+              </Exclude>
+            </ModulePaths>
+            <Attributes>
+              <Exclude>
+                <Attribute>^System\.CodeDom\.Compiler\.GeneratedCodeAttribute$</Attribute>
+              </Exclude>
+            </Attributes>
+            <Sources>
+              <Exclude>
+                <Source>.*\\obj\\.*</Source>
+                <Source>.*\.g\.cs$</Source>
+                <Source>.*\.Designer\.cs$</Source>
+              </Exclude>
+            </Sources>
+          </CodeCoverage>
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>