Parallelise outerloop quarantined test execution

Use the Arcade SDK capabilities to provide a custom test runner
(see in eng/QuarantinedTestRunsheetBuilder) to intercept test execution
to pretend to run tests. Instead, only query test assemblies whether those
contain quarantined tests by running `dotnet test --list-tests`.
Then inspect the output, and if a test project contains quarantined tests
include the test project into a runsheet.

The runsheet generation itself is a two step process.

1. The first step is performed by each project during RunTests target
execution (in QuarantinedTestRunsheetBuilder) and it
results in project-specific runsheets being generated.
E.g., Aspire.Cli.Tests_net8.0_x64.linux.runsheet.json.

2. After all test projects have been inspected, all the individual runsheets
are aggregated into a single runsheet, which is then used as a matrix
by GHA workflow.
This is happening in eng/AfterSolutionBuild.targets.

Author: RussKie

.github/workflows/tests-outerloop.yml

@@ -12,88 +12,60 @@ concurrency:
 
 jobs:
 
-  test:
-    name: ${{ matrix.os.title }}
-    runs-on: ${{ matrix.os.name }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - name: ubuntu-latest
-            title: Linux
-          - name: windows-latest
-            title: Windows
+  generate_tests_matrix:
+    name: Generate test runsheet
+    runs-on: windows-latest
+    if: ${{ github.repository_owner == 'dotnet' }}
+    outputs:
+      runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }}
     steps:
-      - name: Setup vars (Linux)
-        if: ${{ matrix.os.name == 'ubuntu-latest' }}
-        run: |
-          echo "DOTNET_SCRIPT=./dotnet.sh" >> $GITHUB_ENV
-          echo "BUILD_SCRIPT=./build.sh" >> $GITHUB_ENV
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Setup vars (Windows)
-        if: ${{ matrix.os.name == 'windows-latest' }}
+      # We need to build the whole solution, so that we can interrogate each test project
+      # and find out whether it contains any quarantined tests.
+      - name: Build the solution
         run: |
-          echo "DOTNET_SCRIPT=.\dotnet.cmd" >> $env:GITHUB_ENV
-          echo "BUILD_SCRIPT=.\build.cmd" >> $env:GITHUB_ENV
+          ./build.cmd -restore -build -c Release -ci /p:CI=false /p:GeneratePackageOnBuild=false /p:InstallBrowsersForPlaywright=false /bl:./artifacts/log/Release/build.binlog
 
-      - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Trust HTTPS development certificate (Linux)
-        if: matrix.os.name == 'ubuntu-latest'
-        run: ${{ env.DOTNET_SCRIPT }} dev-certs https --trust
-
-      - name: Run quarantined tests
-        env:
-          CI: false
+      - name: Generate test runsheet
+        id: generate_tests_matrix
         run: |
-          ${{ env.BUILD_SCRIPT }} -projects ${{ github.workspace }}/tests/Shared/SolutionTests.proj -ci -restore -build -test -c Release /p:RunQuarantinedTests=true /bl:${{ github.workspace }}/artifacts/log/Release/test-quarantined.binlog
+          ./build.cmd -test /p:TestRunnerName=QuarantinedTestRunsheetBuilder /p:RunQuarantinedTests=true -c Release -ci /p:CI=false /p:Restore=false /p:Build=false /bl:./artifacts/log/Release/runsheet.binlog
 
-      - name: Keep only relevant test logs
-        if: always()
-        shell: pwsh
-        run: |
-          # Define the directory to search for log files
-          $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs"
+      - name: Upload logs, and test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: logs-runsheet
+          path: |
+            ${{ github.workspace }}/artifacts/log/*/*.binlog
+            ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+            ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
+          retention-days: 5
 
-          # Define the text to search for in the log files
-          $searchText = "No test matches the given testcase filter"
-          $resultsFilePattern = "Results File: (.+)"
+  run_tests:
+    name: Test
+    needs: generate_tests_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }}
 
-          # Get all .log files in the specified directory and its subdirectories
-          $logFiles = Get-ChildItem -Path $logDirectory -Filter *.log -Recurse
+    runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
+    if: ${{ github.repository_owner == 'dotnet' }}
 
-          foreach ($logFile in $logFiles) {
-              # Read the content of the log file
-              $content = Get-Content -Path $logFile.FullName
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-              # Check if the content contains the specified text
-              if ($content -match $searchText) {
-                  # Remove the log file if it contains the specified text
-                  Remove-Item -Path $logFile.FullName -Force
-                  Write-Host "Removed file: $($logFile.FullName)"
-              }
-              else {
-                  # Extract paths from lines containing "Results File: <path>"
-                  foreach ($line in $content) {
-                      if ($line -match $resultsFilePattern) {
-                          $resultsFilePath = $matches[1]
-                          Write-Host "Found results file: $resultsFilePath"
-
-                          # Copy the results file to the TestLogs folder
-                          $destinationPath = (Split-Path -Path $logFile.FullName -Parent)
-                          Copy-Item -Path $resultsFilePath -Destination $destinationPath -Force
-                          Write-Host "Copied $resultsFilePath to $destinationPath"
-                      }
-                  }
-              }
-          }
+      - name: Test ${{ matrix.tests.project }}
+        run: |
+          ${{ matrix.tests.command }}
 
       - name: Process logs and post results
         if: always()
         shell: pwsh
         run: |
-          $logDirectory = "${{ github.workspace }}/artifacts/log/**/TestLogs"
+          $logDirectory = "${{ github.workspace }}/artifacts/TestResults"
           $trxFiles = Get-ChildItem -Path $logDirectory -Filter *.trx -Recurse
 
           $testResults = @() # Initialize an array to store test results
@@ -157,19 +129,13 @@ jobs:
           $table | Out-File -FilePath $outputPath -Encoding utf8
           Write-Host "Test results saved to $outputPath"
 
-          # Windows-specific: Check for failed tests and set the exit code accordingly
-          # This is a workaround for the issue with the `exit` command in PowerShell
-          if ($failedTests -gt 0) {
-              Write-Host "::error::Build failed. Check errors above."
-              exit 1
-          }
-
       - name: Upload logs, and test results
-        if: always()
+        if: failure()
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
-          name: logs-${{ matrix.os.name }}
+          name: logs-${{ matrix.tests.os }}-${{ matrix.tests.project }}
           path: |
             ${{ github.workspace }}/artifacts/log/*/*.binlog
             ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+            ${{ github.workspace }}/artifacts/log/TestResults/*/*.trx
           retention-days: 5

eng/AfterSolutionBuild.targets

@@ -0,0 +1,90 @@
+<Project>
+
+  <!--
+    In short, this file automates the creation of a combined JSON runsheet from individual test runsheets,
+    making it easy to integrate quarantined test execution into CI workflows, especially GitHub Actions.
+   -->
+
+  <!--
+    This MSBuild targets file automates the generation of a combined test runsheet after building a solution,
+    using the QuarantinedTestRunsheetBuilder (located in eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets).
+
+    Here's a high-level overview of its logic:
+
+    1. **Conditionally Triggered Target**:
+      - The target _GenerateTestMatrix runs before the Test target, but only if the TestRunnerName property is set to QuarantinedTestRunsheetBuilder.
+
+    2. **Define Paths and Script Content**:
+      - Defines a combined runsheet JSON file path (_CombinedRunsheetFile) in the artifacts temporary directory.
+      - Defines a PowerShell script (_Command) that:
+      - Finds all individual .runsheet.json files in the artifacts directory.
+      - Reads and merges their JSON content into a single combined JSON array.
+      - Writes the combined JSON to the combined runsheet file.
+      - Checks if running in GitHub Actions environment:
+        - If yes, outputs the combined JSON to GitHub Actions output (GITHUB_OUTPUT).
+        - If no, prints the combined JSON to the console.
+
+    3. **Write and Execute the Script**:
+      - Writes the defined PowerShell script content to a temporary script file (create-runsheet.ps1).
+      - Executes this script using MSBuild's <Exec> task.
+
+    4. **Output Results**:
+      - After execution, logs a message indicating the combined runsheet was created, along with the script's output.
+
+    A runsheet is a JSON file that describes what tests to be run.
+    For example:
+
+      ```json
+      [
+        {
+          "project": "Aspire.Test",
+          "os": "windows-latest",
+          "command": "./eng/build.ps1 -restore -build -test -projects \"$(RelativeTestProjectPath)\" /bl:\"$(RelativeTestBinLog)\" -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false"
+        },
+        {
+          "project": "Aspire.Cli.Test",
+          "os": "ubuntu-latest",
+          "command": "./eng/build.sh -restore -build -test -projects \"$(RelativeTestProjectPath)\" /bl:\"$(RelativeTestBinLog)\" -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false"
+        }
+      ]
+      ```
+
+    -->
+  <Target Name="_GenerateTestMatrix" BeforeTargets="Test" Condition=" '$(TestRunnerName)' == 'QuarantinedTestRunsheetBuilder' ">
+    <PropertyGroup>
+      <_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json</_CombinedRunsheetFile>
+      <_Command>
+        $combined = @()
+        Get-ChildItem -Path '$(ArtifactsTmpDir)' -Filter '*.runsheet.json' |
+            ForEach-Object {
+                $content = Get-Content -Raw $_.FullName | ConvertFrom-Json
+                if ($content -is [Array]) {
+                    $combined += $content
+                }
+                else {
+                    $combined += @($content)
+                }
+            }
+        $jsonString = ($combined | ConvertTo-Json -Depth 10 -Compress)
+        $jsonString | Set-Content '$(_CombinedRunsheetFile)';
+
+        # determine if the script is running in a GitHub Actions environment
+        if ($env:CI -and $env:GITHUB_ACTIONS) {
+            "runsheet=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+        }
+        else {
+            Write-Host "runsheet=$jsonString"
+        }
+      </_Command>
+      <_Script>$([MSBuild]::NormalizePath($(ArtifactsTmpDir), 'create-runsheet.ps1'))</_Script>
+    </PropertyGroup>
+
+    <WriteLinesToFile File="$(_Script)"
+                      Lines="$(_Command)"
+                      Overwrite="true" />
+    <Exec Command="pwsh -File $(_Script)" ConsoleToMSBuild="true">
+      <Output TaskParameter="ConsoleOutput" PropertyName="ScriptOutput" />
+    </Exec>
+    <Message Importance="high" Text="Combined runsheet created%0D%0A%0D%0A$(ScriptOutput)" />
+  </Target>
+</Project>