Send host tests to helix (#114187)

elinor-fung · web-flow · commit 2b96e8a7d1e1 · 2025-05-13T10:10:45.000-07:00
Update installer pipelines to send host tests to helix instead of running locally
- Add a post-build step to the installer jobs to send host tests to Helix
- Test runs are named `host-&lt;os&gt;-&lt;arch&gt; @ &lt;helix_queue&gt;`
- Remove explicit upload of build artifacts on failure.
  - This was for test failure repro/investigation. It is now part of the helix correlation payload, so they can be downloaded that way

This change only handles platforms that we were already running tests on (linux_x64, osx_x64, windown_x64, windows_x86).  We would want further updates to add other platforms (for example, *_arm64, linux_musl_*).

This does remove running of `Microsoft.DotNet.CoreSetup.Packaging.Tests` tests in PR/CI. I don't think those are particularly useful at this point - I've haven't seen them fail and find an issue in my time here, 

Test updates:
- Copy debug CRT to output directory when building IJW test library in debug - we need this to run IJW tests on the test machine
- Resolve /tmp symlink on macOS for test artifacts and assets path for expected path comparisons
- Add retry for restoring test file backup
diff --git a/eng/pipelines/installer/helix-queues-setup.yml b/eng/pipelines/installer/helix-queues-setup.yml
@@ -0,0 +1,66 @@
+parameters:
+  jobTemplate: ''
+  variables: []
+  osGroup: ''
+  osSubgroup: ''
+  archType: ''
+  container: ''
+  pool: ''
+  platform: ''
+  shouldContinueOnError: false
+  jobParameters: {}
+
+jobs:
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    variables: ${{ parameters.variables }}
+    osGroup: ${{ parameters.osGroup }}
+    osSubgroup: ${{ parameters.osSubgroup }}
+    archType: ${{ parameters.archType }}
+    container: ${{ parameters.container }}
+    pool: ${{ parameters.pool }}
+    platform: ${{ parameters.platform }}
+    shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+    helixQueues:
+
+    # Linux arm
+    - ${{ if eq(parameters.platform, 'linux_arm') }}:
+      - (Debian.12.Arm32.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm32v7
+
+    # Linux arm64
+    - ${{ if eq(parameters.platform, 'linux_arm64') }}:
+      - (Ubuntu.2404.Arm64.Open)Ubuntu.2204.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04-helix-arm64v8
+
+    # Linux musl x64
+    - ${{ if eq(parameters.platform, 'linux_musl_x64') }}:
+      - (Alpine.321.Amd64.Open)azurelinux.3.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.21-helix-amd64
+
+    # Linux musl arm64
+    - ${{ if and(eq(parameters.platform, 'linux_musl_arm64'), or(eq(parameters.jobParameters.isExtraPlatformsBuild, true), eq(parameters.jobParameters.includeAllPlatforms, true))) }}:
+      - (Alpine.321.Arm64.Open)ubuntu.2204.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.21-helix-arm64v8
+
+    # Linux x64
+    - ${{ if eq(parameters.platform, 'linux_x64') }}:
+      - azurelinux.3.amd64.open
+
+    # OSX arm64
+    - ${{ if eq(parameters.platform, 'osx_arm64') }}:
+      - osx.13.arm64.open
+
+    # OSX x64
+    - ${{ if eq(parameters.platform, 'osx_x64') }}:
+      - OSX.1200.Amd64.Open
+
+    # windows x64
+    - ${{ if eq(parameters.platform, 'windows_x64') }}:
+      - Windows.11.Amd64.Client.Open
+
+    # windows x86
+    - ${{ if eq(parameters.platform, 'windows_x86') }}:
+      - Windows.11.Amd64.Client.Open
+
+    # windows arm64
+    - ${{ if eq(parameters.platform, 'windows_arm64') }}:
+      - Windows.11.Arm64.Open
+
+    ${{ insert }}: ${{ parameters.jobParameters }}
diff --git a/eng/pipelines/installer/helix.yml b/eng/pipelines/installer/helix.yml
@@ -0,0 +1,29 @@
+parameters:
+  archType: ''
+  osGroup: ''
+  creator: ''
+  helixQueues: ''
+
+steps:
+  - template: /eng/common/templates/steps/send-to-helix.yml
+    parameters:
+      DisplayNamePrefix: Send to Helix
+      HelixProjectPath: src/installer/tests/helixpublish.proj
+      ${{ if ne(parameters.osGroup, 'windows') }}:
+        HelixProjectArguments: >-
+          --ci
+          /p:Configuration=$(_BuildConfig)
+          /p:TargetArchitecture=${{ parameters.archType }}
+          /p:TargetOS=${{ parameters.osGroup }}
+      ${{ if eq(parameters.osGroup, 'windows') }}:
+        HelixProjectArguments: >-
+          -ci
+          /p:Configuration=$(_BuildConfig)
+          /p:TargetArchitecture=${{ parameters.archType }}
+          /p:TargetOS=${{ parameters.osGroup }}
+      Creator: ${{ parameters.creator }}
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        HelixAccessToken: $(HelixApiAccessToken)
+        HelixTargetQueues: ${{ replace(lower(join('+', parameters.helixQueues)), '.open', '') }}
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        HelixTargetQueues: ${{ join('+', parameters.helixQueues) }}
diff --git a/eng/pipelines/installer/steps/upload-job-artifacts.yml b/eng/pipelines/installer/steps/upload-job-artifacts.yml
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -1260,9 +1260,10 @@ extends:
           buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
           platforms:
             - windows_x86
+          helixQueuesTemplate: /eng/pipelines/installer/helix-queues-setup.yml
           jobParameters:
             nameSuffix: Installer_Build_And_Test
-            buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+            buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release
             dependsOnGlobalBuilds:
               - nameSuffix: CoreCLR_Libraries
                 buildConfig: release
@@ -1273,12 +1274,11 @@ extends:
                   artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release$(archiveExtension)
                   unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
                   displayName: 'unified artifacts'
-            enablePublishTestResults: true
-            testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+            timeoutInMinutes: 150
             postBuildSteps:
-              - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+              - template: /eng/pipelines/installer/helix.yml
                 parameters:
-                  name: $(osGroup)$(osSubgroup)_$(archType)
+                  creator: dotnet-bot
             condition:
               or(
                 eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
@@ -1292,9 +1292,10 @@ extends:
             - windows_x64
             - osx_x64
             - linux_x64
+          helixQueuesTemplate: /eng/pipelines/installer/helix-queues-setup.yml
           jobParameters:
             nameSuffix: Installer_Build_And_Test
-            buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
+            buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release
             dependsOnGlobalBuilds:
               - nameSuffix: CoreCLR_Libraries
                 buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
@@ -1305,12 +1306,11 @@ extends:
                   artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)$(archiveExtension)
                   unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
                   displayName: 'unified artifacts'
-            enablePublishTestResults: true
-            testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+            timeoutInMinutes: 150
             postBuildSteps:
-              - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+              - template: /eng/pipelines/installer/helix.yml
                 parameters:
-                  name: $(osGroup)$(osSubgroup)_$(archType)
+                  creator: dotnet-bot
             condition:
               or(
                 eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
diff --git a/src/installer/tests/HostActivation.Tests/NativeHosting/Ijwhost.cs b/src/installer/tests/HostActivation.Tests/NativeHosting/Ijwhost.cs
@@ -154,9 +154,17 @@ public SharedTestState()
                 string ijwhostName = "ijwhost.dll";
                 File.Copy(Path.Combine(RepoDirectoriesProvider.Default.HostArtifacts, ijwhostName), Path.Combine(folder, ijwhostName));
 
-                // Copy over the C++/CLI test library
+                // Copy over the C++/CLI test library and any dependencies
                 string ijwLibraryName = "ijw.dll";
                 File.Copy(Path.Combine(RepoDirectoriesProvider.Default.HostTestArtifacts, ijwLibraryName), Path.Combine(folder, ijwLibraryName));
+                string ijwDependencies = Path.Combine(RepoDirectoriesProvider.Default.HostTestArtifacts, "ijw-deps");
+                if (Directory.Exists(ijwDependencies))
+                {
+                    foreach(string file in Directory.GetFiles(ijwDependencies))
+                    {
+                        File.Copy(file, Path.Combine(folder, Path.GetFileName(file)));
+                    }
+                }
 
                 // Create a runtimeconfig.json for the C++/CLI test library
                 new RuntimeConfig(Path.Combine(folder, "ijw.runtimeconfig.json"))
diff --git a/src/installer/tests/TestUtils/TestContext.cs b/src/installer/tests/TestUtils/TestContext.cs
@@ -42,8 +42,8 @@ static TestContext()
             MicrosoftNETCoreAppVersion = GetTestContextVariable("MNA_VERSION");
             Tfm = GetTestContextVariable("MNA_TFM");
 
-            TestAssetsOutput = GetTestContextVariable("TEST_ASSETS_OUTPUT");
-            TestArtifactsPath = GetTestContextVariable("TEST_ARTIFACTS");
+            TestAssetsOutput = ResolveTestContextPath(GetTestContextVariable("TEST_ASSETS_OUTPUT"));
+            TestArtifactsPath = ResolveTestContextPath(GetTestContextVariable("TEST_ARTIFACTS"));
             Directory.CreateDirectory(TestArtifactsPath);
 
             // Create an empty global.json, so running tests from test artifacts is not affected
@@ -68,5 +68,23 @@ public static string GetTestContextVariable(string name)
 
             throw new ArgumentException($"Unable to find variable '{name}' in test context variable file '{_testContextVariableFilePath}'");
         }
+
+        private static string ResolveTestContextPath(string path)
+        {
+            // On macOS, /tmp/ is a symlink. Running apps out of it will resolve the path, so determine the resolved path here.
+            if (!OperatingSystem.IsMacOS())
+                return path;
+
+            string tmpPath = "/tmp/";
+            if (!path.StartsWith(tmpPath))
+                return path;
+
+            // No trailing slash in order to properly check the link target
+            DirectoryInfo tmp = new DirectoryInfo(tmpPath[..^1]);
+            if (tmp.LinkTarget == null)
+                return path;
+
+            return Path.Combine(tmp.ResolveLinkTarget(true).FullName, path[tmpPath.Length..]);
+        }
     }
 }
diff --git a/src/installer/tests/TestUtils/TestFileBackup.cs b/src/installer/tests/TestUtils/TestFileBackup.cs
@@ -76,28 +76,13 @@ public void Dispose()
                 // This is a known problem where the actual Delete call is not 100% synchronous
                 // the OS reports a success but the file/folder is not fully removed yet.
                 // So implement a simple retry with a short timeout.
-                IOException exception = null;
-                for (int retryCount = 5; retryCount > 0; retryCount--)
-                {
-                    try
+                RetryOnIOError(() =>
                     {
                         Directory.Delete(_backupPath, recursive: true);
-                        if (!Directory.Exists(_backupPath))
-                        {
-                            return;
-                        }
-                    }
-                    catch (IOException ex)
-                    {
-                        exception = ex;
-                    }
-
-                    System.Threading.Thread.Sleep(200);
-                }
-                
-                throw new Exception(
-                    $"Failed to delete the backup folder {_backupPath} even after retries.\r\n"
-                    + (exception == null ? "" : exception.ToString()));
+                        return !Directory.Exists(_backupPath);
+                    },
+                    $"Failed to delete the backup folder {_backupPath} even after retries."
+                );
             }
         }
 
@@ -110,8 +95,44 @@ private static void CopyOverDirectory(string source, string destination)
 
             foreach (string file in Directory.GetFiles(source))
             {
-                File.Copy(file, Path.Combine(destination, Path.GetFileName(file)), overwrite: true);
+                // Some files may still be reported as in use my the OS - for example immediately after
+                // process exit. Simple retry to separate this case from a file being intentionally locked.
+                RetryOnIOError(() =>
+                    {
+                        File.Copy(file, Path.Combine(destination, Path.GetFileName(file)), overwrite: true);
+                        return true;
+                    },
+                    $"Failed to restore file {Path.GetFileName(file)}"
+                );
             }
         }
+
+        private static void RetryOnIOError(Func<bool> action, string errorMessage, int maxRetries = 5)
+        {
+            IOException exception = null;
+            for (int i = 0; i < maxRetries; i++)
+            {
+                try
+                {
+                    if (action())
+                    {
+                        return;
+                    }
+                }
+                catch (IOException e) 
+                {
+                    exception = e;
+                }
+
+                System.Threading.Thread.Sleep(200);
+            }
+
+            throw new Exception(
+                $"""
+                {errorMessage}
+                {(exception == null ? "" : exception.ToString())}
+                """);
+
+        }
     }
 }
diff --git a/src/installer/tests/helixpublish.proj b/src/installer/tests/helixpublish.proj
@@ -1,11 +1,12 @@
-<Project Sdk="Microsoft.DotNet.Helix.Sdk">
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
 
   <PropertyGroup>
     <HelixArchitecture>$(TargetArchitecture)</HelixArchitecture>
     <HelixBuild Condition="'$(HelixBuild)' == ''">$(BUILD_BUILDNUMBER)</HelixBuild>
     <HelixBuild Condition="'$(HelixBuild)' == ''">default</HelixBuild>
     <HelixConfiguration>$(Configuration)</HelixConfiguration>
     <HelixType>test/host</HelixType>
+    <TestRunNamePrefix>host-$(TargetOS)-$(TargetArchitecture)-$(Configuration) @ </TestRunNamePrefix>
 
     <IncludeDotNetCli>true</IncludeDotNetCli>
     <DotNetCliPackageType>sdk</DotNetCliPackageType>
@@ -46,7 +47,7 @@
     </MSBuild>
     <ItemGroup>
       <HostTestProject>
-        <Command>dotnet test $([System.IO.Path]::GetFileName($(_TargetPath))) $(TestRunnerAdditionalArguments)</Command>
+        <Command>dotnet test $([System.IO.Path]::GetFileName($(_TargetPath))) $(TestRunnerAdditionalArguments) --logger trx --results-directory .</Command>
         <PayloadDirectory>$(_PayloadDirectory)</PayloadDirectory>
       </HostTestProject>
     </ItemGroup>
@@ -58,6 +59,7 @@
         <PayloadDirectory>%(HostTestProject.PayloadDirectory)</PayloadDirectory>
         <Command>%(HostTestProject.Command)</Command>
         <PreCommands>@(_HelixPreCommands)</PreCommands>
+        <Timeout>00:30:00</Timeout>
       </HelixWorkItem>
     </ItemGroup>
   </Target>
diff --git a/src/native/corehost/test/ijw/CMakeLists.txt b/src/native/corehost/test/ijw/CMakeLists.txt
@@ -10,3 +10,15 @@ remove_ijw_incompatible_target_options(ijw)
 add_ijw_msbuild_project_properties(ijw ijwhost)
 
 install_with_stripped_symbols(ijw TARGETS corehost_test)
+
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+  # Copy over the debug CRT so that it is available for test runs
+  file(
+    GLOB_RECURSE
+    DEBUG_CRT_FILES
+    $ENV{VCToolsRedistDir}/onecore/debug_nonredist/${ARCH_TARGET_NAME}/Microsoft.VC*.DebugCRT/vcruntime*d.dll
+    $ENV{VCToolsRedistDir}/onecore/debug_nonredist/${ARCH_TARGET_NAME}/Microsoft.VC*.DebugCRT/msvcp*d.dll
+    $ENV{ExtensionSdkDir}/Microsoft.UniversalCRT.Debug/$ENV{UCRTVersion}/Redist/Debug/${ARCH_TARGET_NAME}/ucrtbased.dll
+  )
+  install(FILES ${DEBUG_CRT_FILES} DESTINATION corehost_test/ijw-deps)
+endif ()
