[browser] Use standard xunit test runner on multithreaded runtime

eng/testing/tests.browser.targets

@@ -91,7 +91,7 @@
     <WasmTestAppArgs Condition="'$(_XUnitBackgroundExec)' == 'true'">$(WasmTestAppArgs) -backgroundExec</WasmTestAppArgs>
     <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs)</_AppArgs>
 
-    <WasmXHarnessMonoArgs Condition="'$(XunitShowProgress)' == 'true'">$(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=1</WasmXHarnessMonoArgs>
+    <WasmXHarnessMonoArgs Condition="'$(XunitShowProgress)' == 'true'">$(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
     <!-- help unit test with PlatformDetection.IsThreadingSupported via IsBrowserThreadingSupported env variable -->
     <WasmXHarnessMonoArgs Condition="'$(WasmEnableThreads)' == 'true'">$(WasmXHarnessMonoArgs) --setenv=IsBrowserThreadingSupported=true</WasmXHarnessMonoArgs>
   </PropertyGroup>
@@ -108,9 +108,11 @@
     <_XHarnessArgs Condition="'$(_UseWasmSymbolicator)' == 'true'" >$(_XHarnessArgs) --symbolicator WasmSymbolicator.dll,Microsoft.WebAssembly.Internal.SymbolicatorWrapperForXHarness</_XHarnessArgs>
     <_XHarnessArgs Condition="'$(_WasmBrowserPathForTests)' != ''" >$(_XHarnessArgs) &quot;--browser-path=$(_WasmBrowserPathForTests)&quot;</_XHarnessArgs>
     <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) &quot;--timeout=$(WasmXHarnessTestsTimeout)&quot;</_XHarnessArgs>
+    <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''"    >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity)</_XHarnessArgs><!-- Trace, Debug, Information (default), Warning, Error, Critical -->
     <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''"      >$(_XHarnessArgs) $(WasmXHarnessArgsCli)</_XHarnessArgs>
 
-    <!-- There two flavors of WasmXHarnessArgs and WasmXHarnessMonoArgs, one is MSBuild property and the other is environment variable -->
+    <_AppArgs Condition="'$(WasmEnableThreads)' == 'true'">$(_AppArgs) -threads</_AppArgs>
+    <!-- There are two flavors of WasmXHarnessArgs and WasmXHarnessMonoArgs, one is MSBuild property and the other is environment variable -->
     <RunScriptCommand Condition="'$(OS)' != 'Windows_NT'">$HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) %24WasmTestAppArgs</RunScriptCommand>
     <RunScriptCommand Condition="'$(OS)' == 'Windows_NT'">%HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs%  -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_AppArgs) %WasmTestAppArgs%</RunScriptCommand>
   </PropertyGroup>
@@ -147,6 +149,12 @@
     <WasmNestedPublishAppDependsOn>PrepareForWasmBuildApp;$(WasmNestedPublishAppDependsOn)</WasmNestedPublishAppDependsOn>
   </PropertyGroup>
 
+  <Target Name="IncludeTestAssemblyInVFS" BeforeTargets="PrepareForWasmBuildApp">
+    <ItemGroup>
+      <WasmFilesToIncludeFromPublishDir Include="$(AssemblyName).dll" Exclude="@(WasmFilesToIncludeFromPublishDir)" />
+    </ItemGroup>
+  </Target>
+
   <Target Name="PrepareForWasmBuildApp">
     <PropertyGroup>
       <WasmAppDir>$(BundleDir)</WasmAppDir>
@@ -197,7 +205,7 @@
       <!-- Include files specified by test projects from publish dir -->
       <WasmFilesToIncludeInFileSystem
               Include="$(PublishDir)%(WasmFilesToIncludeFromPublishDir.Identity)"
-              TargetPath="%(WasmFilesToIncludeFromPublishDir.Identity)"
+              TargetPath="%(WasmFilesToIncludeFromPublishDir.TargetDir)/%(WasmFilesToIncludeFromPublishDir.Identity)"
               Condition="'%(WasmFilesToIncludeFromPublishDir.Identity)' != ''" />
     </ItemGroup>
 

eng/testing/tests.wasi.targets

@@ -31,7 +31,7 @@
     <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) -- $(WasmTestAppArgs)</_AppArgs>
 
     <!-- FIXME: wasttime specific param name -->
-    <WasmXHarnessMonoArgs Condition="'$(XunitShowProgress)' == 'true'">$(WasmXHarnessMonoArgs) --env=XHARNESS_LOG_TEST_START=1</WasmXHarnessMonoArgs>
+    <WasmXHarnessMonoArgs Condition="'$(XunitShowProgress)' == 'true'">$(WasmXHarnessMonoArgs) --env=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(RunScriptCommand)' == ''">
@@ -51,7 +51,7 @@
 
     <_InvariantGlobalization Condition="'$(InvariantGlobalization)' == 'true'">--env=DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true</_InvariantGlobalization>
 
-    <!-- There two flavors of WasmXHarnessArgs and WasmXHarnessMonoArgs, one is MSBuild property and the other is environment variable -->
+    <!-- There are two flavors of WasmXHarnessArgs and WasmXHarnessMonoArgs, one is MSBuild property and the other is environment variable -->
     <RunScriptCommand Condition="'$(OS)' != 'Windows_NT'">$HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_InvariantGlobalization) %24_InvariantGlobalization  $(_AppArgs) %24WasmTestAppArgs</RunScriptCommand>
     <RunScriptCommand Condition="'$(OS)' == 'Windows_NT'">%HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs%  -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_InvariantGlobalization) %_InvariantGlobalization% $(_AppArgs) %WasmTestAppArgs%</RunScriptCommand>
   </PropertyGroup>

src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs

@@ -5,52 +5,56 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.DotNet.XHarness.TestRunners.Common;
-
 using Microsoft.DotNet.XHarness.TestRunners.Xunit;
 
-public class SimpleWasmTestRunner : WasmApplicationEntryPoint
+public class WasmTestRunner : WasmApplicationEntryPoint
 {
+    // TODO: Set max threads for run in parallel
+    // protected override int? MaxParallelThreads => RunInParallel ? 8 : base.MaxParallelThreads;
+
     public static async Task<int> Main(string[] args)
     {
         if (args.Length == 0)
         {
-            Console.WriteLine ($"No args given");
+            Console.WriteLine($"No args given");
             return -1;
         }
 
-        var testAssembly = args[0];
+        var runner = new WasmTestRunner();
+
+        runner.TestAssembly = args[0];
+
         var excludedTraits = new List<string>();
         var includedTraits = new List<string>();
         var includedNamespaces = new List<string>();
         var includedClasses = new List<string>();
         var includedMethods = new List<string>();
         var backgroundExec = false;
         var untilFailed = false;
-        var minimumLogLevel = MinimumLogLevel.Info;
 
         for (int i = 1; i < args.Length; i++)
         {
             var option = args[i];
             switch (option)
             {
                 case "-notrait":
-                    excludedTraits.Add (args[i + 1]);
+                    excludedTraits.Add(args[i + 1]);
                     i++;
                     break;
                 case "-trait":
-                    includedTraits.Add (args[i + 1]);
+                    includedTraits.Add(args[i + 1]);
                     i++;
                     break;
                 case "-namespace":
-                    includedNamespaces.Add (args[i + 1]);
+                    includedNamespaces.Add(args[i + 1]);
                     i++;
                     break;
                 case "-class":
-                    includedClasses.Add (args[i + 1]);
+                    includedClasses.Add(args[i + 1]);
                     i++;
                     break;
                 case "-method":
-                    includedMethods.Add (args[i + 1]);
+                    includedMethods.Add(args[i + 1]);
                     i++;
                     break;
                 case "-backgroundExec":
@@ -59,25 +63,26 @@ public static async Task<int> Main(string[] args)
                 case "-untilFailed":
                     untilFailed = true;
                     break;
+                case "-threads":
+                    runner.IsThreadless = false;
+                    // TODO: Enable run in parallel
+                    // runner.RunInParallel = true;
+                    // Console.WriteLine($"Running in parallel with {runner.MaxParallelThreads} threads.");
+                    break;
                 case "-verbosity":
-                    minimumLogLevel = Enum.Parse<MinimumLogLevel>(args[i + 1]);
+                    runner.MinimumLogLevel = Enum.Parse<MinimumLogLevel>(args[i + 1]);
                     i++;
                     break;
                 default:
                     throw new ArgumentException($"Invalid argument '{option}'.");
             }
         }
 
-        var runner = new SimpleWasmTestRunner()
-        {
-            TestAssembly = testAssembly,
-            ExcludedTraits = excludedTraits,
-            IncludedTraits = includedTraits,
-            IncludedNamespaces = includedNamespaces,
-            IncludedClasses = includedClasses,
-            IncludedMethods = includedMethods,
-            MinimumLogLevel = minimumLogLevel
-        };
+        runner.ExcludedTraits = excludedTraits;
+        runner.IncludedTraits = includedTraits;
+        runner.IncludedNamespaces = includedNamespaces;
+        runner.IncludedClasses = includedClasses;
+        runner.IncludedMethods = includedMethods;
 
         if (OperatingSystem.IsBrowser())
         {

src/libraries/System.Collections.Immutable/tests/System.Collections.Immutable.Tests.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(TargetOS)' == 'browser'">
-    <WasmXHarnessMonoArgs>--setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
     <!-- This WASM test is problematic and slow right now. This sets the xharness timeout but there is also override in sendtohelix-browser.targets -->
     <WasmXHarnessTestsTimeout>01:15:00</WasmXHarnessTestsTimeout>
   </PropertyGroup>

src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj

@@ -24,7 +24,7 @@
     <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
     <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
     <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
-    <WasmXHarnessMonoArgs>--setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
     <!-- This WASM test is problematic and slow right now. This sets the xharness timeout but there is also override in sendtohelix-browser.targets -->
     <WasmXHarnessTestsTimeout>01:15:00</WasmXHarnessTestsTimeout>
   </PropertyGroup>

src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj

@@ -13,7 +13,7 @@
     <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
     <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
     <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
-    <WasmXHarnessMonoArgs>--setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
 
     <!-- This WASM test is problematic and slow right now. This sets the xharness timeout but there is also override in sendtohelix-browser.targets -->
     <WasmXHarnessTestsTimeout>01:15:00</WasmXHarnessTestsTimeout>

src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj

@@ -14,7 +14,7 @@
     <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
     <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
     <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
-    <WasmXHarnessMonoArgs>--setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
   </PropertyGroup>
 
   <ItemGroup>

src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj

@@ -13,7 +13,7 @@
     <!-- Use following lines to write the generated files to disk. -->
     <EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
     <!-- to see timing and which test aborted the runtime -->
-    <WasmXHarnessMonoArgs>$(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
   </PropertyGroup>
   <!-- Make debugging easier -->
   <PropertyGroup Condition="'$(Configuration)' == 'Debug'">

src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/DisabledFileLockingTests/System.IO.FileSystem.DisabledFileLocking.Tests.csproj

@@ -7,6 +7,9 @@
 
     <WasmXHarnessMonoArgs>--working-dir=/test-dir</WasmXHarnessMonoArgs>
   </PropertyGroup>
+  <ItemGroup Condition="'$(TargetOS)' == 'browser'">
+    <WasmFilesToIncludeFromPublishDir Include="$(AssemblyName).dll" TargetDir="test-dir" />
+  </ItemGroup>
   <ItemGroup>
     <Compile Include="DisabledFileLockingSwitchTests.cs" />
     <Compile Include="..\FSAssert.cs" />

src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/System.IO.FileSystem.Tests.csproj

@@ -6,6 +6,9 @@
 
     <WasmXHarnessMonoArgs>--working-dir=/test-dir</WasmXHarnessMonoArgs>
   </PropertyGroup>
+  <ItemGroup Condition="'$(TargetOS)' == 'browser'">
+    <WasmFilesToIncludeFromPublishDir Include="$(AssemblyName).dll" TargetDir="test-dir" />
+  </ItemGroup>
   <ItemGroup>
     <Compile Include="Base\BaseGetSetAttributes.cs" />
     <Compile Include="Base\BaseGetSetTimes.cs" />

src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj

@@ -3,6 +3,7 @@
     <TestRuntime>true</TestRuntime>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
     <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
+    <XunitShowProgress>true</XunitShowProgress>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="Helpers.cs" />

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj

@@ -18,7 +18,7 @@
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
   </PropertyGroup>
   <PropertyGroup Condition="'$(TargetOS)' == 'browser'">
-    <WasmXHarnessMonoArgs>--setenv=XHARNESS_LOG_TEST_START=true</WasmXHarnessMonoArgs>
+    <XunitShowProgress>true</XunitShowProgress>
     <!-- This WASM test is problematic and slow right now. This sets the xharness timeout but there is also override in sendtohelix-browser.targets -->
     <WasmXHarnessTestsTimeout>01:15:00</WasmXHarnessTestsTimeout>
   </PropertyGroup>

src/mono/browser/README.md

@@ -117,10 +117,24 @@ The wrapper script used to actually run these tests, accepts:
 
 ### Using a local build of xharness
 
+XHarness consists of two pieces for WASM
+
+#### 1. CLI/host
+
 * set `XHARNESS_CLI_PATH=/path/to/xharness/artifacts/bin/Microsoft.DotNet.XHarness.CLI/Debug/net7.0/Microsoft.DotNet.XHarness.CLI.dll`
 
 **Note:** Additional msbuild arguments can be passed with: `make ..  MSBUILD_ARGS="/p:a=b"`
 
+#### 2. Test runner running inside of the browser
+
+All library tests are hosted by `WasmTestRunner.csproj`. The project references XHarness nuget for running tests using Xunit. To make changes and iterate quickly
+
+- Add property `<RestoreAdditionalProjectSources>$(RestoreAdditionalProjectSources);LOCAL_CLONE_OF_XHARNESS\artifacts\packages\Debug\Shipping</RestoreAdditionalProjectSources>` in `WasmTestRunner.csproj`.
+- Set environment variable in your terminal `$env:NUGET_PACKAGES="$pwd\.nuget"` (so that nuget packages are restored to local folder `.nuget`)
+- Run "Pack" in the XHarness solution in Visual Studio on `Microsoft.DotNet.XHarness.TestRunners.Common` or `Microsoft.DotNet.XHarness.TestRunners.Xunit` based on your changes (it will generate a nuget package in `LOCAL_CLONE_OF_XHARNESS\artifacts\packages\Debug\Shipping`).
+- Build WasmTestRunner `.\dotnet.cmd build -c Debug .\src\libraries\Common\tests\WasmTestRunner\WasmTestRunner.csproj`.
+- If you need to iterate, delete Xunit or Common nuget cache `rm -r .\.nuget\microsoft.dotnet.xharness.testrunners.xunit\` or `rm -r .\.nuget\microsoft.dotnet.xharness.testrunners.common\`.
+
 ### Symbolicating traces
 
 Exceptions thrown after the runtime starts get symbolicating from js itself. Exceptions before that, like asserts containing native traces get symbolicated by xharness using `src/mono/wasm/symbolicator`.

src/mono/wasm/sln/WasmBuild.sln

@@ -25,7 +25,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplyUpdateReferencedAssemb
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.Sdk.WebAssembly.Pack.Tasks", "..\..\..\tasks\Microsoft.NET.Sdk.WebAssembly.Pack.Tasks\Microsoft.NET.Sdk.WebAssembly.Pack.Tasks.csproj", "{5EEC2925-2021-4830-B7E9-72BB8B2C283D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wasi.Build.Tests", "..\..\wasi\Wasi.Build.Tests\Wasi.Build.Tests.csproj", "{3A3AEAE5-0110-45D3-89B0-B82AC430535C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wasi.Build.Tests", "..\..\wasi\Wasi.Build.Tests\Wasi.Build.Tests.csproj", "{3A3AEAE5-0110-45D3-89B0-B82AC430535C}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WasmTestRunner", "..\..\..\libraries\Common\tests\WasmTestRunner\WasmTestRunner.csproj", "{2BBE4AA8-5424-44AB-933C-66554B688872}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -81,6 +83,10 @@ Global
 		{3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2BBE4AA8-5424-44AB-933C-66554B688872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2BBE4AA8-5424-44AB-933C-66554B688872}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2BBE4AA8-5424-44AB-933C-66554B688872}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2BBE4AA8-5424-44AB-933C-66554B688872}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE