Fix Windows device tests

- Some views need a few ms to finish animating
  The switch on WinUI needs to animate to "on" so we need to wait a few
  ms so that the control is ready for the screenshots/color comparisons.
- We now have too many tests for WinUI to handle
  We need to break up all the tests by category to avoid "running out of
  windows"

eng/devices/windows.cake

@@ -36,7 +36,6 @@ string DOTNET_PLATFORM = $"win10-x64";
 bool DEVICE_CLEANUP = Argument("cleanup", true);
 string certificateThumbprint = "";
 bool isPackagedTestRun = TEST_DEVICE.ToLower().Equals("packaged");
-bool isControlsProjectTestRun = PROJECT.FullPath.EndsWith("Controls.DeviceTests.csproj");
 
 // Certificate Common Name to use/generate (eg: CN=DotNetMauiTests)
 var certCN = Argument("commonname", "DotNetMAUITests");
@@ -250,61 +249,42 @@ Task("Test")
 		// Install the DeviceTests app
 		StartProcess("powershell", "Add-AppxPackage -Path \"" + MakeAbsolute(msixPath).FullPath + "\"");
 
-		if (isControlsProjectTestRun)
-		{
-			// Start the app once, this will trigger the discovery of the test categories
-			var startArgsInitial = "Start-Process shell:AppsFolder\\$((Get-AppxPackage -Name \"" + PACKAGEID + "\").PackageFamilyName)!App -ArgumentList \"" + testResultsFile + "\", \"-1\"";
-			StartProcess("powershell", startArgsInitial);
-
-			Information($"Waiting 10 seconds for process to finish...");
-			System.Threading.Thread.Sleep(10000);
-
-			var testCategoriesToRun = System.IO.File.ReadAllLines(testsToRunFile).Length;
+		// Start the app once, this will trigger the discovery of the test categories
+		var startArgsInitial = "Start-Process shell:AppsFolder\\$((Get-AppxPackage -Name \"" + PACKAGEID + "\").PackageFamilyName)!App -ArgumentList \"" + testResultsFile + "\", \"-1\"";
+		StartProcess("powershell", startArgsInitial);
 
-			for (int i = 0; i <= testCategoriesToRun; i++)
-			{
-				var startArgs = "Start-Process shell:AppsFolder\\$((Get-AppxPackage -Name \"" + PACKAGEID + "\").PackageFamilyName)!App -ArgumentList \"" + testResultsFile + "\", \"" + i + "\"";
-
-				Information(startArgs);
+		Information($"Waiting 10 seconds for process to finish...");
+		System.Threading.Thread.Sleep(10000);
 
-				// Start the DeviceTests app for packaged
-				StartProcess("powershell", startArgs);
+		var testCategoriesToRun = System.IO.File.ReadAllLines(testsToRunFile).Length;
 
-				Information($"Waiting 10 seconds for the next...");
-				System.Threading.Thread.Sleep(10000);
-			}
-		}
-		else
+		for (int i = 0; i <= testCategoriesToRun; i++)
 		{
-			var startArgs = "Start-Process shell:AppsFolder\\$((Get-AppxPackage -Name \"" + PACKAGEID + "\").PackageFamilyName)!App -ArgumentList \"" + testResultsFile + "\"";
+			var startArgs = "Start-Process shell:AppsFolder\\$((Get-AppxPackage -Name \"" + PACKAGEID + "\").PackageFamilyName)!App -ArgumentList \"" + testResultsFile + "\", \"" + i + "\"";
 
 			Information(startArgs);
 
 			// Start the DeviceTests app for packaged
 			StartProcess("powershell", startArgs);
+
+			Information($"Waiting 10 seconds for the next...");
+			System.Threading.Thread.Sleep(10000);
 		}
 	}
 	else
 	{
 		// Unpackaged process blocks the thread, so we can wait shorter for the results
 		waitForResultTimeoutInSeconds = 30;
 
-		if (isControlsProjectTestRun)
-		{
-			// Start the app once, this will trigger the discovery of the test categories
-			StartProcess(TEST_APP, testResultsFile + " -1");
+		// Start the app once, this will trigger the discovery of the test categories
+		StartProcess(TEST_APP, testResultsFile + " -1");
 
-			var testCategoriesToRun = System.IO.File.ReadAllLines(testsToRunFile).Length;
+		var testCategoriesToRun = System.IO.File.ReadAllLines(testsToRunFile).Length;
 
-			for (int i = 0; i <= testCategoriesToRun; i++)
-			{
-				// Start the DeviceTests app for unpackaged
-				StartProcess(TEST_APP, testResultsFile + " " + i);
-			}
-		}
-		else
+		for (int i = 0; i <= testCategoriesToRun; i++)
 		{
-			StartProcess(TEST_APP, testResultsFile);
+			// Start the DeviceTests app for unpackaged
+			StartProcess(TEST_APP, testResultsFile + " " + i);
 		}
 	}
 
@@ -318,49 +298,45 @@ Task("Test")
 			break;
 	}
 
-	// If we're running the Controls project, double-check if we have all test result files
-	// and if the categories we expected to run match the test result files
-	if (isControlsProjectTestRun)
+	// Double-check if we have all test result files and if the categories we expected to run match the test result files
+	var expectedCategories = System.IO.File.ReadAllLines(testsToRunFile);
+	var expectedCategoriesRanCount = expectedCategories.Length;
+	var actualResultFileCount = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml").Length;
+
+	while (actualResultFileCount < expectedCategoriesRanCount) {
+		actualResultFileCount = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml").Length;
+		System.Threading.Thread.Sleep(1000);
+		waited++;
+
+		Information($"Waiting {waited} additional second(s) for tests to finish...");
+		if (waited >= 30)
+			break;
+	}
+		
+	if (FileExists(testsToRunFile))
 	{
-		var expectedCategories = System.IO.File.ReadAllLines(testsToRunFile);
-		var expectedCategoriesRanCount = expectedCategories.Length;
-		var actualResultFileCount = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml").Length;
-
-		while (actualResultFileCount < expectedCategoriesRanCount) {
-			actualResultFileCount = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml").Length;
-			System.Threading.Thread.Sleep(1000);
-			waited++;
-
-			Information($"Waiting {waited} additional second(s) for tests to finish...");
-			if (waited >= 30)
-				break;
-		}
-			
-		if (FileExists(testsToRunFile))
-		{
-			DeleteFile(testsToRunFile);
-		}
+		DeleteFile(testsToRunFile);
+	}
 
-		// While the count should match exactly, if we get more files somehow we'll allow it
-		// If it's less, throw an exception to fail the pipeline.
-		if (actualResultFileCount < expectedCategoriesRanCount)
-		{
-			// Grab the category name from the file name
-			// Ex: "TestResults-com_microsoft_maui_controls_devicetests_Frame.xml" -> "Frame"
-			var actualFiles = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml");
-			var actualCategories = actualFiles.Select(x => x.Substring(0, x.Length - 4)   // Remove ".xml"
-					 										.Split('_').Last()).ToList();
+	// While the count should match exactly, if we get more files somehow we'll allow it
+	// If it's less, throw an exception to fail the pipeline.
+	if (actualResultFileCount < expectedCategoriesRanCount)
+	{
+		// Grab the category name from the file name
+		// Ex: "TestResults-com_microsoft_maui_controls_devicetests_Frame.xml" -> "Frame"
+		var actualFiles = System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml");
+		var actualCategories = actualFiles.Select(x => x.Substring(0, x.Length - 4)   // Remove ".xml"
+														.Split('_').Last()).ToList();
 
-			foreach (var category in expectedCategories)
+		foreach (var category in expectedCategories)
+		{
+			if (!actualCategories.Contains(category))
 			{
-				if (!actualCategories.Contains(category))
-				{
-					Error($"Error: missing test file result for {category}");
-				}
+				Error($"Error: missing test file result for {category}");
 			}
-
-			throw new Exception($"Expected test result files: {expectedCategoriesRanCount}, actual files: {actualResultFileCount}, some process(es) might have crashed.");
 		}
+
+		throw new Exception($"Expected test result files: {expectedCategoriesRanCount}, actual files: {actualResultFileCount}, some process(es) might have crashed.");
 	}
 
 	if(System.IO.Directory.GetFiles(testResultsPath, "TestResults-*.xml").Length == 0)

src/Core/tests/DeviceTests.Shared/MauiProgramDefaults.cs

@@ -45,28 +45,11 @@ public static MauiApp CreateMauiApp(List<Assembly> testAssemblies)
 					Assemblies = testAssemblies,
 				});
 
-#if WINDOWS
-			if (testAssemblies.Any(a => a.FullName.Contains("Controls.DeviceTests",
-				StringComparison.OrdinalIgnoreCase)))
-			{
-				appBuilder.UseControlsHeadlessRunner(new HeadlessRunnerOptions
-				{
-					RequiresUIContext = true,
-				});
-			}
-			else
-			{
-				appBuilder.UseHeadlessRunner(new HeadlessRunnerOptions
-				{
-					RequiresUIContext = true,
-				});
-			}
-#else
 			appBuilder.UseHeadlessRunner(new HeadlessRunnerOptions
 			{
 				RequiresUIContext = true,
 			});
-#endif
+
 			appBuilder.UseVisualRunner();
 
 			appBuilder.ConfigureContainer(new DefaultServiceProviderFactory(new ServiceProviderOptions

src/Core/tests/DeviceTests/Handlers/Element/ElementTests.cs

@@ -8,6 +8,7 @@
 
 namespace Microsoft.Maui.DeviceTests
 {
+	[Category(TestCategory.Element)]
 	public partial class ElementTests : CoreHandlerTestBase
 	{
 		[Fact]

src/Core/tests/DeviceTests/Handlers/SwipeView/SwipeViewHandlerTests.cs

@@ -6,5 +6,25 @@ namespace Microsoft.Maui.DeviceTests
 	[Category(TestCategory.SwipeView)]
 	public partial class SwipeViewHandlerTests : CoreHandlerTestBase<SwipeViewHandler, SwipeViewStub>
 	{
+		[Fact(DisplayName = "Background Initializes Correctly")]
+		public async Task BackgroundInitializesCorrectly()
+		{
+			var brush = new SolidPaintStub(Colors.Blue);
+
+			var label = new SwipeViewStub()
+			{
+				Background = brush,
+				Content = new LabelStub { Text = "Swipe Me" },
+				LeftItems =
+				{
+					new SwipeItemStub
+					{
+						Background = new SolidPaintStub(Colors.Red)
+					}
+				}
+			};
+
+			await ValidateHasColor(label, Colors.Blue);
+		}
 	}
 }

src/Core/tests/DeviceTests/Handlers/Switch/SwitchHandlerTests.Windows.cs

@@ -17,9 +17,9 @@ bool GetNativeIsOn(SwitchHandler switchHandler) =>
 			GetNativeSwitch(switchHandler).IsOn;
 
 		Task ValidateTrackColor(ISwitch switchStub, Color color, Action action = null, string updatePropertyValue = null) =>
-			ValidateHasColor(switchStub, color, action, updatePropertyValue: updatePropertyValue);
+			ValidateHasColor(switchStub, color, action, updatePropertyValue: updatePropertyValue, .01);
 
 		Task ValidateThumbColor(ISwitch switchStub, Color color, Action action = null, string updatePropertyValue = null) =>
-			ValidateHasColor(switchStub, color, action, updatePropertyValue: updatePropertyValue);
+			ValidateHasColor(switchStub, color, action, updatePropertyValue: updatePropertyValue, .01);
 	}
 }

src/Core/tests/DeviceTests/Handlers/Switch/SwitchHandlerTests.cs

@@ -48,10 +48,7 @@ public async Task TrackColorInitializesCorrectly(bool isToggled)
 				TrackColor = Colors.Red
 			};
 
-			await AttachAndRun(switchStub, async (handler) =>
-			{
-				await ValidateTrackColor(switchStub, Colors.Red);
-			});
+			await ValidateTrackColor(switchStub, Colors.Red);
 		}
 
 		[Fact(DisplayName = "Track Color Updates Correctly")]
@@ -61,13 +58,11 @@ public async Task TrackColorUpdatesCorrectly()
 			{
 				IsOn = true
 			};
-			await AttachAndRun(switchStub, async (handler) =>
-			{
-				await ValidateTrackColor(switchStub, Colors.Red, () => switchStub.TrackColor = Colors.Red);
-			});
+
+			await ValidateTrackColor(switchStub, Colors.Red, () => switchStub.TrackColor = Colors.Red);
 		}
 
-		[Fact(DisplayName = "ThumbColor Initializes Correctly", Skip = "There seems to be an issue, so disable for now: https://github.com/dotnet/maui/issues/1275")]
+		[Fact(DisplayName = "ThumbColor Initializes Correctly")]
 		public async Task ThumbColorInitializesCorrectly()
 		{
 			var switchStub = new SwitchStub()
@@ -92,11 +87,7 @@ public async Task NullThumbColorDoesntCrash()
 			await CreateHandlerAsync(switchStub);
 		}
 
-		[Fact(DisplayName = "Thumb Color Updates Correctly"
-#if WINDOWS
-			, Skip = "Failing on Windows"
-#endif
-			)]
+		[Fact(DisplayName = "Thumb Color Updates Correctly")]
 		public async Task ThumbColorUpdatesCorrectly()
 		{
 			var switchStub = new SwitchStub()

src/Core/tests/DeviceTests/Memory/MemoryTests.cs

@@ -14,6 +14,7 @@ namespace Microsoft.Maui.Handlers.Memory
 	/// which seems to make Android and WinUI happier. Low APIs on Android still have issues running via xHarness
 	/// which is why we only currently run these on API 30+
 	/// </summary>
+	[Category(TestCategory.Memory)]
 	[TestCaseOrderer("Microsoft.Maui.Handlers.Memory.MemoryTestOrdering", "Microsoft.Maui.Core.DeviceTests")]
 	public class MemoryTests : CoreHandlerTestBase, IClassFixture<MemoryTestFixture>
 	{

src/Core/tests/DeviceTests/TestCategory.cs

@@ -42,6 +42,7 @@ public static class TestCategory
 		public const string TextFormatting = "Formatting";
 		public const string TimePicker = "TimePicker";
 		public const string View = "View";
+		public const string Element = "Element";
 		public const string WebView = "WebView";
 		public const string Window = "Window";
 		public const string WindowOverlay = "WindowOverlay";

src/TestUtils/src/DeviceTests.Runners/AppHostBuilderExtensions.cs

@@ -30,26 +30,17 @@ public static MauiAppBuilder UseHeadlessRunner(this MauiAppBuilder appHostBuilde
 		{
 			appHostBuilder.Services.AddSingleton(options);
 
-#if __ANDROID__ || __IOS__ || MACCATALYST || WINDOWS
+#if __ANDROID__ || __IOS__ || MACCATALYST
 			appHostBuilder.Services.AddTransient(svc => new HeadlessTestRunner(
 					svc.GetRequiredService<HeadlessRunnerOptions>(),
 					svc.GetRequiredService<TestOptions>()));
-#endif
-
-			return appHostBuilder;
-		}
-
-#if WINDOWS
-		public static MauiAppBuilder UseControlsHeadlessRunner(this MauiAppBuilder appHostBuilder, HeadlessRunnerOptions options)
-		{
-			appHostBuilder.Services.AddSingleton(options);
-
-			appHostBuilder.Services.AddTransient(svc => new ControlsHeadlessTestRunner(
+#elif WINDOWS
+			appHostBuilder.Services.AddTransient(svc => new PerCategoryHeadlessTestRunner(
 					svc.GetRequiredService<HeadlessRunnerOptions>(),
 					svc.GetRequiredService<TestOptions>()));
+#endif
 
 			return appHostBuilder;
 		}
-#endif
 	}
 }