Misc test updates (backport #18387)

build/ci/.azure-devops-android-tests.yml

@@ -178,6 +178,8 @@ jobs:
 
   ## First tests run (https://github.com/unoplatform/uno/issues/6714)
   - bash: |
+      # always set execute bit, as git has a tendency to remove it
+      chmod +x $(build.sourcesdirectory)/build/test-scripts/android-uitest-run.sh
       $(build.sourcesdirectory)/build/test-scripts/android-uitest-run.sh
 
     displayName: Run Android Tests

build/test-scripts/android-uitest-run.sh

@@ -231,15 +231,19 @@ else
 
 	cd $UNO_TESTS_LOCAL_TESTS_FILE
 
+	# Response file for testing to avoid the command line length limitation
+	# new parameters must include the ":" to separate parameter options
+	# the response file contains only the filters, in order to get proper stderr
+	echo "--filter:\"$UNO_TESTS_FILTER\"" > tests.rsp
+
 	## Run NUnit tests
 	dotnet test \
 		-c Release \
 		-l:"console;verbosity=normal" \
 		--logger "nunit;LogFileName=$UNO_ORIGINAL_TEST_RESULTS" \
-		--filter "$UNO_TESTS_FILTER" \
 		--blame-hang-timeout 120m \
-		-v m || true
-
+		-v m \
+		@tests.rsp || true
 fi
 
 ## Dump the emulator's system log

build/test-scripts/ios-uitest-run.sh

@@ -287,7 +287,7 @@ if [ ! -f "$UNO_ORIGINAL_TEST_RESULTS" ]; then
 fi
 
 echo "Copying crash reports"
-cp -R ~/Library/Logs/DiagnosticReports/* $LOG_FILE_DIRECTORY
+cp -R ~/Library/Logs/DiagnosticReports/* $LOG_FILE_DIRECTORY || true
 
 pushd $BUILD_SOURCESDIRECTORY/src/Uno.NUnitTransformTool
 mkdir -p $(dirname ${UNO_TESTS_FAILED_LIST})

src/SamplesApp/UITests.Shared/Windows_UI_Xaml_Controls/Flyout/Flyout_ShowAt_Window_Content.xaml.cs

@@ -2,6 +2,7 @@
 using Microsoft.UI.Xaml;
 using Microsoft.UI.Xaml.Controls;
 using Microsoft.UI.Xaml.Media;
+using Uno.UI.RuntimeTests.Tests.Windows_UI_Xaml;
 
 // The Blank Page item template is documented at https://go.microsoft.com/fwlink/?LinkId=234238
 
@@ -16,6 +17,16 @@ public sealed partial class Flyout_ShowAt_Window_Content : Page
 		public Flyout_ShowAt_Window_Content()
 		{
 			this.InitializeComponent();
+
+#if HAS_UNO
+			XamlRoot _xamlRoot = null;
+			Loaded += (s, e) => _xamlRoot = XamlRoot;
+			Unloaded += (s, e) =>
+			{
+				// close all popups at the end of the test.
+				VisualTreeHelper.CloseAllPopups(_xamlRoot);
+			};
+#endif
 		}
 
 		private void ButtonButton_Click(object sender, RoutedEventArgs e)

src/Uno.UI.RuntimeTests/IntegrationTests/dxaml/controls/calendardatepicker/CalendarDatePickerIntegrationTests.cs

@@ -263,46 +263,50 @@ public async Task CanOpenCloseFlyoutBySettingIsCalendarOpen()
 		{
 			TestCleanupWrapper cleanup;
 
-			Grid rootPanel = null;
-			CalendarDatePickerHelper helper = new CalendarDatePickerHelper();
-			await helper.PrepareLoadedEvent();
-			Microsoft.UI.Xaml.Controls.CalendarDatePicker cp = await helper.GetCalendarDatePicker();
+			// This test may be unstable on iOS
+			await TestHelper.RetryAssert(async () =>
+			{
+				Grid rootPanel = null;
+				CalendarDatePickerHelper helper = new CalendarDatePickerHelper();
+				await helper.PrepareLoadedEvent();
+				Microsoft.UI.Xaml.Controls.CalendarDatePicker cp = await helper.GetCalendarDatePicker();
 
-			rootPanel = await CreateTestResources();
+				rootPanel = await CreateTestResources();
 
-			// load into visual tree
-			await RunOnUIThread(() =>
-			{
-				rootPanel.Children.Append(cp);
-			});
+				// load into visual tree
+				await RunOnUIThread(() =>
+				{
+					rootPanel.Children.Append(cp);
+				});
 
-			await helper.WaitForLoaded();
+				await helper.WaitForLoaded();
 
-			await TestServices.WindowHelper.WaitForIdle();
+				await TestServices.WindowHelper.WaitForIdle();
 
-			await helper.PrepareOpenedEvent();
+				await helper.PrepareOpenedEvent();
 
-			await RunOnUIThread(() =>
-			{
-				cp.IsCalendarOpen = true;
-			});
-			await helper.WaitForOpened();
+				await RunOnUIThread(() =>
+				{
+					cp.IsCalendarOpen = true;
+				});
+				await helper.WaitForOpened();
 
-			await helper.PrepareClosedEvent();
+				await helper.PrepareClosedEvent();
 
-			await RunOnUIThread(() =>
-			{
-				cp.IsCalendarOpen = false;
-			});
+				await RunOnUIThread(() =>
+				{
+					cp.IsCalendarOpen = false;
+				});
 
-			await helper.WaitForClosed();
+				await helper.WaitForClosed();
 
-			await RunOnUIThread(() =>
-			{
-				// disable CP to make sure input pane is not open during clean up.
-				cp.IsEnabled = false;
+				await RunOnUIThread(() =>
+				{
+					// disable CP to make sure input pane is not open during clean up.
+					cp.IsEnabled = false;
+				});
+				await TestServices.WindowHelper.WaitForIdle();
 			});
-			await TestServices.WindowHelper.WaitForIdle();
 		}
 
 		[TestMethod]

src/Uno.UI.RuntimeTests/IntegrationTests/dxaml/controls/commandbar/CommandBarIntegrationTests.cs

@@ -1364,13 +1364,18 @@ await RunOnUIThread(() =>
 			KeyboardHelper.Down(secondaryItemsPresenter);
 			await WindowHelper.WaitForIdle();
 
-			await RunOnUIThread(() =>
+			await TestHelper.RetryAssert(async () =>
 			{
-				var item = (AppBarButton)cmdBar.SecondaryCommands[0];
-				var transform = item.TransformToVisual(null);
-				var firstItemNewPosition = transform.TransformPoint(new Point(0, 0));
+				// This keyboard and animation driven, it may need a few retries to get the right position.
 
-				VERIFY_ARE_EQUAL(firstItemNewPosition, firstItemOriginalPosition);
+				await RunOnUIThread(() =>
+				{
+					var item = (AppBarButton)cmdBar.SecondaryCommands[0];
+					var transform = item.TransformToVisual(null);
+					var firstItemNewPosition = transform.TransformPoint(new Point(0, 0));
+
+					VERIFY_ARE_EQUAL(firstItemNewPosition, firstItemOriginalPosition);
+				});
 			});
 		}
 
@@ -3251,27 +3256,32 @@ await RunOnUIThread(() =>
 				VERIFY_ARE_EQUAL(primaryItemsControl.Visibility, Visibility.Collapsed);
 			});
 
-			LOG_OUTPUT("Now open and close the CommandBar.");
-			await OpenCommandBar(cmdBar, OpenMethod.Programmatic);
-			await CloseCommandBar(cmdBar);
-
-			await RunOnUIThread(() =>
+			// in some unknown conditions, the OpenCommandBar may not happen on android
+			// Retry a few times until it does.
+			await TestHelper.RetryAssert(async () =>
 			{
-				LOG_OUTPUT("The primary items control should still be collapsed.");
-				VERIFY_ARE_EQUAL(primaryItemsControl.Visibility, Visibility.Collapsed);
+				LOG_OUTPUT("Now open and close the CommandBar.");
+				await OpenCommandBar(cmdBar, OpenMethod.Programmatic);
+				await CloseCommandBar(cmdBar);
 
-				LOG_OUTPUT("Change the width of the CommandBar back to 600.  The AppBarButton should be moved back from the overflow.");
-				expectItemsAdded = false;
-				cmdBar.Width = 600;
-			});
+				await RunOnUIThread(() =>
+				{
+					LOG_OUTPUT("The primary items control should still be collapsed.");
+					VERIFY_ARE_EQUAL(primaryItemsControl.Visibility, Visibility.Collapsed);
 
-			await dynamicOverflowItemsChangingEvent.WaitForDefault();
-			await WindowHelper.WaitForIdle();
+					LOG_OUTPUT("Change the width of the CommandBar back to 600.  The AppBarButton should be moved back from the overflow.");
+					expectItemsAdded = false;
+					cmdBar.Width = 600;
+				});
 
-			await RunOnUIThread(() =>
-			{
-				LOG_OUTPUT("The primary items control should now be visible since the AppBarButton is back in it.");
-				VERIFY_ARE_EQUAL(primaryItemsControl.Visibility, Visibility.Visible);
+				await dynamicOverflowItemsChangingEvent.WaitForDefault();
+				await WindowHelper.WaitForIdle();
+
+				await RunOnUIThread(() =>
+				{
+					LOG_OUTPUT("The primary items control should now be visible since the AppBarButton is back in it.");
+					VERIFY_ARE_EQUAL(primaryItemsControl.Visibility, Visibility.Visible);
+				});
 			});
 		}
 
@@ -5226,7 +5236,11 @@ private async Task CloseCommandBar(CommandBar cmdBar)
 			var closedRegistration = CreateSafeEventRegistration<CommandBar, EventHandler<object>>("Closed");
 			closedRegistration.Attach(cmdBar, (s, e) => closedEvent.Set());
 
-			await RunOnUIThread(() => cmdBar.IsOpen = false);
+			await RunOnUIThread(() =>
+			{
+				Assert.IsTrue(cmdBar.IsOpen, "Command bar is not opened");
+				cmdBar.IsOpen = false;
+			});
 			await closedEvent.WaitForDefault();
 			await WindowHelper.WaitForIdle();
 		}

src/Uno.UI.RuntimeTests/Tests/Microsoft_UI_Xaml_Controls/Given_WebView2.cs

@@ -75,14 +75,14 @@ public async Task When_NavigateToString()
 		webView.NavigationCompleted += (s, e) => navigationDone = true;
 		webView.Source = uri;
 		Assert.IsNotNull(webView.Source);
-		await TestServices.WindowHelper.WaitFor(() => navigationStarting, 3000);
-		await TestServices.WindowHelper.WaitFor(() => navigationDone, 3000);
+		await TestServices.WindowHelper.WaitFor(() => navigationStarting, 10000);
+		await TestServices.WindowHelper.WaitFor(() => navigationDone, 10000);
 		Assert.IsNotNull(webView.Source);
 		navigationStarting = false;
 		navigationDone = false;
 		webView.NavigateToString("<html></html>");
-		await TestServices.WindowHelper.WaitFor(() => navigationStarting, 3000);
-		await TestServices.WindowHelper.WaitFor(() => navigationDone, 3000);
+		await TestServices.WindowHelper.WaitFor(() => navigationStarting, 10000);
+		await TestServices.WindowHelper.WaitFor(() => navigationDone, 10000);
 		Assert.AreEqual(new Uri("about:blank"), webView.Source);
 	}
 
@@ -106,24 +106,24 @@ public async Task When_GoBack()
 		Assert.IsFalse(webView.CanGoForward);
 
 		webView.NavigationCompleted += (sender, e) => navigated = true;
-		webView.CoreWebView2.Navigate("https://example.com/1");
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		webView.CoreWebView2.Navigate("https://uno-assets.platform.uno/tests/docs/WebView_NavigateToAnchor.html");
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsFalse(webView.CoreWebView2.CanGoBack);
 		Assert.IsFalse(webView.CanGoBack);
 		Assert.IsFalse(webView.CoreWebView2.CanGoForward);
 		Assert.IsFalse(webView.CanGoForward);
 
 		navigated = false;
-		webView.CoreWebView2.Navigate("https://example.com/2");
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		webView.CoreWebView2.Navigate("https://platform.uno");
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsTrue(webView.CoreWebView2.CanGoBack);
 		Assert.IsTrue(webView.CanGoBack);
 
 		navigated = false;
 		webView.GoBack();
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsFalse(webView.CoreWebView2.CanGoBack);
 		Assert.IsFalse(webView.CanGoBack);

src/Uno.UI.RuntimeTests/Tests/Windows_UI_Xaml_Controls/Given_Image.cs

@@ -356,13 +356,20 @@ public async Task When_Image_Source_Nullify()
 			{
 				Width = 100,
 				Height = 100,
-				Source = new BitmapImage(new Uri("ms-appx:///Assets/square100.png")),
 				Stretch = Stretch.Fill
 			};
 
+			var imageOpened = false;
+			SUT.ImageOpened += (_, _) => imageOpened = true;
+
+			SUT.Source = new BitmapImage(new Uri("ms-appx:///Assets/square100.png"));
+
 			parent.Child = SUT;
 			WindowHelper.WindowContent = parent;
 			await WindowHelper.WaitForLoaded(parent);
+
+			await TestServices.WindowHelper.WaitFor(() => imageOpened, 3000);
+
 			var result = await TakeScreenshot(parent);
 
 			var sample = parent.GetRelativeCoords(SUT);

src/Uno.UI.RuntimeTests/Tests/Windows_UI_Xaml_Controls/Given_WebView.cs

@@ -141,21 +141,21 @@ public async Task When_GoBack()
 		Assert.IsFalse(webView.CanGoForward);
 
 		webView.NavigationCompleted += (sender, e) => navigated = true;
-		webView.Navigate(new Uri("https://example.com/1"));
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		webView.Navigate(new Uri("https://uno-assets.platform.uno/tests/docs/WebView_NavigateToAnchor.html"));
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsFalse(webView.CanGoBack);
 		Assert.IsFalse(webView.CanGoForward);
 
 		navigated = false;
-		webView.Navigate(new Uri("https://example.com/2"));
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		webView.Navigate(new Uri("https://platform.uno"));
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsTrue(webView.CanGoBack);
 
 		navigated = false;
 		webView.GoBack();
-		await TestServices.WindowHelper.WaitFor(() => navigated, 3000);
+		await TestServices.WindowHelper.WaitFor(() => navigated, 10000);
 
 		Assert.IsFalse(webView.CanGoBack);
 		Assert.IsTrue(webView.CanGoForward);

src/Uno.UI.RuntimeTests/Tests/Windows_UI_Xaml_Media/Given_ImageBrush.cs

@@ -80,36 +80,40 @@ public async Task When_Stretch(Stretch stretch, bool useRectangle)
 			WindowHelper.WindowContent = SUT;
 			await WindowHelper.WaitForLoaded(SUT);
 
-			float BorderOffset =
+			// Retry is required because the brush even is not always raised
+			await TestHelper.RetryAssert(async () =>
+			{
+				float BorderOffset =
 #if __SKIA__
-				useRectangle ? 4 : 7;
+					useRectangle ? 4 : 7;
 #elif __IOS__
-				6;
+					6;
 #else
-				3;
+					3;
 #endif
-			float width = (float)SUT.Width, height = (float)SUT.Height;
-			float centerX = width / 2, centerY = height / 2;
-			var expectations = stretch switch
-			{
-				// All edges are red-ish
-				Stretch.Fill => (Top: Redish, Bottom: Redish, Left: Redish, Right: Redish),
-				// Top and bottom are red-ish. Left and right are yellow-ish
-				Stretch.UniformToFill => (Top: Redish, Bottom: Redish, Left: Yellowish, Right: Yellowish),
-				// Top and bottom are same as background. Left and right are red-ish
-				Stretch.Uniform => (Top: Transparent, Bottom: Transparent, Left: Redish, Right: Redish),
-				// Everything is green-ish
-				Stretch.None => (Top: Greenish, Bottom: Greenish, Left: Greenish, Right: Greenish),
+				float width = (float)SUT.Width, height = (float)SUT.Height;
+				float centerX = width / 2, centerY = height / 2;
+				var expectations = stretch switch
+				{
+					// All edges are red-ish
+					Stretch.Fill => (Top: Redish, Bottom: Redish, Left: Redish, Right: Redish),
+					// Top and bottom are red-ish. Left and right are yellow-ish
+					Stretch.UniformToFill => (Top: Redish, Bottom: Redish, Left: Yellowish, Right: Yellowish),
+					// Top and bottom are same as background. Left and right are red-ish
+					Stretch.Uniform => (Top: Transparent, Bottom: Transparent, Left: Redish, Right: Redish),
+					// Everything is green-ish
+					Stretch.None => (Top: Greenish, Bottom: Greenish, Left: Greenish, Right: Greenish),
 
-				_ => throw new ArgumentOutOfRangeException($"unexpected stretch: {stretch}"),
-			};
+					_ => throw new ArgumentOutOfRangeException($"unexpected stretch: {stretch}"),
+				};
 
-			var bitmap = await UITestHelper.ScreenShot(SUT);
+				var bitmap = await UITestHelper.ScreenShot(SUT);
 
-			ImageAssert.HasColorAt(bitmap, centerX, BorderOffset, expectations.Top, tolerance: 28);
-			ImageAssert.HasColorAt(bitmap, centerX, height - BorderOffset, expectations.Bottom, tolerance: 28);
-			ImageAssert.HasColorAt(bitmap, BorderOffset, centerY, expectations.Left, tolerance: 28);
-			ImageAssert.HasColorAt(bitmap, width - BorderOffset, centerY, expectations.Right, tolerance: 28);
+				ImageAssert.HasColorAt(bitmap, centerX, BorderOffset, expectations.Top, tolerance: 28);
+				ImageAssert.HasColorAt(bitmap, centerX, height - BorderOffset, expectations.Bottom, tolerance: 28);
+				ImageAssert.HasColorAt(bitmap, BorderOffset, centerY, expectations.Left, tolerance: 28);
+				ImageAssert.HasColorAt(bitmap, width - BorderOffset, centerY, expectations.Right, tolerance: 28);
+			}, 10);
 		}
 
 #if __SKIA__