[ci] Improvements for emulator test jobs

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

@@ -30,11 +30,10 @@ public class StartAndroidEmulator : Task
 		public                  string          AvdManagerHome  {get; set;}
 		public                  string          Port            {get; set;}
 		public                  string          ImageName       {get; set;} = "XamarinAndroidTestRunner64";
-		public			string		Arguments	{get; set;}
+		public                  string          Arguments       {get; set;}
 		public                  string          ToolPath        {get; set;}
 		public                  string          ToolExe         {get; set;}
 		public                  string          LogcatFile      {get; set;}
-		public                  bool            ShowWindow      {get; set;} = true;
 
 		public override bool Execute ()
 		{
@@ -69,10 +68,9 @@ void Run (string emulator)
 			if (emulator == null)
 				return;
 
-			var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
-			var showWindow = ShowWindow ? "" : " -no-window";
-			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-boot-anim -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {showWindow}{port} -avd {ImageName}";
-			Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
+			var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}";
+			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}";
+			Log.LogMessage ($"Tool {emulator} execution started with arguments: {arguments}");
 			var psi = new ProcessStartInfo () {
 				FileName                = emulator,
 				Arguments               = arguments,

build-tools/automation/yaml-templates/run-systemapp-tests.yaml → build-tools/automation/yaml-templates/run-emulator-tests.yaml

@@ -1,12 +1,17 @@
-# Runs TimeZoneInfo tests against an emulator running on macOS
+# Runs a test or set of tests on an emulator running on macOS
+
+parameters:
+  emulatorMSBuildArgs: ''
+  jobName: CheckTimeZoneInfoIsCorrectNode1
+  jobTimeout: 360
+  testSteps: []
 
 jobs:
-  - job: mac_systemapp_tests
-    displayName: System App Emulator Tests
+  - job: mac_${{ parameters.jobName }}_tests
+    displayName: ${{ parameters.jobName }} Emulator Tests
     pool:
       vmImage: $(HostedMacImage)
-    timeoutInMinutes: 120
-    cancelTimeoutInMinutes: 5
+    timeoutInMinutes: ${{ parameters.jobTimeout }}
     workspace:
       clean: all
     steps:
@@ -27,14 +32,9 @@ jobs:
         solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
         configuration: $(XA.Build.Configuration)
         msbuildArguments: >-
-          /t:AcquireAndroidTarget /p:TestEmulatorArguments=-writable-system /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
+          /t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
 
-    - template: run-nunit-tests.yaml
-      parameters:
-        testRunTitle: System App On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll
-        nunitConsoleExtraArgs: --where "cat == SystemApplication"
-        testResultsFile: TestResult-SystemApp--$(XA.Build.Configuration).xml
+    - ${{ parameters.testSteps }}
 
     - task: MSBuild@1
       displayName: shut down emulator
@@ -48,6 +48,6 @@ jobs:
 
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - System App With Emulator - macOS
+        artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS
 
-    - template: fail-on-issue.yaml
+    - template: fail-on-issue.yaml

build-tools/scripts/NUnitReferences.projitems

@@ -4,10 +4,6 @@
     <PackageReference Include="NUnit"               Version="3.13.3" />
     <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleVersion)" />
     <PackageReference Include="NUnit3TestAdapter"   Version="4.3.1" />
-  </ItemGroup>
-  <!-- Required packages for .NET Core -->
-  <ItemGroup Condition=" '$(TargetFramework)' != 'net472' and '$(TargetFramework)' != 'netstandard2.0' ">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
-    <PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
   </ItemGroup>
 </Project>

build-tools/scripts/TestApks.targets

@@ -21,12 +21,13 @@
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' and '$(HostOS)' == 'Darwin' and  '$(HostOSArchitecture)' == 'Arm64' ">arm64-v8a</TestAvdAbi>
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' ">x86_64</TestAvdAbi>
     <TestAvdType Condition=" '$(TestAvdType)' == '' ">default</TestAvdType>
+    <TestAvdForceCreation Condition=" '$(TestAvdForceCreation)' == '' ">true</TestAvdForceCreation>
+    <TestAvdShowWindow Condition=" '$(TestAvdShowWindow)' == '' and '$(RunningOnCI)' == 'true' ">false</TestAvdShowWindow>
+    <TestAvdExtraBootArgs Condition=" '$(TestAvdShowWindow)' == 'false' ">-no-window -no-boot-anim $(TestAvdExtraBootArgs)</TestAvdExtraBootArgs>
     <TestDeviceName Condition=" '$(TestDeviceName)' == '' ">pixel_4</TestDeviceName>
     <SdkManagerImageName Condition=" '$(SdkManagerImageName)' == '' ">system-images;android-$(TestAvdApiLevel);$(TestAvdType);$(TestAvdAbi)</SdkManagerImageName>
     <TestAvdName>XamarinAndroidTestRunner$(TestAvdApiLevel)-$(TestAvdAbi)</TestAvdName>
     <_AdbEmulatorPort>5570</_AdbEmulatorPort>
-    <_AdbEmulatorShowWindow Condition=" '$(RunningOnCI)' == 'True' And '$(_AdbEmulatorShowWindow)' == '' ">False</_AdbEmulatorShowWindow>
-    <_AdbEmulatorShowWindow Condition=" '$(_AdbEmulatorShowWindow)' == '' ">True</_AdbEmulatorShowWindow>
     <_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference</_ApkSizesReferenceDirectory>
     <AvdLaunchTimeoutMinutes Condition=" '$(AvdLaunchTimeoutMinutes)' == '' ">10</AvdLaunchTimeoutMinutes>
     <AvdLaunchTimeoutSeconds>$([MSBuild]::Multiply($(AvdLaunchTimeoutMinutes), 60))</AvdLaunchTimeoutSeconds>
@@ -56,7 +57,7 @@
       <Output TaskParameter="IsValidTarget" PropertyName="_ValidAdbTarget"  />
     </Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget>
     <CreateAndroidEmulator
-        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        Condition=" '$(_ValidAdbTarget)' != 'True' and ( '$(TestAvdForceCreation)' == 'true' or !Exists('$(AvdManagerHome)\.android\avd\$(TestAvdName).avd') ) "
         AndroidAbi="$(TestAvdAbi)"
         AvdManagerHome="$(AvdManagerHome)"
         JavaSdkHome="$(JavaSdkDirectory)"
@@ -74,11 +75,10 @@
         Condition=" '$(_ValidAdbTarget)' != 'True' "
         AndroidSdkDirectory="$(AndroidSdkDirectory)"
         AvdManagerHome="$(AvdManagerHome)"
-        Arguments="$(TestEmulatorArguments)"
+        Arguments="$(TestAvdExtraBootArgs)"
         ImageName="$(TestAvdName)"
         LogcatFile="$(_LogcatFilenameBase)-full.txt"
         Port="$(_AdbEmulatorPort)"
-        ShowWindow="$(_AdbEmulatorShowWindow)"
         ToolExe="$(EmulatorToolExe)"
         ToolPath="$(EmulatorToolPath)">
       <Output TaskParameter="AdbTarget" PropertyName="_AdbTarget" />
@@ -124,6 +124,9 @@
   </Target>
 
   <Target Name="ReleaseAndroidTarget">
+    <PropertyGroup>
+      <_EmuTarget Condition=" '$(_EmuTarget)' == '' ">-s emulator-$(_AdbEmulatorPort)</_EmuTarget>
+    </PropertyGroup>
     <Xamarin.Android.Tools.BootstrapTasks.Adb
         Condition="'@(_FailedComponent)' != ''"
         ContinueOnError="ErrorAndContinue"
@@ -141,7 +144,7 @@
         Timeout="60000"
     />
     <KillProcess
-        Condition=" '$(_EmuTarget)' != '' "
+        Condition=" '$(_EmuPid)' != '' "
         ContinueOnError="ErrorAndContinue"
         ProcessId="$(_EmuPid)"
     />

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs

@@ -25,16 +25,6 @@ public class BaseTest
 		[SetUpFixture]
 		public class SetUp
 		{
-			public static string DeviceAbi {
-				get;
-				private set;
-			}
-
-			public static int DeviceSdkVersion {
-				get;
-				private set;
-			}
-
 			public static string TestDirectoryRoot {
 				get;
 				private set;
@@ -44,41 +34,6 @@ public static string TestDirectoryRoot {
 			public void BeforeAllTests ()
 			{
 				TestDirectoryRoot = XABuildPaths.TestOutputDirectory;
-
-				try {
-					DeviceSdkVersion = GetSdkVersion ();
-					if (DeviceSdkVersion != -1) {
-						if (DeviceSdkVersion >= 21)
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim ();
-
-						if (string.IsNullOrEmpty (DeviceAbi))
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2");
-
-						if (DeviceAbi.Contains (",")) {
-							DeviceAbi = DeviceAbi.Split (',')[0];
-						}
-					}
-				} catch (Exception ex) {
-					Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex);
-				}
-			}
-
-			int GetSdkVersion ()
-			{
-				var command = $"shell getprop ro.build.version.sdk";
-				var result = RunAdbCommand (command);
-				if (result.Contains ("*")) {
-					// Run the command again, we likely got:
-					// * daemon not running; starting now at tcp:5037
-					// * daemon started successfully
-					// adb.exe: device offline
-					TestContext.WriteLine ($"Retrying:\n{command}\n{result}");
-					result = RunAdbCommand (command);
-				}
-				if (!int.TryParse (result, out var sdkVersion)) {
-					sdkVersion = -1;
-				}
-				return sdkVersion;
 			}
 
 			[OneTimeTearDown]
@@ -208,7 +163,7 @@ protected static string RunAdbCommand (string command, bool ignoreErrors = true,
 			string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".exe" : "";
 			string adb = Path.Combine (AndroidSdkPath, "platform-tools", "adb" + ext);
 			string adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
-			return RunProcess (adb, $"{adbTarget} {command}");
+			return RunProcess (adb, $"{adbTarget} {command}", timeout);
 		}
 
 		protected static (int code, string stdOutput, string stdError) RunApkDiffCommand (string args)
@@ -225,9 +180,9 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand
 			}
 		}
 
-		protected static string RunProcess (string exe, string args)
+		protected static string RunProcess (string exe, string args, int timeoutInSeconds = 30)
 		{
-			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args);
+			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args, timeoutInSeconds);
 
 			return stdOutput + stdError;
 		}