[ci] Use commandlinetools' avdmanager (dotnet#4770)

jonpryor · web-flow · commit d99facb16a23 · 2020-06-05T14:34:20.000-04:00
Context: dotnet#4567 Context: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3711144&view=logs&jobId=4348ab7c-72f0-52b0-44fd-6f17a16a558f&j=8556562a-ae5f-5bd1-7c4d-bf1af4b6f1e1&t=9f016295-34d3-5b55-55cb-51df877e2cc6 Usage of `avdmanager` fails under JDK 11: TestApks.targets(45,5): error : Exception in thread "main" java.lang.NoClassDefFoundError: javax/xml/bind/annotation/XmlSchema TestApks.targets(45,5): error : at com.android.repository.api.SchemaModule$SchemaModuleVersion.<init>(SchemaModule.java:156) TestApks.targets(45,5): error : at com.android.repository.api.SchemaModule.<init>(SchemaModule.java:75) TestApks.targets(45,5): error : at com.android.sdklib.repository.AndroidSdkHandler.<clinit>(AndroidSdkHandler.java:81) TestApks.targets(45,5): error : at com.android.sdklib.tool.AvdManagerCli.run(AvdManagerCli.java:213) TestApks.targets(45,5): error : at com.android.sdklib.tool.AvdManagerCli.main(AvdManagerCli.java:200) TestApks.targets(45,5): error : Caused by: java.lang.ClassNotFoundException: javax.xml.bind.annotation.XmlSchema TestApks.targets(45,5): error : at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583) TestApks.targets(45,5): error : at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) TestApks.targets(45,5): error : at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) TestApks.targets(45,5): error : ... 5 more TestApks.targets(45,5): error : Process `/Users/runner/Library/Android/sdk/tools/bin/avdmanager` exited with value 1. This is because the "normal" `tools`' `avdmanager` doesn't support JDK 11, and has been replaced with a *new* `commandlinetools` package whicih *does* support JDK 11. Begin installing the `commandlinetools` SDK package, and use it for `sdkmanager` invocations.
diff --git a/Configuration.props b/Configuration.props
@@ -135,6 +135,9 @@
     <AndroidToolPath Condition=" '$(AndroidToolPath)' == '' ">$(AndroidSdkFullPath)\tools</AndroidToolPath>
     <AndroidToolsBinPath Condition=" '$(AndroidToolsBinPath)' == '' ">$(AndroidToolPath)\bin</AndroidToolsBinPath>
     <AndroidToolExe Condition=" '$(AndroidToolExe)' == '' ">android</AndroidToolExe>
+    <CommandLineToolsFolder Condition=" '$(CommandLineToolsFolder)' == '' ">1.0</CommandLineToolsFolder>
+    <CommandLineToolsVersion Condition=" '$(CommandLineToolsVersion)' == '' ">6200805_latest</CommandLineToolsVersion>
+    <CommandLineToolsBinPath Condition=" '$(CommandLineToolsBinPath)' == '' ">$(AndroidSdkFullPath)\cmdline-tools\$(CommandLineToolsFolder)\bin</CommandLineToolsBinPath>
     <EmulatorVersion Condition=" '$(EmulatorVersion)' == '' ">6466327</EmulatorVersion>
     <EmulatorPkgRevision Condition=" '$(EmulatorPkgRevision)' == '' ">30.0.12</EmulatorPkgRevision>
     <EmulatorToolPath Condition=" '$(EmulatorToolPath)' == '' ">$(AndroidSdkFullPath)\emulator</EmulatorToolPath>
diff --git a/build-tools/check-boot-times/Program.cs b/build-tools/check-boot-times/Program.cs
@@ -509,21 +509,29 @@ static string GetProgramPath (params string [] filenames)
 					if (!string.IsNullOrWhiteSpace (sdkPath)) {
 						potentialLocations.AddRange (new []{
 							$"{sdkPath}/platform-tools",
+							$"{sdkPath}/cmdline-tools/1.0/bin",
+							$"{sdkPath}/cmdline-tools/latest/bin",
 							$"{sdkPath}/emulator",
 							$"{sdkPath}/tools",
 							$"{sdkPath}/tools/bin",
 						});
 					} else {
 						potentialLocations.AddRange (new []{
 							"AppData/Local/Android/Sdk/platform-tools",
+							"AppData/Local/Android/Sdk/cmdline-tools/1.0/bin",
+							"AppData/Local/Android/Sdk/cmdline-tools/latest/bin",
 							"AppData/Local/Android/Sdk/emulator",
 							"AppData/Local/Android/Sdk/tools",
 							"AppData/Local/Android/Sdk/tools/bin",
 							"Library/Android/sdk/platform-tools",
+							"Library/Android/sdk/cmdline-tools/1.0/bin",
+							"Library/Android/sdk/cmdline-tools/latest/bin",
 							"Library/Android/sdk/emulator",
 							"Library/Android/sdk/tools",
 							"Library/Android/sdk/tools/bin",
 							"android-toolchain/sdk/platform-tools",
+							"android-toolchain/sdk/cmdline-tools/1.0/bin",
+							"android-toolchain/sdk/cmdline-tools/latest/bin",
 							"android-toolchain/sdk/emulator",
 							"android-toolchain/sdk/tools",
 							"android-toolchain/sdk/tools/bin",
@@ -532,6 +540,8 @@ static string GetProgramPath (params string [] filenames)
 						if (RunningOnWindowsEnvironment) {
 							potentialLocations.AddRange (new []{
 								"C:/Program Files (x86)/Android/android-sdk/platform-tools",
+								"C:/Program Files (x86)/Android/android-sdk/cmdline-tools/1.0/bin",
+								"C:/Program Files (x86)/Android/android-sdk/cmdline-tools/latest/bin",
 								"C:/Program Files (x86)/Android/android-sdk/emulator",
 								"C:/Program Files (x86)/Android/android-sdk/tools",
 								"C:/Program Files (x86)/Android/android-sdk/tools/bin",
diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets
@@ -50,7 +50,7 @@
         SdkVersion="29"
         ImageName="$(_TestImageName)"
         ToolExe="$(AvdManagerToolExe)"
-        ToolPath="$(AndroidToolsBinPath)"
+        ToolPath="$(CommandLineToolsBinPath)"
         RamSizeMB="3072"
         DataPartitionSizeMB="4096"
     />
diff --git a/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs b/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs
@@ -16,6 +16,8 @@ static class KnownProperties
 		public const string AutoProvision                       = "AutoProvision";
 		public const string AutoProvisionUsesSudo               = "AutoProvisionUsesSudo";
 		public const string Configuration                       = "Configuration";
+		public const string CommandLineToolsVersion             = nameof (CommandLineToolsVersion);
+		public const string CommandLineToolsFolder              = nameof (CommandLineToolsFolder);
 		public const string EmulatorVersion                     = "EmulatorVersion";
 		public const string EmulatorPkgRevision                 = "EmulatorPkgRevision";
 		public const string IgnoreMaxMonoVersion                = "IgnoreMaxMonoVersion";
diff --git a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in
@@ -20,6 +20,8 @@ namespace Xamarin.Android.Prepare
 			properties.Add (KnownProperties.AutoProvision,                       StripQuotes ("@AutoProvision@"));
 			properties.Add (KnownProperties.AutoProvisionUsesSudo,               StripQuotes ("@AutoProvisionUsesSudo@"));
 			properties.Add (KnownProperties.Configuration,                       StripQuotes ("@Configuration@"));
+			properties.Add (KnownProperties.CommandLineToolsFolder,              StripQuotes ("@CommandLineToolsFolder@"));
+			properties.Add (KnownProperties.CommandLineToolsVersion,             StripQuotes ("@CommandLineToolsVersion@"));
 			properties.Add (KnownProperties.EmulatorVersion,                     StripQuotes ("@EmulatorVersion@"));
 			properties.Add (KnownProperties.EmulatorPkgRevision,                 StripQuotes ("@EmulatorPkgRevision@"));
 			properties.Add (KnownProperties.IgnoreMaxMonoVersion,                StripQuotes ("@IgnoreMaxMonoVersion@"));
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs
@@ -8,5 +8,6 @@ partial class AndroidToolchain
 	{
 		static readonly string osTag = "linux";
 		static readonly string altOsTag = osTag;
+		static readonly string cltOsTag = osTag;
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs
@@ -4,5 +4,6 @@ partial class AndroidToolchain
 	{
 		static readonly string osTag = "darwin";
 		static readonly string altOsTag = "macosx";
+		static readonly string cltOsTag = "mac";
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs
@@ -4,5 +4,6 @@ partial class AndroidToolchain
 	{
 		static readonly string osTag = "windows";
 		static readonly string altOsTag = osTag;
+		static readonly string cltOsTag = "win";
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs
@@ -17,6 +17,8 @@ public AndroidToolchain ()
 			string AndroidNdkDirectory     = GetRequiredProperty (KnownProperties.AndroidNdkDirectory);
 			string AndroidCmakeVersion     = GetRequiredProperty (KnownProperties.AndroidCmakeVersion);
 			string AndroidCmakeVersionPath = GetRequiredProperty (KnownProperties.AndroidCmakeVersionPath);
+			string CommandLineToolsVersion = GetRequiredProperty (KnownProperties.CommandLineToolsVersion);
+			string CommandLineToolsFolder  = GetRequiredProperty (KnownProperties.CommandLineToolsFolder);
 			string EmulatorVersion         = GetRequiredProperty (KnownProperties.EmulatorVersion);
 			string EmulatorPkgRevision     = GetRequiredProperty (KnownProperties.EmulatorPkgRevision);
 			string XABuildToolsFolder      = GetRequiredProperty (KnownProperties.XABuildToolsFolder);
@@ -51,6 +53,8 @@ public AndroidToolchain ()
 				new AndroidToolchainComponent ($"x86-29_r07-{osTag}",                               destDir: Path.Combine ("system-images", "android-29", "default", "x86"), relativeUrl: new Uri ("sys-img/android/", UriKind.Relative), pkgRevision: "7"),
 				new AndroidToolchainComponent ($"android-ndk-r{AndroidNdkVersion}-{osTag}-x86_64",  destDir: AndroidNdkDirectory, pkgRevision: AndroidPkgRevision),
 				new AndroidToolchainComponent ($"build-tools_r{XABuildToolsVersion}-{altOsTag}",    destDir: Path.Combine ("build-tools", XABuildToolsFolder), isMultiVersion: true),
+				new AndroidToolchainComponent ($"commandlinetools-{cltOsTag}-{CommandLineToolsVersion}",
+					destDir: Path.Combine ("cmdline-tools", CommandLineToolsFolder), isMultiVersion: true),
 				new AndroidToolchainComponent ($"platform-tools_r{XAPlatformToolsVersion}-{osTag}", destDir: "platform-tools", pkgRevision: XAPlatformToolsVersion),
 				new AndroidToolchainComponent ($"sdk-tools-{osTag}-4333796",                        destDir: "tools", pkgRevision: "26.1.1"),
 				new AndroidToolchainComponent ($"emulator-{osTag}-{EmulatorVersion}",               destDir: "emulator", pkgRevision: EmulatorPkgRevision),
diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs
@@ -4,6 +4,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using Kajabity.Tools.Java;
 
 namespace Xamarin.Android.Prepare
@@ -50,6 +51,7 @@ protected override async Task<bool> Execute (Context context)
 					Log.ErrorLine ("Failed to accept Android SDK licenses");
 					return false;
 				}
+				WritePackageXmls (sdkRoot);
 				return GatherNDKInfo (context, ndkRoot);
 			}
 
@@ -94,12 +96,24 @@ protected override async Task<bool> Execute (Context context)
 				return false;
 			}
 
+			WritePackageXmls (sdkRoot);
+
 			return GatherNDKInfo (context, ndkRoot);
 		}
 
 		bool AcceptLicenses (Context context, string sdkRoot)
 		{
-			string sdkManager = context.OS.Which (Path.Combine (sdkRoot, "tools", "bin", "sdkmanager"));
+			string[] sdkManagerPaths = new[]{
+				Path.Combine (sdkRoot, "cmdline-tools", context.Properties [KnownProperties.CommandLineToolsFolder], "bin", "sdkmanager"),
+				Path.Combine (sdkRoot, "cmdline-tools", "latest", "bin", "sdkmanager"),
+				Path.Combine (sdkRoot, "tools", "bin", "sdkmanager"),
+			};
+			string sdkManager = "";
+			foreach (var sdkManagerPath in sdkManagerPaths) {
+				sdkManager = context.OS.Which (sdkManagerPath, required: false);
+				if (!string.IsNullOrEmpty (sdkManager))
+					break;
+			}
 			if (sdkManager.Length == 0)
 				throw new InvalidOperationException ("sdkmanager not found");
 			string jdkDir = context.OS.JavaHome;
@@ -317,5 +331,60 @@ bool IsNdk (AndroidToolchainComponent component)
 		{
 			return component.Name.StartsWith ("android-ndk", StringComparison.OrdinalIgnoreCase);
 		}
+
+		static  readonly    XNamespace  AndroidRepositoryCommon     = "http://schemas.android.com/repository/android/common/01";
+		static  readonly    XNamespace  AndroidRepositoryGeneric    = "http://schemas.android.com/repository/android/generic/01";
+
+		void WritePackageXmls (string sdkRoot)
+		{
+			string[] packageXmlDirs = new[]{
+				Path.Combine (sdkRoot, "emulator"),
+			};
+			foreach (var path in packageXmlDirs) {
+				var properties = ReadSourceProperties (path);
+				if (properties == null)
+					continue;
+				string packageXml = Path.Combine (path, "package.xml");
+				Log.DebugLine ($"Writing '{packageXml}'");
+				var doc = new XDocument(
+						new XElement (AndroidRepositoryCommon + "repository",
+							new XAttribute (XNamespace.Xmlns + "ns2", AndroidRepositoryCommon.NamespaceName),
+							new XAttribute (XNamespace.Xmlns + "ns3", AndroidRepositoryGeneric.NamespaceName),
+							new XElement ("localPackage",
+								new XAttribute ("path", properties ["Pkg.Path"]),
+								new XAttribute ("obsolete", "false"),
+								new XElement ("revision", GetRevision (properties ["Pkg.Revision"])),
+								new XElement ("display-name", properties ["Pkg.Desc"]))));
+				doc.Save (packageXml, SaveOptions.None);
+			}
+		}
+
+		Dictionary<string, string>? ReadSourceProperties (string dir)
+		{
+			var path = Path.Combine (dir, "source.properties");
+			if (!File.Exists (path))
+				return null;
+			var dict = new Dictionary<string, string> ();
+			foreach (var line in File.ReadLines (path)) {
+				if (line.Length == 0)
+					continue;
+				var entry = line.Split (new[]{'='}, 2, StringSplitOptions.None);
+				if (entry.Length != 2)
+					continue;
+				dict.Add (entry [0], entry [1]);
+			}
+			return dict;
+		}
+
+		IEnumerable<XElement> GetRevision (string revision)
+		{
+			var parts = revision.Split ('.');
+			if (parts.Length > 0)
+				yield return new XElement ("major", parts [0]);
+			if (parts.Length > 1)
+				yield return new XElement ("minor", parts [1]);
+			if (parts.Length > 2)
+				yield return new XElement ("micro", parts [2]);
+		}
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/xaprepare.targets b/build-tools/xaprepare/xaprepare/xaprepare.targets
@@ -53,6 +53,8 @@
       <Replacement Include="@AutoProvision@=$(AutoProvision)" />
       <Replacement Include="@AutoProvisionUsesSudo@=$(AutoProvisionUsesSudo)" />
       <Replacement Include="@Configuration@=$(Configuration)" />
+      <Replacement Include="@CommandLineToolsFolder@=$(CommandLineToolsFolder)" />
+      <Replacement Include="@CommandLineToolsVersion@=$(CommandLineToolsVersion)" />
       <Replacement Include="@EmulatorVersion@=$(EmulatorVersion)" />
       <Replacement Include="@EmulatorPkgRevision@=$(EmulatorPkgRevision)" />
       <Replacement Include="@IgnoreMaxMonoVersion@=$(IgnoreMaxMonoVersion)" />

Original file line number	Diff line number	Diff line change
`@@ -8,5 +8,6 @@ partial class AndroidToolchain`
`8`	`8`	`{`
`9`	`9`	`static readonly string osTag = "linux";`
`10`	`10`	`static readonly string altOsTag = osTag;`
	`11`	`+ static readonly string cltOsTag = osTag;`
`11`	`12`	`}`
`12`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,6 @@ partial class AndroidToolchain`
`4`	`4`	`{`
`5`	`5`	`static readonly string osTag = "darwin";`
`6`	`6`	`static readonly string altOsTag = "macosx";`
	`7`	`+ static readonly string cltOsTag = "mac";`
`7`	`8`	`}`
`8`	`9`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,6 @@ partial class AndroidToolchain`
`4`	`4`	`{`
`5`	`5`	`static readonly string osTag = "windows";`
`6`	`6`	`static readonly string altOsTag = osTag;`
	`7`	`+ static readonly string cltOsTag = "win";`
`7`	`8`	`}`
`8`	`9`	`}`