[build] Use own build of GNU Binutils

build-tools/create-packs/SignList.xml

@@ -26,7 +26,6 @@
     <ThirdParty Include="x86_64-linux-android-as.exe" />
     <ThirdParty Include="x86_64-linux-android-ld.exe" />
     <ThirdParty Include="x86_64-linux-android-strip.exe" />
-    <ThirdParty Include="libwinpthread-1.dll" />
   </ItemGroup>
 
   <ItemGroup>

build-tools/installers/create-installers.targets

@@ -322,7 +322,6 @@
     <_MSBuildFilesWin Include="$(MSBuildSrcDir)\ndk\x86_64-linux-android-as.exe" />
     <_MSBuildFilesWin Include="$(MSBuildSrcDir)\ndk\x86_64-linux-android-ld.exe" />
     <_MSBuildFilesWin Include="$(MSBuildSrcDir)\ndk\x86_64-linux-android-strip.exe" />
-    <_MSBuildFilesWin Include="$(MSBuildSrcDir)\ndk\libwinpthread-1.dll" />
     <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libmono-android.debug.dll"   Condition=" '$(HostOS)' != 'Windows' " />
     <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libmono-android.release.dll" Condition=" '$(HostOS)' != 'Windows' " />
     <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libMonoPosixHelper.dll" />

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs

@@ -15,6 +15,8 @@ namespace Xamarin.Android.Prepare
 	//
 	partial class Configurables
 	{
+		const string BinutilsVersion                = "2.36.1-XA.4";
+
 		const string MicrosoftOpenJDK11Version      = "11.0.10";
 		const string MicrosoftOpenJDK11Release      = "9.1";
 		const string MicrosoftOpenJDK11RootDirName  = "jdk-11.0.10+9";
@@ -50,10 +52,13 @@ public static partial class Urls
 			public static readonly Uri NugetUri = new Uri ("https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe");
 
 			public static Uri MonoArchive_BaseUri = new Uri ("https://xamjenkinsartifact.azureedge.net/mono-sdks/");
+
+			public static Uri BinutilsArchive = new Uri ($"https://github.com/grendello/xamarin-android-binutils/releases/download/{BinutilsVersion}/xamarin-android-binutils-{BinutilsVersion}.7z");
 		}
 
 		public static partial class Defaults
 		{
+			public static readonly string BinutilsVersion            = Configurables.BinutilsVersion;
 			public static readonly char[] PropertyListSeparator            = new [] { ':' };
 
 			public static readonly string JdkFolder                        = "jdk-11";
@@ -271,7 +276,7 @@ public static partial class Defaults
 
 			public static readonly List <NDKTool> NDKTools = new List<NDKTool> {
 				new NDKTool (name: "as"),
-				new NDKTool (name: "ld.gold", destinationName: "ld"),
+				new NDKTool (name: "ld"),
 				new NDKTool (name: "strip"),
 			};
 		}
@@ -388,6 +393,7 @@ public static partial class Paths
 			public static string AndroidToolchainSysrootLibDirectory => GetCachedPath (ref androidToolchainSysrootLibDirectory, () => Path.Combine (AndroidToolchainRootDirectory, "sysroot", "usr", "lib"));
 			public static string WindowsBinutilsInstallDir           => GetCachedPath (ref windowsBinutilsInstallDir,           () => Path.Combine (InstallMSBuildDir, "ndk"));
 			public static string HostBinutilsInstallDir              => GetCachedPath (ref hostBinutilsInstallDir,              () => Path.Combine (InstallMSBuildDir, ctx.Properties.GetRequiredValue (KnownProperties.HostOS), "ndk"));
+			public static string BinutilsCacheDir                    => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory);
 
 			// not really configurables, merely convenience aliases for more frequently used paths that come from properties
 			public static string XAInstallPrefix                => ctx.Properties.GetRequiredValue (KnownProperties.XAInstallPrefix);

build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.Unix.cs

@@ -9,16 +9,6 @@ partial void AddRequiredOSSpecificSteps (bool beforeBundle)
 		{
 			AddRequiredMacOSSteps (beforeBundle);
 
-			if (!beforeBundle) {
-				// It has to go after the bundle step because bundle unpacking or creation *always* cleans its
-				// destination directory and this is where we download the GAS binaries. They are not part of the bundle
-				// (because they're not useful for every day work with XA) so they must be downloaded after the bundle
-				// is unpacked.
-				Log.DebugLine ("Adding Windows GAS download step (AFTER bundle)");
-				Steps.Add (new Step_Get_Windows_Binutils ());
-				return;
-			}
-
 			if (Context.Instance.WindowsJitAbisEnabled) {
 				Log.DebugLine ("Windows JIT ABIs ENABLED, ADDING MinGW dependencies build step (BEFORE bundle)");
 				Steps.Add (new Step_BuildMingwDependencies ());

build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs

@@ -34,8 +34,7 @@ protected override void AddSteps (Context context)
 
 			// The next two steps MUST be after InstallMonoRuntimes above since the latter cleans up the target
 			// directory where the NDK binutils are installed
-			Steps.Add (new Step_InstallNDKBinutils ());
-			Steps.Add (new Step_Get_Windows_Binutils ());
+			Steps.Add (new Step_InstallGNUBinutils ());
 			Steps.Add (new Step_GenerateCGManifest ());
 
 			AddRequiredOSSpecificSteps (false);

build-tools/xaprepare/xaprepare/Steps/Step_InstallGNUBinutils.Linux.cs

@@ -0,0 +1,7 @@
+namespace Xamarin.Android.Prepare
+{
+	partial class Step_InstallGNUBinutils
+	{
+		const string HostName = "linux";
+	}
+}

build-tools/xaprepare/xaprepare/Steps/Step_InstallGNUBinutils.MacOS.cs

@@ -0,0 +1,7 @@
+namespace Xamarin.Android.Prepare
+{
+	partial class Step_InstallGNUBinutils
+	{
+		const string HostName = "darwin";
+	}
+}

build-tools/xaprepare/xaprepare/Steps/Step_InstallGNUBinutils.Unix.cs

@@ -0,0 +1,7 @@
+namespace Xamarin.Android.Prepare
+{
+	partial class Step_InstallGNUBinutils
+	{
+		const string? ExecutableExtension = null;
+	}
+}

build-tools/xaprepare/xaprepare/Steps/Step_InstallGNUBinutils.Windows.cs

@@ -0,0 +1,8 @@
+namespace Xamarin.Android.Prepare
+{
+	partial class Step_InstallGNUBinutils
+	{
+		const string HostName = "windows";
+		const string ExecutableExtension = ".exe";
+	}
+}

build-tools/xaprepare/xaprepare/Steps/Step_InstallGNUBinutils.cs

@@ -0,0 +1,147 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+
+namespace Xamarin.Android.Prepare
+{
+	partial class Step_InstallGNUBinutils : StepWithDownloadProgress
+	{
+		static readonly string ProductName = $"GNU Binutils {Configurables.Defaults.BinutilsVersion}";
+
+		public Step_InstallGNUBinutils ()
+			: base ("Install GNU Binutils")
+		{}
+
+		protected override async Task<bool> Execute (Context context)
+		{
+			string hostDestinationDirectory = Configurables.Paths.HostBinutilsInstallDir;
+			string windowsDestinationDirectory = Configurables.Paths.WindowsBinutilsInstallDir;
+
+			bool hostHaveAll = HaveAllBinutils (hostDestinationDirectory);
+			bool windowsHaveAll = HaveAllBinutils (windowsDestinationDirectory, ".exe");
+
+			if (hostHaveAll && windowsHaveAll) {
+				Log.StatusLine ("All Binutils are already installed");
+				return true;
+			}
+
+			string packageName = Path.GetFileName (Configurables.Urls.BinutilsArchive.LocalPath);
+			string localArchivePath = Path.Combine (Configurables.Paths.BinutilsCacheDir, packageName);
+
+			if (!await DownloadBinutils (context, localArchivePath, Configurables.Urls.BinutilsArchive)) {
+				return false;
+			}
+
+			string tempDir = Path.Combine (Path.GetTempPath (), "xaprepare-binutils");
+			Utilities.DeleteDirectorySilent (tempDir);
+			Utilities.CreateDirectory (tempDir);
+
+			Log.DebugLine ($"Unpacking {ProductName} archive {localArchivePath} into {tempDir}");
+			if (!await Utilities.Unpack (localArchivePath, tempDir, cleanDestinatioBeforeUnpacking: true)) {
+				return false;
+			}
+
+			if (!hostHaveAll) {
+				CopyToDestination (context, "Host", tempDir, hostDestinationDirectory, executableExtension: ExecutableExtension);
+			}
+
+			if (!windowsHaveAll) {
+				CopyToDestination (context, "Windows", tempDir, windowsDestinationDirectory, "windows", ".exe");
+			}
+
+			return true;
+		}
+
+		bool CopyToDestination (Context context, string label, string sourceDir, string destinationDir, string osName = HostName, string? executableExtension = null)
+		{
+			Log.StatusLine ();
+			Log.StatusLine ($"Installing for {label}:");
+
+			string sourcePath = Path.Combine (sourceDir, osName);
+			foreach (var kvp in Configurables.Defaults.AndroidToolchainPrefixes) {
+				string prefix = kvp.Value;
+
+				foreach (NDKTool tool in Configurables.Defaults.NDKTools) {
+					string toolName = GetToolName (prefix, tool, executableExtension);
+					string toolSourcePath = Path.Combine (sourcePath, toolName);
+					string toolDestinationPath = Path.Combine (destinationDir, toolName);
+					string versionMarkerPath = GetVersionMarker (toolDestinationPath);
+
+					Log.StatusLine ($"  {context.Characters.Bullet} Installing ", toolName, tailColor: ConsoleColor.White);
+					Utilities.CopyFile (toolSourcePath, toolDestinationPath);
+					File.WriteAllText (versionMarkerPath, DateTime.UtcNow.ToString ());
+				}
+			}
+
+			return true;
+		}
+
+		async Task<bool> DownloadBinutils (Context context, string localPackagePath, Uri url)
+		{
+			if (Utilities.FileExists (localPackagePath)) {
+				Log.StatusLine ($"{ProductName} archive already downloaded");
+				return true;
+			}
+
+			Log.StatusLine ($"Downloading {ProductName} from ", url.ToString (), tailColor: ConsoleColor.White);
+			(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (url);
+			if (!success) {
+				if (status == HttpStatusCode.NotFound) {
+					Log.ErrorLine ($"{ProductName} archive URL not found");
+					return false;
+				}
+				Log.WarningLine ($"Failed to obtain {ProductName} size. HTTP status code: {status} ({(int)status})");
+			}
+
+			DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
+			Log.StatusLine ($"  {context.Characters.Link} {url}", ConsoleColor.White);
+			await Download (context, url, localPackagePath, ProductName, Path.GetFileName (localPackagePath), downloadStatus);
+
+			if (!File.Exists (localPackagePath)) {
+				Log.ErrorLine ($"Download of {ProductName} from {url} failed.");
+				return false;
+			}
+
+			return true;
+		}
+
+		bool HaveAllBinutils (string dir, string? executableExtension = null)
+		{
+			Log.DebugLine ("Checking if all binutils are installed in {dir}");
+			string extension = executableExtension ?? String.Empty;
+			foreach (var kvp in Configurables.Defaults.AndroidToolchainPrefixes) {
+				string prefix = kvp.Value;
+
+				foreach (NDKTool tool in Configurables.Defaults.NDKTools) {
+					string toolName = GetToolName (prefix, tool, executableExtension);
+					string toolPath = Path.Combine (dir, toolName);
+					string versionMarkerPath = GetVersionMarker (toolPath);
+
+					Log.DebugLine ($"Checking {toolName}");
+					if (!Utilities.FileExists (toolPath)) {
+						Log.DebugLine ($"Binutils tool {toolPath} does not exist");
+						return false;
+					}
+
+					if (!Utilities.FileExists (versionMarkerPath)) {
+						Log.DebugLine ($"Binutils tool {toolPath} exists, but its version is incorrect");
+						return false;
+					}
+				}
+			}
+
+			return true;
+		}
+
+		string GetToolName (string prefix, NDKTool tool, string? executableExtension = null)
+		{
+			return $"{prefix}-{(String.IsNullOrEmpty (tool.DestinationName) ? tool.Name : tool.DestinationName)}{executableExtension}";
+		}
+
+		string GetVersionMarker (string toolPath)
+		{
+			return $"{toolPath}.{Configurables.Defaults.BinutilsVersion}";
+		}
+	}
+}

build-tools/xaprepare/xaprepare/ThirdPartyNotices/Xamarin.Android.Build.Tasks.cs

@@ -43,7 +43,7 @@ class XamarinAndroidBuildTasks_External_Dependencies_Group : ThirdPartyNoticeGro
 			new XamarinAndroidBuildTasks_IronyProject_Irony_TPN (),
 			new XamarinAndroidBuildTasks_JamesNK_NewtonsoftJson_TPN (),
 			new XamarinAndroidBuildTasks_NuGet_NuGetClient_TPN (),
-			new XamarinAndroidBuildTasks_android_platform_ndk_TPN (),
+			new XamarinAndroidBuildTasks_gnu_binutils_TPN (),
 		};
 	}
 
@@ -141,13 +141,13 @@ specific language governing permissions and limitations under the License.
 ";
 	}
 
-	class XamarinAndroidBuildTasks_android_platform_ndk_TPN : ThirdPartyNotice {
+	class XamarinAndroidBuildTasks_gnu_binutils_TPN : ThirdPartyNotice {
 
-		static readonly Uri    url         = new Uri ("https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;a=tree;f=gas");
+		static readonly Uri    url         = new Uri ("https://sourceware.org/git/?p=binutils-gdb.git;a=tree;hb=HEAD");
 
 		public override string LicenseText => String.Empty;
 		public override string LicenseFile => CommonLicenses.GPLv3Path;
-		public override string Name        => "android/platform/ndk";
+		public override string Name        => "gnu/binutils";
 		public override Uri    SourceUrl   => url;
 	}
 }

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc

@@ -8,7 +8,7 @@
       "Size": 54406
     },
     "assemblies/Mono.Android.dll": {
-      "Size": 78829
+      "Size": 82044
     },
     "assemblies/System.Linq.dll": {
       "Size": 10200
@@ -20,7 +20,7 @@
       "Size": 2267
     },
     "assemblies/UnnamedProject.dll": {
-      "Size": 3161
+      "Size": 3160
     },
     "classes.dex": {
       "Size": 316792
@@ -41,7 +41,7 @@
       "Size": 145864
     },
     "lib/arm64-v8a/libxamarin-app.so": {
-      "Size": 68560
+      "Size": 11504
     },
     "META-INF/ANDROIDD.RSA": {
       "Size": 1213
@@ -74,5 +74,5 @@
       "Size": 1724
     }
   },
-  "PackageSize": 2856724
+  "PackageSize": 2885396
 }