[CoreCLR] Ignore some assemblies when asked to load them (#10249)

Ignore assemblies whose names end in `.ni` and `.ni.dll`, when 
a load request is made via the CoreCLR host contract callback.

For each CoreCLR will ask to first load `Assembly.ni.dll`, NI
standing for Native Image. This is something that was used on
Windows long ago and it never was, nor will it be, supported on
Unix.

This is mostly a cosmetic change in that it prevents a false
warning about a missing assembly from being logged in logcat.
This, in itself, is a startup time improvement since those
warnings would always be logged during every application launch,
with logcat logging being very expensive (and unpredictable).

The message about a missing `.ni.dll` is still logged, but with
the Debug severity, which will cause it to show up only if the
`assembly` logging category is enabled (it is disabled by default)

Author: grendello

src/Xamarin.Android.Build.Tasks/Tasks/CreateAssemblyStore.cs

@@ -28,13 +28,20 @@ public class CreateAssemblyStore : AndroidTask
 	[Required]
 	public string [] SupportedAbis { get; set; } = [];
 
+	[Required]
+	public string TargetRuntime { get; set; } = "";
+
 	public bool UseAssemblyStore { get; set; }
 
 	[Output]
 	public ITaskItem [] AssembliesToAddToArchive { get; set; } = [];
 
+	AndroidRuntime targetRuntime;
+
 	public override bool RunTask ()
 	{
+		targetRuntime = MonoAndroidHelper.ParseAndroidRuntime (TargetRuntime);
+
 		// Get all the user and framework assemblies we may need to package
 		var assemblies = ResolvedFrameworkAssemblies.Concat (ResolvedUserAssemblies).Where (asm => !(ShouldSkipAssembly (asm))).ToArray ();
 
@@ -43,7 +50,7 @@ public override bool RunTask ()
 			return !Log.HasLoggedErrors;
 		}
 
-		var store_builder = new AssemblyStoreBuilder (Log);
+		var store_builder = new AssemblyStoreBuilder (Log, targetRuntime);
 		var per_arch_assemblies = MonoAndroidHelper.GetPerArchAssemblies (assemblies, SupportedAbis, true);
 
 		foreach (var kvp in per_arch_assemblies) {

src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs

@@ -16,15 +16,17 @@ class AssemblyStoreAssemblyInfo
 	public byte[] AssemblyNameNoExtBytes { get; }
 	public FileInfo? SymbolsFile         { get; set; }
 	public FileInfo? ConfigFile          { get; set; }
+	public bool Ignored                  { get; }
 
-	public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly)
+	public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly, bool assemblyIsIgnored = false)
 	{
 		Arch = MonoAndroidHelper.GetTargetArch (assembly);
 		if (Arch == AndroidTargetArch.None) {
 			throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata");
 		}
 
 		SourceFile = new FileInfo (sourceFilePath);
+		Ignored = assemblyIsIgnored;
 
 		string? name = Path.GetFileName (SourceFile.Name);
 		if (name == null) {

src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreBuilder.cs

@@ -13,10 +13,12 @@ class AssemblyStoreBuilder
 {
 	readonly TaskLoggingHelper log;
 	readonly AssemblyStoreGenerator storeGenerator;
+	readonly AndroidRuntime targetRuntime;
 
-	public AssemblyStoreBuilder (TaskLoggingHelper log)
+	public AssemblyStoreBuilder (TaskLoggingHelper log, AndroidRuntime targetRuntime)
 	{
 		this.log = log;
+		this.targetRuntime = targetRuntime;
 		storeGenerator = new (log);
 	}
 
@@ -39,6 +41,24 @@ public void AddAssembly (string assemblySourcePath, ITaskItem assemblyItem, bool
 		}
 
 		storeGenerator.Add (storeAssemblyInfo);
+
+		ClrAddIgnoredNativeImageAssembly (assemblyItem);
+	}
+
+	// When CoreCLR tries to load an assembly (say `AssemblyName.dll`) it will always first try to load
+	// a "native image" assembly from `AssemblyName.ni.dll` which will **never** exist. The native image
+	// assemblies were once supported only on Windows and were never (nor will ever be) supported on
+	// Unix. In order to speed up load times, we add an empty entry for each `*.ni.dll` to the assembly
+	// store index.
+	void ClrAddIgnoredNativeImageAssembly (ITaskItem assemblyItem)
+	{
+		if (targetRuntime != AndroidRuntime.CoreCLR) {
+			return;
+		}
+
+		string ignoredName = Path.GetFileName (Path.ChangeExtension (assemblyItem.ItemSpec, ".ni.dll"));
+		var storeAssemblyInfo = new AssemblyStoreAssemblyInfo (ignoredName, assemblyItem, assemblyIsIgnored: true);
+		storeGenerator.Add (storeAssemblyInfo);
 	}
 
 	public Dictionary<AndroidTargetArch, string> Generate (string outputDirectoryPath) => storeGenerator.Generate (outputDirectoryPath);

src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs

@@ -32,18 +32,21 @@ public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint ind
 
 	sealed class AssemblyStoreIndexEntry
 	{
-		public const uint NativeSize32 = 2 * sizeof (uint);
-		public const uint NativeSize64 = sizeof (ulong) + sizeof (uint);
+		// We treat `bool` as `byte` here, since that's what gets written to the binary
+		public const uint NativeSize32 = 2 * sizeof (uint) + sizeof (byte);
+		public const uint NativeSize64 = sizeof (ulong) + sizeof (uint) + sizeof (byte);
 
 		public readonly string name;
 		public readonly ulong name_hash;
 		public readonly uint  descriptor_index;
+		public readonly bool ignore;
 
-		public AssemblyStoreIndexEntry (string name, ulong name_hash, uint descriptor_index)
+		public AssemblyStoreIndexEntry (string name, ulong name_hash, uint descriptor_index, bool ignore)
 		{
 			this.name = name;
 			this.name_hash = name_hash;
 			this.descriptor_index = descriptor_index;
+			this.ignore = ignore;
 		}
 	}
 

src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs

@@ -16,6 +16,7 @@ namespace Xamarin.Android.Tasks;
 // [HEADER]
 // [INDEX]
 // [ASSEMBLY_DESCRIPTORS]
+// [ASSEMBLY_NAMES]
 // [ASSEMBLY DATA]
 //
 // Formats of the sections above are as follows:
@@ -30,6 +31,7 @@ namespace Xamarin.Android.Tasks;
 // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension)
 //  [NAME_HASH]          uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name
 //  [DESCRIPTOR_INDEX]   uint; index into in-store assembly descriptor array
+//  [IGNORE]             byte; if set to anything other than 0, the assembly is to be ignored when loading
 //
 // ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows:
 //  [MAPPING_INDEX]      uint; index into a runtime array where assembly data pointers are stored
@@ -50,8 +52,8 @@ partial class AssemblyStoreGenerator
 	const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant
 
 	// Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones
-	const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant
-	const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002;
+	const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000003; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant
+	const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000003;
 
 	const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000;
 	const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000;
@@ -122,19 +124,29 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List<Assemb
 		using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read);
 		fs.Seek ((long)curPos, SeekOrigin.Begin);
 
+		uint mappingIndex = 0;
 		foreach (AssemblyStoreAssemblyInfo info in infos) {
 			(AssemblyStoreEntryDescriptor desc, curPos) = MakeDescriptor (info, curPos);
-			desc.mapping_index = (uint)descriptors.Count;
+			if (info.Ignored) {
+				desc.mapping_index = 0;
+			} else {
+				desc.mapping_index = mappingIndex++;
+			}
+			uint entryIndex = (uint)descriptors.Count;
 			descriptors.Add (desc);
 
-			if ((uint)fs.Position != desc.data_offset) {
+			if (!info.Ignored && (uint)fs.Position != desc.data_offset) {
 				throw new InvalidOperationException ($"Internal error: corrupted store '{storePath}' stream");
 			}
 
 			ulong name_with_ext_hash = MonoAndroidHelper.GetXxHash (info.AssemblyNameBytes, is64Bit);
 			ulong name_no_ext_hash = MonoAndroidHelper.GetXxHash (info.AssemblyNameNoExtBytes, is64Bit);
-			index.Add (new AssemblyStoreIndexEntry (info.AssemblyName, name_with_ext_hash, desc.mapping_index));
-			index.Add (new AssemblyStoreIndexEntry (info.AssemblyNameNoExt, name_no_ext_hash, desc.mapping_index));
+			index.Add (new AssemblyStoreIndexEntry (info.AssemblyName, name_with_ext_hash, entryIndex, info.Ignored));
+			index.Add (new AssemblyStoreIndexEntry (info.AssemblyNameNoExt, name_no_ext_hash, entryIndex, info.Ignored));
+
+			if (info.Ignored) {
+				continue;
+			}
 
 			CopyData (info.SourceFile, fs, storePath);
 			CopyData (info.SymbolsFile, fs, storePath);
@@ -184,8 +196,8 @@ void CopyData (FileInfo? src, Stream dest, string storePath)
 	static (AssemblyStoreEntryDescriptor desc, ulong newPos) MakeDescriptor (AssemblyStoreAssemblyInfo info, ulong curPos)
 	{
 		var ret = new AssemblyStoreEntryDescriptor {
-			data_offset = (uint)curPos,
-			data_size = GetDataLength (info.SourceFile),
+			data_offset = info.Ignored ? 0 : (uint)curPos,
+			data_size = info.Ignored ? 0 : GetDataLength (info.SourceFile),
 		};
 		if (info.SymbolsFile != null) {
 			ret.debug_data_offset = ret.data_offset + ret.data_size;
@@ -197,9 +209,11 @@ void CopyData (FileInfo? src, Stream dest, string storePath)
 			ret.config_data_size = GetDataLength (info.ConfigFile);
 		}
 
-		curPos += ret.data_size + ret.debug_data_size + ret.config_data_size;
-		if (curPos > UInt32.MaxValue) {
-			throw new NotSupportedException ("Assembly store size exceeds the maximum supported value");
+		if (!info.Ignored) {
+			curPos += ret.data_size + ret.debug_data_size + ret.config_data_size;
+			if (curPos > UInt32.MaxValue) {
+				throw new NotSupportedException ("Assembly store size exceeds the maximum supported value");
+			}
 		}
 
 		return (ret, curPos);
@@ -252,8 +266,9 @@ void WriteIndex (BinaryWriter writer, StreamWriter manifestWriter, List<Assembly
 				manifestWriter.Write ($"0x{(uint)entry.name_hash:x}");
 			}
 			writer.Write (entry.descriptor_index);
-			manifestWriter.Write ($" di:{entry.descriptor_index}");
+			writer.Write ((byte)(entry.ignore ? 1 : 0));
 
+			manifestWriter.Write ($" di:{entry.descriptor_index}");
 			AssemblyStoreEntryDescriptor desc = descriptors[(int)entry.descriptor_index];
 			manifestWriter.Write ($" mi:{desc.mapping_index}");
 			manifestWriter.Write ($" do:{desc.data_offset}");
@@ -262,7 +277,11 @@ void WriteIndex (BinaryWriter writer, StreamWriter manifestWriter, List<Assembly
 			manifestWriter.Write ($" dds:{desc.debug_data_size}");
 			manifestWriter.Write ($" cdo:{desc.config_data_offset}");
 			manifestWriter.Write ($" cds:{desc.config_data_size}");
-			manifestWriter.WriteLine ($" {entry.name}");
+			manifestWriter.Write ($" {entry.name}");
+			if (entry.ignore) {
+				manifestWriter.Write (" (ignored)");
+			}
+			manifestWriter.WriteLine ();
 		}
 	}
 
@@ -285,7 +304,9 @@ List<AssemblyStoreIndexEntry> ReadIndex (BinaryReader reader, AssemblyStoreHeade
 			}
 
 			uint descriptor_index = reader.ReadUInt32 ();
-			index.Add (new AssemblyStoreIndexEntry (String.Empty, name_hash, descriptor_index));
+			bool ignored = reader.ReadByte () != 0;
+
+			index.Add (new AssemblyStoreIndexEntry (String.Empty, name_hash, descriptor_index, ignored));
 		}
 
 		return index;

src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets

@@ -2288,6 +2288,7 @@ because xbuild doesn't support framework reference assemblies.
   -->
   <CreateAssemblyStore
       Condition=" '$(_AndroidRuntime)' != 'NativeAOT' "
+      TargetRuntime="$(_AndroidRuntime)"
       AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)"
       IncludeDebugSymbols="$(AndroidIncludeDebugSymbols)"
       ResolvedFrameworkAssemblies="@(_BuildApkResolvedFrameworkAssemblies)"

src/native/clr/host/assembly-store.cc

@@ -195,11 +195,8 @@ auto AssemblyStore::find_assembly_store_entry (hash_t hash, const AssemblyStoreI
 auto AssemblyStore::open_assembly (std::string_view const& name, int64_t &size) noexcept -> void*
 {
 	hash_t name_hash = xxhash::hash (name.data (), name.length ());
-	log_debug (LOG_ASSEMBLY, "AssemblyStore::open_assembly: looking for bundled name: '{}' (hash {:x})"sv, optional_string (name.data ()), name_hash);
 
 	if constexpr (Constants::is_debug_build) {
-		// TODO: implement filesystem lookup here
-
 		// In fastdev mode we might not have any assembly store.
 		if (assembly_store_hashes == nullptr) {
 			log_warn (LOG_ASSEMBLY, "Assembly store not registered. Unable to look up assembly '{}'"sv, name);
@@ -208,12 +205,15 @@ auto AssemblyStore::open_assembly (std::string_view const& name, int64_t &size)
 	}
 
 	const AssemblyStoreIndexEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, assembly_store.index_entry_count);
-	if (hash_entry == nullptr) {
-		// This message should really be `log_warn`, but since CoreCLR attempts to load `AssemblyName.ni.dll` for each
-		// `AssemblyName.dll`, it creates a lot of non-actionable noise.
-		// TODO (in separate PR): generate hashes for the .ni.dll names and ignore them at the top of the function. Then restore
-		// `log_warn` here.
-		log_debug (LOG_ASSEMBLY, "Assembly '{}' (hash 0x{:x}) not found"sv, optional_string (name.data ()), name_hash);
+	if (hash_entry == nullptr) [[unlikely]] {
+		size = 0;
+		log_warn (LOG_ASSEMBLY, "Assembly '{}' (hash 0x{:x}) not found"sv, name, name_hash);
+		return nullptr;
+	}
+
+	if (hash_entry->ignore != 0) {
+		size = 0;
+		log_debug (LOG_ASSEMBLY, "Assembly '{}' ignored"sv, name);
 		return nullptr;
 	}
 

src/native/clr/include/constants.hh

@@ -3,6 +3,7 @@
 #include <fcntl.h>
 #include <sys/system_properties.h>
 
+#include <limits>
 #include <string_view>
 
 #include <shared/cpp-util.hh>

src/native/clr/include/host/assembly-store.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstdint>
+#include <limits>
 #include <mutex>
 #include <string_view>
 #include <tuple>

src/native/clr/include/xamarin-app.hh

@@ -31,7 +31,7 @@ static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000;
 #endif
 
 // Increase whenever an incompatible change is made to the assembly store format
-static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 2 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI;
+static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 3 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI;
 
 static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian
 static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian
@@ -153,6 +153,7 @@ struct XamarinAndroidBundledAssembly
 // [HEADER]
 // [INDEX]
 // [ASSEMBLY_DESCRIPTORS]
+// [ASSEMBLY_NAMES]
 // [ASSEMBLY DATA]
 //
 // Formats of the sections above are as follows:
@@ -167,6 +168,7 @@ struct XamarinAndroidBundledAssembly
 // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension)
 //  [NAME_HASH]          uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name
 //  [DESCRIPTOR_INDEX]   uint; index into in-store assembly descriptor array
+//  [IGNORE]             byte; if set to anything other than 0, the assembly is to be ignored when loading
 //
 // ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows:
 //  [MAPPING_INDEX]      uint; index into a runtime array where assembly data pointers are stored
@@ -199,6 +201,7 @@ struct [[gnu::packed]] AssemblyStoreIndexEntry final
 {
 	xamarin::android::hash_t name_hash;
 	uint32_t descriptor_index;
+	uint8_t ignore; // Assembly should be ignored when loading, its data isn't actually there
 };
 
 struct [[gnu::packed]] AssemblyStoreEntryDescriptor final