Make typemap dump utility work with current libxamarin-app.so

src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs

@@ -132,7 +132,7 @@ sealed class XamarinAndroidBundledAssembly
 		}
 
 		// Keep in sync with FORMAT_TAG in src/monodroid/jni/xamarin-app.hh
-		const ulong FORMAT_TAG = 0x015E6972616D58;
+		const ulong FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version
 
 		SortedDictionary <string, string>? environmentVariables;
 		SortedDictionary <string, string>? systemProperties;

src/Xamarin.Android.Build.Tasks/Utilities/TypeMapHelper.cs

@@ -0,0 +1,31 @@
+using System;
+using System.IO.Hashing;
+using System.Text;
+
+namespace Xamarin.Android.Tasks;
+
+static class TypeMapHelper
+{
+	/// <summary>
+	/// Hash the given Java type name for use in java-to-managed typemap array.
+	/// </summary>
+	public static ulong HashJavaName (string name, bool is64Bit)
+	{
+		if (name.Length == 0) {
+			return UInt64.MaxValue;
+		}
+
+		// Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do
+		// the same
+		return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit);
+	}
+
+	static ulong HashBytes (byte[] bytes, bool is64Bit)
+	{
+		if (is64Bit) {
+			return XxHash64.HashToUInt64 (bytes);
+		}
+
+		return (ulong)XxHash32.HashToUInt32 (bytes);
+	}
+}

src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs

@@ -403,32 +403,12 @@ void HashJavaNames (ConstructionState cs)
 				TypeMapJava entry = cs.JavaMap[i].Instance;
 
 				// The cast is safe, xxHash will return a 32-bit value which (for convenience) was upcast to 64-bit
-				entry.JavaNameHash32 = (uint)HashName (entry.JavaName, is64Bit: false);
+				entry.JavaNameHash32 = (uint)TypeMapHelper.HashJavaName (entry.JavaName, is64Bit: false);
 				hashes32.Add (entry.JavaNameHash32);
 
-				entry.JavaNameHash64 = HashName (entry.JavaName, is64Bit: true);
+				entry.JavaNameHash64 = TypeMapHelper.HashJavaName (entry.JavaName, is64Bit: true);
 				hashes64.Add (entry.JavaNameHash64);
 			}
-
-			ulong HashName (string name, bool is64Bit)
-			{
-				if (name.Length == 0) {
-					return UInt64.MaxValue;
-				}
-
-				// Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do
-				// the same
-				return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit);
-			}
-
-			ulong HashBytes (byte[] bytes, bool is64Bit)
-			{
-				if (is64Bit) {
-					return XxHash64.HashToUInt64 (bytes);
-				}
-
-				return (ulong)XxHash32.HashToUInt32 (bytes);
-			}
 		}
 	}
 }

src/monodroid/jni/xamarin-app.hh

@@ -10,7 +10,7 @@
 #include "monodroid.h"
 #include "xxhash.hh"
 
-static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58;
+static constexpr uint64_t FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version
 static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian
 static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian
 static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the

tools/assembly-store-reader/AssemblyStoreExplorer.cs

@@ -87,6 +87,17 @@ public AssemblyStoreExplorer (string storePath, Action<AssemblyStoreExplorerLogL
 			ProcessStores ();
 		}
 
+		public AssemblyStoreExplorer (ZipArchive archive, string basePathInArchive, Action<AssemblyStoreExplorerLogLevel, string>? customLogger = null, bool keepStoreInMemory = false)
+		{
+			logger = customLogger;
+			this.keepStoreInMemory = keepStoreInMemory;
+			StorePath = "<in-memory-archive>";
+			StoreSetName = StorePath;
+			ReadStoreSetFromArchive (archive, basePathInArchive);
+
+			ProcessStores ();
+		}
+
 		void Logger (AssemblyStoreExplorerLogLevel level, string message)
 		{
 			if (level == AssemblyStoreExplorerLogLevel.Error) {

tools/tmt/AnELF.cs

@@ -74,28 +74,35 @@ public bool HasSymbol (string symbolName)
 			return GetSymbol (symbolName) != null;
 		}
 
-		public byte[] GetData (string symbolName)
+		public (byte[] data, ISymbolEntry? symbol) GetData (string symbolName)
 		{
 			Log.Debug ($"Looking for symbol: {symbolName}");
 			ISymbolEntry? symbol = GetSymbol (symbolName);
-			if (symbol == null)
-				return EmptyArray;
+			if (symbol == null) {
+				return (EmptyArray, null);
+			}
 
 			if (Is64Bit) {
 				var symbol64 = symbol as SymbolEntry<ulong>;
 				if (symbol64 == null)
 					throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 64-bit symbol");
-				return GetData (symbol64);
+				return (GetData (symbol64), symbol);
 			}
 
 			var symbol32 = symbol as SymbolEntry<uint>;
 			if (symbol32 == null)
 				throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 32-bit symbol");
 
-			return GetData (symbol32);
+			return (GetData (symbol32), symbol);
 		}
 
 		public abstract byte[] GetData (ulong symbolValue, ulong size);
+		public abstract byte[] GetDataFromPointer (ulong pointerValue, ulong size);
+
+		public string GetASCIIZFromPointer (ulong pointerValue)
+		{
+			return GetASCIIZ (GetDataFromPointer (pointerValue, 0), 0);
+		}
 
 		public string GetASCIIZ (ulong symbolValue)
 		{
@@ -134,14 +141,27 @@ protected byte[] GetData (ISymbolEntry symbol, ulong size, ulong offset)
 			return GetData (symbol.PointedSection, size, offset);
 		}
 
+		void LogSectionInfo<T> (Section<T> section) where T: struct
+		{
+			Log.Debug ($"  section offset in file: 0x{section.Offset:x}; section size: {section.Size}; alignment: {section.Alignment}");
+		}
+
 		protected byte[] GetData (ISection section, ulong size, ulong offset)
 		{
-			Log.Debug ($"AnELF.GetData: section == {section.Name}; size == {size}; offset == {offset:X}");
+			Log.Debug ($"AnELF.GetData: section == '{section.Name}'; requested data size == {size}; offset in section == 0x{offset:x}");
 			byte[] data = section.GetContents ();
 
-			Log.Debug ($"  data length: {data.Length} (long: {data.LongLength})");
-			Log.Debug ($"  offset: {offset}; size: {size}");
+			if (section is Section<ulong> sec64) {
+				LogSectionInfo (sec64);
+			} else if (section is Section<uint> sec32) {
+				LogSectionInfo (sec32);
+			} else {
+				throw new NotSupportedException ($"Are we in the 128-bit future yet? Unsupported section type {section.GetType ()}");
+			}
+
+			Log.Debug ($"  section data length: {data.Length} (long: {data.LongLength})");
 			if ((ulong)data.LongLength < (offset + size)) {
+				Log.Debug ($"  not enough data in section");
 				return EmptyArray;
 			}
 
@@ -156,9 +176,18 @@ protected byte[] GetData (ISection section, ulong size, ulong offset)
 			return ret;
 		}
 
+		/// <summary>
+		/// Find a relocation corresponding to a pointer at offset <paramref name="pointerOffset"/> into
+		/// the specified <paramref name="symbol"/>.  Returns an `ulong`, which needs to be cast to `uint`
+		/// for 32-pointers (it can be done safely as the upper 32-bits will be 0 in such cases)
+		/// </summary>
+		public abstract ulong DeterminePointerAddress (ISymbolEntry symbol, ulong pointerOffset);
+		public abstract ulong DeterminePointerAddress (ulong symbolValue, ulong pointerOffset);
+
 		public uint GetUInt32 (string symbolName)
 		{
-			return GetUInt32 (GetData (symbolName), 0, symbolName);
+			(byte[] data, _) = GetData (symbolName);
+			return GetUInt32 (data, 0, symbolName);
 		}
 
 		public uint GetUInt32 (ulong symbolValue)
@@ -177,7 +206,8 @@ protected uint GetUInt32 (byte[] data, ulong offset, string symbolName)
 
 		public ulong GetUInt64 (string symbolName)
 		{
-			return GetUInt64 (GetData (symbolName), 0, symbolName);
+			(byte[] data, _) = GetData (symbolName);
+			return GetUInt64 (data, 0, symbolName);
 		}
 
 		public ulong GetUInt64 (ulong symbolValue)

tools/tmt/ApkManagedTypeResolver.cs

@@ -4,6 +4,7 @@
 
 using K4os.Compression.LZ4;
 using Mono.Cecil;
+using Xamarin.Android.AssemblyStore;
 using Xamarin.Tools.Zip;
 
 namespace tmt
@@ -12,15 +13,44 @@ class ApkManagedTypeResolver : ManagedTypeResolver
 	{
 		const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian
 
-		Dictionary<string, ZipEntry> assemblies;
-		ZipArchive apk;
+		readonly Dictionary<string, ZipEntry>? individualAssemblies;
+		readonly Dictionary<string, AssemblyStoreAssembly>? blobAssemblies;
+		readonly ZipArchive apk;
+		readonly AssemblyStoreExplorer? assemblyStoreExplorer;
 
 		public ApkManagedTypeResolver (ZipArchive apk, string assemblyEntryPrefix)
 		{
 			this.apk = apk;
-			assemblies = new Dictionary<string, ZipEntry> (StringComparer.Ordinal);
 
-			foreach (ZipEntry entry in apk) {
+			if (apk.ContainsEntry ($"{assemblyEntryPrefix}assemblies.blob")) {
+				blobAssemblies = new Dictionary<string, AssemblyStoreAssembly> (StringComparer.Ordinal);
+				assemblyStoreExplorer = new AssemblyStoreExplorer (apk, assemblyEntryPrefix, keepStoreInMemory: true);
+				LoadAssemblyBlobs (apk, assemblyEntryPrefix, assemblyStoreExplorer);
+			} else {
+				individualAssemblies = new Dictionary<string, ZipEntry> (StringComparer.Ordinal);
+				LoadIndividualAssemblies (apk, assemblyEntryPrefix);
+			}
+		}
+
+		void LoadAssemblyBlobs (ZipArchive apkArchive, string assemblyEntryPrefix, AssemblyStoreExplorer explorer)
+		{
+			foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) {
+				string assemblyName = assembly.Name;
+				string dllName = assembly.DllName;
+
+				if (!String.IsNullOrEmpty (assembly.Store.Arch)) {
+					assemblyName = $"{assembly.Store.Arch}/{assemblyName}";
+					dllName = $"{assembly.Store.Arch}/{dllName}";
+				}
+
+				blobAssemblies!.Add (assemblyName, assembly);
+				blobAssemblies!.Add (dllName, assembly);
+			}
+		}
+
+		void LoadIndividualAssemblies (ZipArchive apkArchive, string assemblyEntryPrefix)
+		{
+			foreach (ZipEntry entry in apkArchive) {
 				if (!entry.FullName.StartsWith (assemblyEntryPrefix, StringComparison.Ordinal)) {
 					continue;
 				}
@@ -29,61 +59,104 @@ public ApkManagedTypeResolver (ZipArchive apk, string assemblyEntryPrefix)
 					continue;
 				}
 
-				assemblies.Add (Path.GetFileNameWithoutExtension (entry.FullName), entry);
-				assemblies.Add (entry.FullName, entry);
+				string relativeName = entry.FullName.Substring (assemblyEntryPrefix.Length);
+				string? dir = Path.GetDirectoryName (relativeName);
+				string name = Path.GetFileNameWithoutExtension (relativeName);
+				if (!String.IsNullOrEmpty (dir)) {
+					name = $"{dir}/{name}";
+				}
+
+				individualAssemblies!.Add (name, entry);
+				individualAssemblies.Add (entry.FullName, entry);
 			}
 		}
 
 		protected override string? FindAssembly (string assemblyName)
 		{
-			if (assemblies.Count == 0) {
-				return null;
+			if (individualAssemblies != null) {
+				if (individualAssemblies.Count == 0) {
+					return null;
+				}
+
+				if (!individualAssemblies.TryGetValue (assemblyName, out ZipEntry? entry) || entry == null) {
+					return null;
+				}
+
+				return entry.FullName;
 			}
 
-			if (!assemblies.TryGetValue (assemblyName, out ZipEntry? entry) || entry == null) {
+			if (blobAssemblies == null || !blobAssemblies.TryGetValue (assemblyName, out AssemblyStoreAssembly? assembly) || assembly == null) {
 				return null;
 			}
 
-			return entry.FullName;
+			return assembly.Name;
 		}
 
-		protected override AssemblyDefinition ReadAssembly (string assemblyPath)
+		Stream GetAssemblyStream (string assemblyPath)
 		{
-			if (!assemblies.TryGetValue (assemblyPath, out ZipEntry? entry) || entry == null) {
+			MemoryStream? stream = null;
+			if (individualAssemblies != null) {
+				if (!individualAssemblies.TryGetValue (assemblyPath, out ZipEntry? entry) || entry == null) {
+					// Should "never" happen - if the assembly wasn't there, FindAssembly should have returned `null`
+					throw new InvalidOperationException ($"Should not happen: assembly '{assemblyPath}' not found in the APK archive.");
+				}
+
+				stream = new MemoryStream ();
+				entry.Extract (stream);
+				return PrepStream (stream);
+			}
+
+			if (blobAssemblies == null) {
+				throw new InvalidOperationException ("Internal error: blobAssemblies shouldn't be null");
+			}
+
+			if (blobAssemblies == null || !blobAssemblies.TryGetValue (assemblyPath, out AssemblyStoreAssembly? assembly) || assembly == null) {
 				// Should "never" happen - if the assembly wasn't there, FindAssembly should have returned `null`
-				throw new InvalidOperationException ($"Should not happen: assembly {assemblyPath} not found in the APK archive.");
+				throw new InvalidOperationException ($"Should not happen: assembly '{assemblyPath}' not found in the assembly blob.");
 			}
 
+			stream = new MemoryStream ();
+                        assembly.ExtractImage (stream);
+
+			return PrepStream (stream);
+
+			Stream PrepStream (Stream stream)
+			{
+				stream.Seek (0, SeekOrigin.Begin);
+				return stream;
+			}
+		}
+
+		protected override AssemblyDefinition ReadAssembly (string assemblyPath)
+		{
 			byte[]? assemblyBytes = null;
-			var stream = new MemoryStream ();
-			entry.Extract (stream);
-			stream.Seek (0, SeekOrigin.Begin);
+			Stream stream = GetAssemblyStream (assemblyPath);
 
 			//
 			// LZ4 compressed assembly header format:
 			//   uint magic;                 // 0x5A4C4158; 'XALZ', little-endian
 			//   uint descriptor_index;      // Index into an internal assembly descriptor table
 			//   uint uncompressed_length;   // Size of assembly, uncompressed
 			//
-			using (var reader = new BinaryReader (stream)) {
-				uint magic = reader.ReadUInt32 ();
-				if (magic == CompressedDataMagic) {
-					reader.ReadUInt32 (); // descriptor index, ignore
-					uint decompressedLength = reader.ReadUInt32 ();
-
-					int inputLength = (int)(stream.Length - 12);
-					byte[] sourceBytes = Utilities.BytePool.Rent (inputLength);
-					reader.Read (sourceBytes, 0, inputLength);
-
-					assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength);
-					int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength);
-					if (decoded != (int)decompressedLength) {
-						throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})");
-					}
-					Utilities.BytePool.Return (sourceBytes);
+			using var reader = new BinaryReader (stream);
+			uint magic = reader.ReadUInt32 ();
+			if (magic == CompressedDataMagic) {
+				reader.ReadUInt32 (); // descriptor index, ignore
+				uint decompressedLength = reader.ReadUInt32 ();
+
+				int inputLength = (int)(stream.Length - 12);
+				byte[] sourceBytes = Utilities.BytePool.Rent (inputLength);
+				reader.Read (sourceBytes, 0, inputLength);
+
+				assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength);
+				int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength);
+				if (decoded != (int)decompressedLength) {
+					throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})");
 				}
+				Utilities.BytePool.Return (sourceBytes);
 			}
 
+
 			if (assemblyBytes != null) {
 				stream.Close ();
 				stream.Dispose ();