-
Notifications
You must be signed in to change notification settings - Fork 564
[marshal methods] Part 2 of ? #7123
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
3f6fc67
2c9e2be
20cff42
9b2711c
146b745
776cb7d
52586cd
86de7c2
e99febb
f3c1d34
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -124,7 +124,12 @@ void Run (DirectoryAssemblyResolver res) | |
| if (Directory.Exists (dir.ItemSpec)) | ||
| res.SearchDirectories.Add (dir.ItemSpec); | ||
| } | ||
|
|
||
| #if ENABLE_MARSHAL_METHODS | ||
| Dictionary<string, HashSet<string>> marshalMethodsAssemblyPaths = null; | ||
| if (!Debug) { | ||
| marshalMethodsAssemblyPaths = new Dictionary<string, HashSet<string>> (StringComparer.Ordinal); | ||
| } | ||
| #endif | ||
| // Put every assembly we'll need in the resolver | ||
| bool hasExportReference = false; | ||
| bool haveMonoAndroid = false; | ||
|
|
@@ -159,6 +164,11 @@ void Run (DirectoryAssemblyResolver res) | |
| } | ||
|
|
||
| res.Load (assembly.ItemSpec); | ||
| #if ENABLE_MARSHAL_METHODS | ||
| if (!Debug) { | ||
| StoreMarshalAssemblyPath (Path.GetFileNameWithoutExtension (assembly.ItemSpec), assembly); | ||
| } | ||
| #endif | ||
| } | ||
|
|
||
| // However we only want to look for JLO types in user code for Java stub code generation | ||
|
|
@@ -172,6 +182,9 @@ void Run (DirectoryAssemblyResolver res) | |
| string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); | ||
| if (!userAssemblies.ContainsKey (name)) | ||
| userAssemblies.Add (name, asm.ItemSpec); | ||
| #if ENABLE_MARSHAL_METHODS | ||
| StoreMarshalAssemblyPath (name, asm); | ||
| #endif | ||
| } | ||
|
|
||
| // Step 1 - Find all the JLO types | ||
|
|
@@ -182,10 +195,6 @@ void Run (DirectoryAssemblyResolver res) | |
|
|
||
| List<TypeDefinition> allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); | ||
|
|
||
| // Step 2 - Generate type maps | ||
| // Type mappings need to use all the assemblies, always. | ||
| WriteTypeMappings (allJavaTypes, cache); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why move this? Type mappings should be unrelated to the contents of the
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Generating marshal methods also potentially rewrites assemblies, thus changing token ids and type maps must absolutely see the final ones. |
||
|
|
||
| var javaTypes = new List<TypeDefinition> (); | ||
| foreach (TypeDefinition td in allJavaTypes) { | ||
| if (!userAssemblies.ContainsKey (td.Module.Assembly.Name.Name) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) { | ||
|
|
@@ -195,11 +204,27 @@ void Run (DirectoryAssemblyResolver res) | |
| javaTypes.Add (td); | ||
| } | ||
|
|
||
| // Step 3 - Generate Java stub code | ||
| var success = CreateJavaSources (javaTypes, cache); | ||
| MarshalMethodsClassifier classifier = null; | ||
| #if ENABLE_MARSHAL_METHODS | ||
| if (!Debug) { | ||
| classifier = new MarshalMethodsClassifier (cache, res, Log); | ||
| } | ||
| #endif | ||
| // Step 2 - Generate Java stub code | ||
| var success = CreateJavaSources (javaTypes, cache, classifier); | ||
| if (!success) | ||
| return; | ||
|
|
||
| #if ENABLE_MARSHAL_METHODS | ||
| if (!Debug) { | ||
| var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log); | ||
| rewriter.Rewrite (res); | ||
| } | ||
| #endif | ||
| // Step 3 - Generate type maps | ||
| // Type mappings need to use all the assemblies, always. | ||
| WriteTypeMappings (allJavaTypes, cache); | ||
|
|
||
| // We need to save a map of .NET type -> ACW type for resource file fixups | ||
| var managed = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal); | ||
| var java = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal); | ||
|
|
@@ -210,7 +235,7 @@ void Run (DirectoryAssemblyResolver res) | |
| using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { | ||
| foreach (TypeDefinition type in javaTypes) { | ||
| string managedKey = type.FullName.Replace ('/', '.'); | ||
| string javaKey = JavaNativeTypeManager.ToJniName (type).Replace ('/', '.'); | ||
| string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); | ||
|
|
||
| acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); | ||
| acw_map.Write (';'); | ||
|
|
@@ -332,17 +357,30 @@ void Run (DirectoryAssemblyResolver res) | |
| string applicationTemplateFile = "ApplicationRegistration.java"; | ||
| SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, | ||
| template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); | ||
|
|
||
| #if ENABLE_MARSHAL_METHODS | ||
| void StoreMarshalAssemblyPath (string name, ITaskItem asm) | ||
| { | ||
| if (Debug) { | ||
| return; | ||
| } | ||
|
|
||
| if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet<string> assemblyPaths)) { | ||
| assemblyPaths = new HashSet<string> (); | ||
| marshalMethodsAssemblyPaths.Add (name, assemblyPaths); | ||
| } | ||
|
|
||
| assemblyPaths.Add (asm.ItemSpec); | ||
| } | ||
| #endif | ||
| } | ||
|
|
||
| bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache) | ||
| bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier) | ||
| { | ||
| string outputPath = Path.Combine (OutputDirectory, "src"); | ||
| string monoInit = GetMonoInitSource (AndroidSdkPlatform); | ||
| bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); | ||
| bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; | ||
| #if ENABLE_MARSHAL_METHODS | ||
| var overriddenMethodDescriptors = new List<OverriddenMethodDescriptor> (); | ||
| #endif | ||
|
|
||
| bool ok = true; | ||
| foreach (var t in javaTypes) { | ||
|
|
@@ -353,15 +391,19 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac | |
|
|
||
| using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { | ||
| try { | ||
| var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache) { | ||
| var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache, classifier) { | ||
| GenerateOnCreateOverrides = generateOnCreateOverrides, | ||
| ApplicationJavaClass = ApplicationJavaClass, | ||
| MonoRuntimeInitialization = monoInit, | ||
| }; | ||
|
|
||
| jti.Generate (writer); | ||
| #if ENABLE_MARSHAL_METHODS | ||
| overriddenMethodDescriptors.AddRange (jti.OverriddenMethodDescriptors); | ||
| if (!Debug) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We could remove this But that means relying on "implicit context". Not sure which tradeoff is "better": minimizing indentation or keeping things explicit.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I much prefer if the code is explicit, makes it easier to understand the flow for someone not familiar with it |
||
| if (classifier.FoundDynamicallyRegisteredMethods) { | ||
| Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName ()}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); | ||
| } | ||
| } | ||
| #endif | ||
| writer.Flush (); | ||
|
|
||
|
|
@@ -397,7 +439,9 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac | |
| } | ||
| } | ||
| #if ENABLE_MARSHAL_METHODS | ||
| BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, overriddenMethodDescriptors, RegisteredTaskObjectLifetime.Build); | ||
| if (!Debug) { | ||
| BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, new MarshalMethodsState (classifier.MarshalMethods), RegisteredTaskObjectLifetime.Build); | ||
| } | ||
| #endif | ||
| return ok; | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| #if ENABLE_MARSHAL_METHODS | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
|
|
||
| using Java.Interop.Tools.Cecil; | ||
| using Microsoft.Android.Build.Tasks; | ||
| using Microsoft.Build.Utilities; | ||
| using Mono.Cecil; | ||
| using Xamarin.Android.Tools; | ||
|
|
||
| namespace Xamarin.Android.Tasks | ||
| { | ||
| class MarshalMethodsAssemblyRewriter | ||
| { | ||
| IDictionary<string, IList<MarshalMethodEntry>> methods; | ||
| ICollection<AssemblyDefinition> uniqueAssemblies; | ||
| IDictionary <string, HashSet<string>> assemblyPaths; | ||
| TaskLoggingHelper log; | ||
|
|
||
| public MarshalMethodsAssemblyRewriter (IDictionary<string, IList<MarshalMethodEntry>> methods, ICollection<AssemblyDefinition> uniqueAssemblies, IDictionary <string, HashSet<string>> assemblyPaths, TaskLoggingHelper log) | ||
| { | ||
| this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); | ||
| this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); | ||
| this.assemblyPaths = assemblyPaths ?? throw new ArgumentNullException (nameof (assemblyPaths)); | ||
| this.log = log ?? throw new ArgumentNullException (nameof (log)); | ||
| } | ||
|
|
||
| public void Rewrite (DirectoryAssemblyResolver resolver) | ||
| { | ||
| MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); | ||
| var unmanagedCallersOnlyAttributes = new Dictionary<AssemblyDefinition, CustomAttribute> (); | ||
| foreach (AssemblyDefinition asm in uniqueAssemblies) { | ||
| unmanagedCallersOnlyAttributes.Add (asm, CreateImportedUnmanagedCallersOnlyAttribute (asm, unmanagedCallersOnlyAttributeCtor)); | ||
| } | ||
|
|
||
| Console.WriteLine ("Adding the [UnmanagedCallersOnly] attribute to native callback methods and removing unneeded fields+methods"); | ||
| foreach (IList<MarshalMethodEntry> methodList in methods.Values) { | ||
| foreach (MarshalMethodEntry method in methodList) { | ||
| Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})"); | ||
| method.NativeCallback.CustomAttributes.Add (unmanagedCallersOnlyAttributes [method.NativeCallback.Module.Assembly]); | ||
| method.Connector.DeclaringType.Methods.Remove (method.Connector); | ||
| method.CallbackField?.DeclaringType.Fields.Remove (method.CallbackField); | ||
| } | ||
| } | ||
|
|
||
| Console.WriteLine (); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we'll be wanting
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The CWLs will be gone eventually, I will remove them in the last PR in the series |
||
| Console.WriteLine ("Rewriting assemblies"); | ||
|
|
||
| var newAssemblyPaths = new List<string> (); | ||
| foreach (AssemblyDefinition asm in uniqueAssemblies) { | ||
| foreach (string path in GetAssemblyPaths (asm)) { | ||
| var writerParams = new WriterParameters { | ||
| WriteSymbols = (File.Exists (path + ".mdb") || File.Exists (Path.ChangeExtension (path, ".pdb"))), | ||
| }; | ||
|
|
||
| string output = $"{path}.new"; | ||
| Console.WriteLine ($"\t{asm.Name} => {output}"); | ||
| asm.Write (output, writerParams); | ||
| newAssemblyPaths.Add (output); | ||
| } | ||
| } | ||
|
|
||
| // Replace old versions of the assemblies only after we've finished rewriting without issues, otherwise leave the new | ||
| // versions around. | ||
| foreach (string path in newAssemblyPaths) { | ||
| string target = Path.Combine (Path.GetDirectoryName (path), Path.GetFileNameWithoutExtension (path)); | ||
| MoveFile (path, target); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We may want/need to rework the build system so that we write files into new/separate directories instead of modifying files in-place, otherwise we'll provoke the incremental build + file sharing gods.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I asked @jonathanpeppers about this and he thought it was better to write them in place. We can optimize this in the last PR in the series though |
||
|
|
||
| string source = Path.ChangeExtension (path, ".pdb"); | ||
| if (File.Exists (source)) { | ||
| target = Path.ChangeExtension (Path.Combine (Path.GetDirectoryName (source), Path.GetFileNameWithoutExtension (source)), ".pdb"); | ||
|
|
||
| MoveFile (source, target); | ||
| } | ||
|
|
||
| source = $"{path}.mdb"; | ||
| if (File.Exists (source)) { | ||
| target = Path.ChangeExtension (path, ".mdb"); | ||
| MoveFile (source, target); | ||
| } | ||
| } | ||
|
|
||
| Console.WriteLine (); | ||
| Console.WriteLine ("Method tokens:"); | ||
| foreach (IList<MarshalMethodEntry> methodList in methods.Values) { | ||
| foreach (MarshalMethodEntry method in methodList) { | ||
| Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})"); | ||
| } | ||
| } | ||
|
|
||
| void MoveFile (string source, string target) | ||
| { | ||
| Console.WriteLine ($"Moving '{source}' => '{target}'"); | ||
dellis1972 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| Files.CopyIfChanged (source, target); | ||
| try { | ||
| File.Delete (source); | ||
| } catch (Exception ex) { | ||
| log.LogWarning ($"Unable to delete source file '{source}' when moving it to '{target}'"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| ICollection<string> GetAssemblyPaths (AssemblyDefinition asm) | ||
| { | ||
| if (!assemblyPaths.TryGetValue (asm.Name.Name, out HashSet<string> paths)) { | ||
| throw new InvalidOperationException ($"Unable to determine file path for assembly '{asm.Name.Name}'"); | ||
| } | ||
|
|
||
| return paths; | ||
| } | ||
|
|
||
| MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (DirectoryAssemblyResolver resolver) | ||
| { | ||
| AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices"); | ||
| TypeDefinition unmanagedCallersOnlyAttribute = null; | ||
| foreach (ModuleDefinition md in asm.Modules) { | ||
| foreach (ExportedType et in md.ExportedTypes) { | ||
| if (!et.IsForwarder) { | ||
| continue; | ||
| } | ||
|
|
||
| if (String.Compare ("System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute", et.FullName, StringComparison.Ordinal) != 0) { | ||
| continue; | ||
| } | ||
|
|
||
| unmanagedCallersOnlyAttribute = et.Resolve (); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| if (unmanagedCallersOnlyAttribute == null) { | ||
| throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type"); | ||
| } | ||
|
|
||
| foreach (MethodDefinition md in unmanagedCallersOnlyAttribute.Methods) { | ||
| if (!md.IsConstructor) { | ||
| continue; | ||
| } | ||
|
|
||
| return md; | ||
| } | ||
|
|
||
| throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor"); | ||
| } | ||
|
|
||
| CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition targetAssembly, MethodDefinition unmanagedCallersOnlyAtributeCtor) | ||
| { | ||
| return new CustomAttribute (targetAssembly.MainModule.ImportReference (unmanagedCallersOnlyAtributeCtor)); | ||
| } | ||
| } | ||
| } | ||
| #endif | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. At this point I suspect I'm the only one who remembers this convention, but please place a comment on #endif // ENABLE_MARSHAL_METHODS
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I apply the convention when it's there to stay :) This macro is going to go away in the last PR of the series, it's just a measure to not "poison" current code until marshal methods are finished |
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why not skip these
if (!Debug) StoreMarshalAssemblyPath()checks and just callStoreMarshalAssemblyPath())?StoreMarshalAssemblyPath()checksDebuganyway, so this is just adding extra levels of complexity & indentation, for no observable benefit.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In the first case we save one string allocation, in the second instance yes, it's not necessary