Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion external/Java.Interop
74 changes: 59 additions & 15 deletions src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,12 @@ void Run (DirectoryAssemblyResolver res)
if (Directory.Exists (dir.ItemSpec))
res.SearchDirectories.Add (dir.ItemSpec);
}

#if ENABLE_MARSHAL_METHODS
Dictionary<string, HashSet<string>> marshalMethodsAssemblyPaths = null;
if (!Debug) {
marshalMethodsAssemblyPaths = new Dictionary<string, HashSet<string>> (StringComparer.Ordinal);
}
#endif
// Put every assembly we'll need in the resolver
bool hasExportReference = false;
bool haveMonoAndroid = false;
Expand Down Expand Up @@ -159,6 +164,11 @@ void Run (DirectoryAssemblyResolver res)
}

res.Load (assembly.ItemSpec);
#if ENABLE_MARSHAL_METHODS
if (!Debug) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not skip these if (!Debug) StoreMarshalAssemblyPath() checks and just call StoreMarshalAssemblyPath())? StoreMarshalAssemblyPath() checks Debug anyway, so this is just adding extra levels of complexity & indentation, for no observable benefit.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first case we save one string allocation, in the second instance yes, it's not necessary

StoreMarshalAssemblyPath (Path.GetFileNameWithoutExtension (assembly.ItemSpec), assembly);
}
#endif
}

// However we only want to look for JLO types in user code for Java stub code generation
Expand All @@ -172,6 +182,9 @@ void Run (DirectoryAssemblyResolver res)
string name = Path.GetFileNameWithoutExtension (asm.ItemSpec);
if (!userAssemblies.ContainsKey (name))
userAssemblies.Add (name, asm.ItemSpec);
#if ENABLE_MARSHAL_METHODS
StoreMarshalAssemblyPath (name, asm);
#endif
}

// Step 1 - Find all the JLO types
Expand All @@ -182,10 +195,6 @@ void Run (DirectoryAssemblyResolver res)

List<TypeDefinition> allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res);

// Step 2 - Generate type maps
// Type mappings need to use all the assemblies, always.
WriteTypeMappings (allJavaTypes, cache);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why move this? Type mappings should be unrelated to the contents of the .java files, and allJavaTypes appears to be unchanged, so…. What "hidden"/"non-obvious" bits am I not seeing?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generating marshal methods also potentially rewrites assemblies, thus changing token ids and type maps must absolutely see the final ones.


var javaTypes = new List<TypeDefinition> ();
foreach (TypeDefinition td in allJavaTypes) {
if (!userAssemblies.ContainsKey (td.Module.Assembly.Name.Name) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) {
Expand All @@ -195,11 +204,27 @@ void Run (DirectoryAssemblyResolver res)
javaTypes.Add (td);
}

// Step 3 - Generate Java stub code
var success = CreateJavaSources (javaTypes, cache);
MarshalMethodsClassifier classifier = null;
#if ENABLE_MARSHAL_METHODS
if (!Debug) {
classifier = new MarshalMethodsClassifier (cache, res, Log);
}
#endif
// Step 2 - Generate Java stub code
var success = CreateJavaSources (javaTypes, cache, classifier);
if (!success)
return;

#if ENABLE_MARSHAL_METHODS
if (!Debug) {
var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log);
rewriter.Rewrite (res);
}
#endif
// Step 3 - Generate type maps
// Type mappings need to use all the assemblies, always.
WriteTypeMappings (allJavaTypes, cache);

// We need to save a map of .NET type -> ACW type for resource file fixups
var managed = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal);
var java = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal);
Expand All @@ -210,7 +235,7 @@ void Run (DirectoryAssemblyResolver res)
using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) {
foreach (TypeDefinition type in javaTypes) {
string managedKey = type.FullName.Replace ('/', '.');
string javaKey = JavaNativeTypeManager.ToJniName (type).Replace ('/', '.');
string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.');

acw_map.Write (type.GetPartialAssemblyQualifiedName (cache));
acw_map.Write (';');
Expand Down Expand Up @@ -332,17 +357,30 @@ void Run (DirectoryAssemblyResolver res)
string applicationTemplateFile = "ApplicationRegistration.java";
SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir,
template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()));

#if ENABLE_MARSHAL_METHODS
void StoreMarshalAssemblyPath (string name, ITaskItem asm)
{
if (Debug) {
return;
}

if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet<string> assemblyPaths)) {
assemblyPaths = new HashSet<string> ();
marshalMethodsAssemblyPaths.Add (name, assemblyPaths);
}

assemblyPaths.Add (asm.ItemSpec);
}
#endif
}

bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache)
bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier)
{
string outputPath = Path.Combine (OutputDirectory, "src");
string monoInit = GetMonoInitSource (AndroidSdkPlatform);
bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll");
bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10;
#if ENABLE_MARSHAL_METHODS
var overriddenMethodDescriptors = new List<OverriddenMethodDescriptor> ();
#endif

bool ok = true;
foreach (var t in javaTypes) {
Expand All @@ -353,15 +391,19 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac

using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) {
try {
var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache) {
var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache, classifier) {
GenerateOnCreateOverrides = generateOnCreateOverrides,
ApplicationJavaClass = ApplicationJavaClass,
MonoRuntimeInitialization = monoInit,
};

jti.Generate (writer);
#if ENABLE_MARSHAL_METHODS
overriddenMethodDescriptors.AddRange (jti.OverriddenMethodDescriptors);
if (!Debug) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could remove this if + indentation if we just did if (classifier?.FoundDynamicallyRegisteredMethods), thus implicitly relying on the fact that classifier is null in Debug builds.

But that means relying on "implicit context".

Not sure which tradeoff is "better": minimizing indentation or keeping things explicit.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I much prefer if the code is explicit, makes it easier to understand the flow for someone not familiar with it

if (classifier.FoundDynamicallyRegisteredMethods) {
Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName ()}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods.");
}
}
#endif
writer.Flush ();

Expand Down Expand Up @@ -397,7 +439,9 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac
}
}
#if ENABLE_MARSHAL_METHODS
BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, overriddenMethodDescriptors, RegisteredTaskObjectLifetime.Build);
if (!Debug) {
BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, new MarshalMethodsState (classifier.MarshalMethods), RegisteredTaskObjectLifetime.Build);
}
#endif
return ok;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -442,10 +442,12 @@ void AddEnvironment ()
};
appConfigAsmGen.Init ();
#if ENABLE_MARSHAL_METHODS
var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator () {
var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<MarshalMethodsState> (GenerateJavaStubs.MarshalMethodsRegisterTaskKey, RegisteredTaskObjectLifetime.Build);

var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator {
NumberOfAssembliesInApk = assemblyCount,
UniqueAssemblyNames = uniqueAssemblyNames,
OverriddenMethodDescriptors = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<List<Java.Interop.Tools.JavaCallableWrappers.OverriddenMethodDescriptor>> (GenerateJavaStubs.MarshalMethodsRegisterTaskKey, RegisteredTaskObjectLifetime.Build)
MarshalMethods = marshalMethodsState?.MarshalMethods,
};
marshalMethodsAsmGen.Init ();
#endif
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#if ENABLE_MARSHAL_METHODS
using System;
using System.Collections.Generic;
using System.IO;

using Java.Interop.Tools.Cecil;
using Microsoft.Android.Build.Tasks;
using Microsoft.Build.Utilities;
using Mono.Cecil;
using Xamarin.Android.Tools;

namespace Xamarin.Android.Tasks
{
class MarshalMethodsAssemblyRewriter
{
IDictionary<string, IList<MarshalMethodEntry>> methods;
ICollection<AssemblyDefinition> uniqueAssemblies;
IDictionary <string, HashSet<string>> assemblyPaths;
TaskLoggingHelper log;

public MarshalMethodsAssemblyRewriter (IDictionary<string, IList<MarshalMethodEntry>> methods, ICollection<AssemblyDefinition> uniqueAssemblies, IDictionary <string, HashSet<string>> assemblyPaths, TaskLoggingHelper log)
{
this.methods = methods ?? throw new ArgumentNullException (nameof (methods));
this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies));
this.assemblyPaths = assemblyPaths ?? throw new ArgumentNullException (nameof (assemblyPaths));
this.log = log ?? throw new ArgumentNullException (nameof (log));
}

public void Rewrite (DirectoryAssemblyResolver resolver)
{
MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver);
var unmanagedCallersOnlyAttributes = new Dictionary<AssemblyDefinition, CustomAttribute> ();
foreach (AssemblyDefinition asm in uniqueAssemblies) {
unmanagedCallersOnlyAttributes.Add (asm, CreateImportedUnmanagedCallersOnlyAttribute (asm, unmanagedCallersOnlyAttributeCtor));
}

Console.WriteLine ("Adding the [UnmanagedCallersOnly] attribute to native callback methods and removing unneeded fields+methods");
foreach (IList<MarshalMethodEntry> methodList in methods.Values) {
foreach (MarshalMethodEntry method in methodList) {
Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})");
method.NativeCallback.CustomAttributes.Add (unmanagedCallersOnlyAttributes [method.NativeCallback.Module.Assembly]);
method.Connector.DeclaringType.Methods.Remove (method.Connector);
method.CallbackField?.DeclaringType.Fields.Remove (method.CallbackField);
}
}

Console.WriteLine ();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we'll be wanting Log.LogDebugMessage(), not Console.WriteLine()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The CWLs will be gone eventually, I will remove them in the last PR in the series

Console.WriteLine ("Rewriting assemblies");

var newAssemblyPaths = new List<string> ();
foreach (AssemblyDefinition asm in uniqueAssemblies) {
foreach (string path in GetAssemblyPaths (asm)) {
var writerParams = new WriterParameters {
WriteSymbols = (File.Exists (path + ".mdb") || File.Exists (Path.ChangeExtension (path, ".pdb"))),
};

string output = $"{path}.new";
Console.WriteLine ($"\t{asm.Name} => {output}");
asm.Write (output, writerParams);
newAssemblyPaths.Add (output);
}
}

// Replace old versions of the assemblies only after we've finished rewriting without issues, otherwise leave the new
// versions around.
foreach (string path in newAssemblyPaths) {
string target = Path.Combine (Path.GetDirectoryName (path), Path.GetFileNameWithoutExtension (path));
MoveFile (path, target);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We may want/need to rework the build system so that we write files into new/separate directories instead of modifying files in-place, otherwise we'll provoke the incremental build + file sharing gods.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I asked @jonathanpeppers about this and he thought it was better to write them in place. We can optimize this in the last PR in the series though


string source = Path.ChangeExtension (path, ".pdb");
if (File.Exists (source)) {
target = Path.ChangeExtension (Path.Combine (Path.GetDirectoryName (source), Path.GetFileNameWithoutExtension (source)), ".pdb");

MoveFile (source, target);
}

source = $"{path}.mdb";
if (File.Exists (source)) {
target = Path.ChangeExtension (path, ".mdb");
MoveFile (source, target);
}
}

Console.WriteLine ();
Console.WriteLine ("Method tokens:");
foreach (IList<MarshalMethodEntry> methodList in methods.Values) {
foreach (MarshalMethodEntry method in methodList) {
Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})");
}
}

void MoveFile (string source, string target)
{
Console.WriteLine ($"Moving '{source}' => '{target}'");
Files.CopyIfChanged (source, target);
try {
File.Delete (source);
} catch (Exception ex) {
log.LogWarning ($"Unable to delete source file '{source}' when moving it to '{target}'");
}
}
}

ICollection<string> GetAssemblyPaths (AssemblyDefinition asm)
{
if (!assemblyPaths.TryGetValue (asm.Name.Name, out HashSet<string> paths)) {
throw new InvalidOperationException ($"Unable to determine file path for assembly '{asm.Name.Name}'");
}

return paths;
}

MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (DirectoryAssemblyResolver resolver)
{
AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices");
TypeDefinition unmanagedCallersOnlyAttribute = null;
foreach (ModuleDefinition md in asm.Modules) {
foreach (ExportedType et in md.ExportedTypes) {
if (!et.IsForwarder) {
continue;
}

if (String.Compare ("System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute", et.FullName, StringComparison.Ordinal) != 0) {
continue;
}

unmanagedCallersOnlyAttribute = et.Resolve ();
break;
}
}

if (unmanagedCallersOnlyAttribute == null) {
throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type");
}

foreach (MethodDefinition md in unmanagedCallersOnlyAttribute.Methods) {
if (!md.IsConstructor) {
continue;
}

return md;
}

throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor");
}

CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition targetAssembly, MethodDefinition unmanagedCallersOnlyAtributeCtor)
{
return new CustomAttribute (targetAssembly.MainModule.ImportReference (unmanagedCallersOnlyAtributeCtor));
}
}
}
#endif
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At this point I suspect I'm the only one who remembers this convention, but please place a comment on #endif with the #if expression:

#endif  // ENABLE_MARSHAL_METHODS

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I apply the convention when it's there to stay :) This macro is going to go away in the last PR of the series, it's just a measure to not "poison" current code until marshal methods are finished

Loading