[generator] Import NRT data from managed assemblies (#912)

Fixes: #857

Context: #917

Imagine you have a `Mono.Android.dll` build which has nullable
reference types enabled, with a type declaration of:

	namespace Java.Lang {
	    public partial interface IIterable {
	        Java.Util.IIterator Iterator ();
	    }
	}

Then imagine that you bind a Java library which contains an interface
which implements `java.lang.Iterable`, resulting in a binding:

	namespace My.Example {
	    partial interface IMyIterable : Java.Lang.IIterable {
	    }
	}

`generator` also implements `IMyIterableInvoker`, to support
invoking the `IMyIterable` interface.  Unfortunately, because
`IIterable` comes from IL, `generator` doesn't know about nullability,
and thus assumes that *everything* can be nullable:

	namespace My.Example {
	    partial class IMyIterableInvoker : Java.Lang.Object, IMyIterable {
	        public Java.Util.IIterator? Iterator() {…}
	    }
	}

This results in a CS8766 warning:

	warning CS8766: Nullability of reference types in return type of
	'IIterator? IMyIterableInvoker.Iterator()' doesn't match implicitly implemented member
	'IIterator IIterable.Iterator()' (possibly because of nullability attributes).

Fix this use-case by updating `Java.Interop.Tools.Cecil.dll` to know
enough about the encoding of nullability information within
[`NullableAttribute` and C# Nullable Metadata][0] to determine if a
parameter type or return type can be null.  This allows `generator`
to appropriately emit:

	namespace My.Example {
	    partial class IMyIterableInvoker : Java.Lang.Object, IMyIterable {
	        public Java.Util.IIterator Iterator() {…}
	    }
	}

Note that our import doesn't support "nested nullability"; it only
handles nullability of the "outermost type".  It cannot differentiate
between `java.lang.String[]?` and `java.lang.String[]?` (possibly
null array that can have possibly null elements vs. possibly null
array that can only have non-null elements).  We don't currently
consider this to be a problem because most of the Java annotations
*also* don't support this form of differentiation.  Kotlin *can*;
`Array<String?>` differs from `Array<String?>?`.  However, that's
apparently not handled via Java annotations, but instead via the
Kotlin Protobuf data.

We don't feel a need to support nested nullability at this time.

TODO: All the above said… we think we made a mistake in 01d0684
around the binding of arrays, and other "nested generic-like" types.
We currently bind an annotation-less Java method such as
`example(Object[])` as `Example(Java.Lang.Object[]?)`, but we should
*actually* bind this as `Example(Java.Lang.Object?[]?)`!
This is tracked in #917

[0]: https://github.com/dotnet/roslyn/blob/2da789af0f404e15871b6e6e17706185a2707ab4/docs/features/nullable-metadata.md

Author: jpobst

src/Java.Interop.Tools.Cecil/Java.Interop.Tools.Cecil/NullableReferenceTypesRocks.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Linq;
+using Mono.Cecil;
+using Mono.Collections.Generic;
+
+namespace Java.Interop.Tools.Cecil
+{
+	// Partial support for determining NRT status of method and field types.
+	// Reference: https://github.com/dotnet/roslyn/blob/main/docs/features/nullable-metadata.md
+	// The basics are supported, but advanced annotations like array elements,
+	// type parameters, and tuples are not supported.  Our use case doesn't really need them.
+	public static class NullableReferenceTypesRocks
+	{
+		public static Nullability GetTypeNullability (this FieldDefinition field)
+		{
+			if (field.FieldType.FullName == "System.Void")
+				return Nullability.NotNull;
+
+			// Look for explicit annotation on field
+			var metadata = NullableMetadata.FromAttributeCollection (field.CustomAttributes);
+
+			if (metadata != null)
+				return (Nullability) metadata.Data [0];
+
+			// Default nullability status for type
+			return GetNullableContext (field.DeclaringType.CustomAttributes);
+		}
+
+		public static Nullability GetReturnTypeNullability (this MethodDefinition method)
+		{
+			if (method.MethodReturnType.ReturnType.FullName == "System.Void")
+				return Nullability.NotNull;
+
+			// Look for explicit annotation on return type
+			var metadata = NullableMetadata.FromAttributeCollection (method.MethodReturnType.CustomAttributes);
+
+			if (metadata != null)
+				return (Nullability) metadata.Data [0];
+
+			// Default nullability status for method
+			var nullable = GetNullableContext (method.CustomAttributes);
+
+			if (nullable != Nullability.Oblivous)
+				return nullable;
+
+			// Default nullability status for type
+			return GetNullableContext (method.DeclaringType.CustomAttributes);
+		}
+
+		public static Nullability GetTypeNullability (this ParameterDefinition parameter, MethodDefinition method)
+		{
+			if (parameter.ParameterType.FullName == "System.Void")
+				return Nullability.NotNull;
+
+			// Look for explicit annotation on parameter
+			var metadata = NullableMetadata.FromAttributeCollection (parameter.CustomAttributes);
+
+			if (metadata != null)
+				return (Nullability) metadata.Data [0];
+
+			// Default nullability status for method
+			var nullable = GetNullableContext (method.CustomAttributes);
+
+			if (nullable != Nullability.Oblivous)
+				return nullable;
+
+			// Default nullability status for type
+			return GetNullableContext (method.DeclaringType.CustomAttributes);
+		}
+
+		static Nullability GetNullableContext (Collection<CustomAttribute> attrs)
+		{
+			var attribute = attrs.FirstOrDefault (t => t.AttributeType.FullName == "System.Runtime.CompilerServices.NullableContextAttribute");
+
+			if (attribute != null)
+				return (Nullability) (byte) attribute.ConstructorArguments.First ().Value;
+
+			return Nullability.Oblivous;
+		}
+	}
+
+	public enum Nullability
+	{
+		Oblivous,
+		NotNull,
+		Nullable
+	}
+
+	class NullableMetadata
+	{
+		public byte [] Data { get; private set; }
+
+		NullableMetadata (byte [] data) => Data = data;
+
+		NullableMetadata (byte data) => Data = new [] { data };
+
+		public static NullableMetadata? FromAttributeCollection (Collection<CustomAttribute> attrs)
+		{
+			if (attrs is null)
+				return null;
+
+			var attribute = attrs.FirstOrDefault (t => t.AttributeType.FullName == "System.Runtime.CompilerServices.NullableAttribute");
+
+			if (attribute is null)
+				return null;
+
+			var ctor_arg = attribute.ConstructorArguments.First ();
+
+			if (ctor_arg.Value is CustomAttributeArgument [] caa)
+				ctor_arg = caa [0];
+
+			if (ctor_arg.Value is byte b)
+				return new NullableMetadata (b);
+
+			if (ctor_arg.Value is byte [] b2)
+				return new NullableMetadata (b2);
+
+			return null;
+		}
+	}
+}

tests/generator-Tests/Unit-Tests/ManagedTests.cs

@@ -64,6 +64,28 @@ public Dictionary<T, List<string>> DoStuff (IEnumerable<KeyValuePair<T, List<Lis
 	}
 }
 
+namespace NullableTestTypes
+{
+#nullable enable
+	public class NullableClass
+	{
+		[Register ("<init>", "(Ljava/lang/String;Ljava/lang/String;)", "")]
+		public NullableClass (string notnull, string? nullable)
+		{
+		}
+
+		public string? null_field;
+		public string not_null_field = string.Empty;
+
+		[Register ("nullable_return_method", "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", "")]
+		public string? NullableReturnMethod (string notnull, string? nullable) => null;
+
+		[Register ("not_null_return_method", "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", "")]
+		public string NotNullReturnMethod (string notnull, string? nullable) => string.Empty;
+	}
+}
+#nullable disable
+
 namespace generatortests
 {
 	[TestFixture]
@@ -250,5 +272,32 @@ public void StripArity ()
 			Assert.AreEqual ("System.Collections.Generic.Dictionary<T,System.Collections.Generic.List<System.String>>", @class.Methods [0].ReturnType);
 			Assert.AreEqual ("System.Collections.Generic.Dictionary<T,System.Collections.Generic.List<System.String>>", @class.Methods [0].ManagedReturn);
 		}
+
+		[Test]
+		public void TypeNullability ()
+		{
+			var type = module.GetType ("NullableTestTypes.NullableClass");
+			var gen = CecilApiImporter.CreateClass (module.GetType ("NullableTestTypes.NullableClass"), options);
+
+			var not_null_field = CecilApiImporter.CreateField (type.Fields.First (f => f.Name == "not_null_field"));
+			Assert.AreEqual (true, not_null_field.NotNull);
+
+			var null_field = CecilApiImporter.CreateField (type.Fields.First (f => f.Name == "null_field"));
+			Assert.AreEqual (false, null_field.NotNull);
+
+			var null_method = CecilApiImporter.CreateMethod (gen, type.Methods.First (f => f.Name == "NullableReturnMethod"));
+			Assert.AreEqual (false, null_method.ReturnNotNull);
+			Assert.AreEqual (true, null_method.Parameters.First (f => f.Name == "notnull").NotNull);
+			Assert.AreEqual (false, null_method.Parameters.First (f => f.Name == "nullable").NotNull);
+
+			var not_null_method = CecilApiImporter.CreateMethod (gen, type.Methods.First (f => f.Name == "NotNullReturnMethod"));
+			Assert.AreEqual (true, not_null_method.ReturnNotNull);
+			Assert.AreEqual (true, not_null_method.Parameters.First (f => f.Name == "notnull").NotNull);
+			Assert.AreEqual (false, not_null_method.Parameters.First (f => f.Name == "nullable").NotNull);
+
+			var ctor = CecilApiImporter.CreateCtor (gen, type.Methods.First (f => f.Name == ".ctor"));
+			Assert.AreEqual (true, ctor.Parameters.First (f => f.Name == "notnull").NotNull);
+			Assert.AreEqual (false, ctor.Parameters.First (f => f.Name == "nullable").NotNull);
+		}
 	}
 }

tests/generator-Tests/generator-Tests.csproj

@@ -4,6 +4,7 @@
     <TargetFrameworks>net472;net6.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <LangVersion>8.0</LangVersion>
   </PropertyGroup>
 
   <PropertyGroup>

tools/generator/Extensions/ManagedExtensions.cs

@@ -1,4 +1,5 @@
-using Java.Interop.Tools.TypeNameMappings;
+using Java.Interop.Tools.Cecil;
+using Java.Interop.Tools.TypeNameMappings;
 using Mono.Cecil;
 using System.Collections.Generic;
 using System.Linq;
@@ -42,7 +43,9 @@ public static IEnumerable<Parameter> GetParameters (this MethodDefinition m, Cus
 				// custom enum types and cannot simply use JNI signature here.
 				var rawtype = e?.Current.Type;
 				var type = p.ParameterType.FullName == "System.IO.Stream" && e != null ? e.Current.Type : null;
-				yield return CecilApiImporter.CreateParameter (p, type, rawtype);
+				var isNotNull = p.GetTypeNullability (m) == Nullability.NotNull;
+
+				yield return CecilApiImporter.CreateParameter (p, type, rawtype, isNotNull);
 			}
 		}
 

tools/generator/Java.Interop.Tools.Generator.Importers/CecilApiImporter.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using Java.Interop.Tools.Cecil;
 using Java.Interop.Tools.TypeNameMappings;
 using Mono.Cecil;
 using Mono.Collections.Generic;
@@ -101,6 +102,7 @@ public static Field CreateField (FieldDefinition f)
 				IsStatic = f.IsStatic,
 				JavaName = reg_attr != null ? ((string) reg_attr.ConstructorArguments [0].Value).Replace ('/', '.') : f.Name,
 				Name = f.Name,
+				NotNull = f.GetTypeNullability () == Nullability.NotNull,
 				TypeName = f.FieldType.FullNameCorrected ().StripArity (),
 				Value = f.Constant == null ? null : f.FieldType.FullName == "System.String" ? '"' + f.Constant.ToString () + '"' : f.Constant.ToString (),
 				Visibility = f.IsPublic ? "public" : f.IsFamilyOrAssembly ? "protected internal" : f.IsFamily ? "protected" : f.IsAssembly ? "internal" : "private"
@@ -198,6 +200,7 @@ public static Method CreateMethod (GenBase declaringType, MethodDefinition m)
 				JavaName = reg_attr != null ? ((string) reg_attr.ConstructorArguments [0].Value) : m.Name,
 				ManagedReturn = m.ReturnType.FullNameCorrected ().StripArity ().FilterPrimitive (),
 				Return = m.ReturnType.FullNameCorrected ().StripArity ().FilterPrimitive (),
+				ReturnNotNull = m.GetReturnTypeNullability () == Nullability.NotNull,
 				Visibility = m.Visibility ()
 			};
 
@@ -221,12 +224,12 @@ public static Method CreateMethod (GenBase declaringType, MethodDefinition m)
 			return method;
 		}
 
-		public static Parameter CreateParameter (ParameterDefinition p, string jnitype, string rawtype)
+		public static Parameter CreateParameter (ParameterDefinition p, string jnitype, string rawtype, bool isNotNull)
 		{
 			// FIXME: safe to use CLR type name? assuming yes as we often use it in metadatamap.
 			// FIXME: IsSender?
 			var isEnumType = GetGeneratedEnumAttribute (p.CustomAttributes) != null;
-			return new Parameter (TypeNameUtilities.MangleName (p.Name), jnitype ?? p.ParameterType.FullNameCorrected ().StripArity (), null, isEnumType, rawtype);
+			return new Parameter (TypeNameUtilities.MangleName (p.Name), jnitype ?? p.ParameterType.FullNameCorrected ().StripArity (), null, isEnumType, rawtype, isNotNull);
 		}
 
 		public static Parameter CreateParameter (string managedType, string javaType)