Fix type parameter mapping for static closures (#118607)

Roslyn can generate code (specifically for nested async lambda state
machines) where a static closure environment is referenced from a
nested type of the display class. Handle this case by detecting nested
classes, and never tracking type parameter instantiations from nested
classes to outer classes. This fixes a reported stack overflow in
ILLink and ILC.

Author: sbomer

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Dataflow/CompilerGeneratedState.cs

@@ -175,8 +175,8 @@ void ProcessMethod(MethodDesc method)
                                         // Find calls to state machine constructors that occur outside the type
                                         if (referencedMethod.IsConstructor &&
                                             referencedMethod.OwningType is MetadataType generatedType &&
-                                            // Don't consider calls in the same type, like inside a static constructor
-                                            method.OwningType != generatedType &&
+                                            // Don't consider calls in the same/nested type, like inside a static constructor
+                                            !IsSameOrNestedType(method.OwningType, generatedType) &&
                                             CompilerGeneratedNames.IsLambdaDisplayClass(generatedType.Name))
                                         {
                                             Debug.Assert(generatedType.IsTypeDefinition);
@@ -216,8 +216,8 @@ referencedMethod.OwningType is MetadataType generatedType &&
                                         field = field.GetTypicalFieldDefinition();
 
                                         if (field.OwningType is MetadataType generatedType &&
-                                            // Don't consider field accesses in the same type, like inside a static constructor
-                                            method.OwningType != generatedType &&
+                                            // Don't consider field accesses in the same/nested type, like inside a static constructor
+                                            !IsSameOrNestedType(method.OwningType, generatedType) &&
                                             CompilerGeneratedNames.IsLambdaDisplayClass(generatedType.Name))
                                         {
                                             Debug.Assert(generatedType.IsTypeDefinition);
@@ -259,6 +259,22 @@ referencedMethod.OwningType is MetadataType generatedType &&
                         // Fill in null for argument providers now, the real providers will be filled in later
                         generatedTypeToTypeArgs[stateMachineType] = new TypeArgumentInfo(method, null);
                     }
+
+                    static bool IsSameOrNestedType(TypeDesc type, TypeDesc potentialOuterType)
+                    {
+                        do
+                        {
+                            if (type == potentialOuterType)
+                                return true;
+
+                            if (type is not EcmaType ecmaType)
+                                return false;
+
+                            type = ecmaType.ContainingType;
+                        } while (type != null);
+
+                        return false;
+                    }
                 }
 
                 // Look for state machine methods, and methods which call local functions.

src/tools/illink/src/linker/Linker.Dataflow/CompilerGeneratedState.cs

@@ -180,8 +180,8 @@ void ProcessMethod(MethodDefinition method)
                                 // Find calls to state machine constructors that occur outside the type
                                 if (referencedMethod.IsConstructor &&
                                     referencedMethod.DeclaringType is var generatedType &&
-                                    // Don't consider calls in the same type, like inside a static constructor
-                                    method.DeclaringType != generatedType &&
+                                    // Don't consider calls in the same/nested type, like inside a static constructor
+                                    !IsSameOrNestedType(method.DeclaringType, generatedType) &&
                                     CompilerGeneratedNames.IsLambdaDisplayClass(generatedType.Name))
                                 {
                                     // fill in null for now, attribute providers will be filled in later
@@ -219,8 +219,8 @@ referencedMethod.DeclaringType is var generatedType &&
                                     continue;
 
                                 if (field.DeclaringType is var generatedType &&
-                                    // Don't consider field accesses in the same type, like inside a static constructor
-                                    method.DeclaringType != generatedType &&
+                                    // Don't consider field accesses in the same/nested type, like inside a static constructor
+                                    !IsSameOrNestedType(method.DeclaringType, generatedType) &&
                                     CompilerGeneratedNames.IsLambdaDisplayClass(generatedType.Name))
                                 {
                                     if (!generatedTypeToTypeArgs.TryAdd(generatedType, new TypeArgumentInfo(method, null)))
@@ -253,6 +253,19 @@ referencedMethod.DeclaringType is var generatedType &&
                     // Fill in null for argument providers now, the real providers will be filled in later
                     generatedTypeToTypeArgs[stateMachineType] = new TypeArgumentInfo(method, null);
                 }
+
+                static bool IsSameOrNestedType(TypeDefinition type, TypeDefinition potentialOuterType)
+                {
+                    do
+                    {
+                        if (type == potentialOuterType)
+                            return true;
+
+                        type = type.DeclaringType;
+                    } while (type != null);
+
+                    return false;
+                }
             }
 
             // Look for state machine methods, and methods which call local functions.

src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/CompilerGeneratedTypes.cs

@@ -48,6 +48,8 @@ public static void Main()
             CapturingLocalFunctionInsideIterator<int>();
             LambdaInsideAsync<int>();
             LocalFunctionInsideAsync<int>();
+            NestedAsyncLambda.Test<int>();
+            NestedAsyncLocalFunction.Test<int>();
             NestedStaticLambda.Test<int>();
         }
 
@@ -423,6 +425,50 @@ void LocalFunction()
             LocalFunction();
         }
 
+        class NestedAsyncLambda
+        {
+            public static async void Test<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] T>()
+            {
+                var outer = async () =>
+                {
+                    var inner =
+                    [ExpectedWarning("IL2090", "T", nameof(DynamicallyAccessedMemberTypes.PublicProperties))]
+                    () =>
+                    {
+                        _ = typeof(T).GetMethods();
+                        _ = typeof(T).GetProperties();
+                    };
+
+                    inner();
+                };
+
+                outer().Wait();
+            }
+        }
+
+        class NestedAsyncLocalFunction
+        {
+            public static void Test<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] T>()
+            {
+                Local1();
+
+#if INCLUDE_UNEXPECTED_LOWERING_WARNINGS
+                [UnexpectedWarning("IL2091", "T", nameof(DynamicallyAccessedMemberTypes.PublicMethods), Tool.Trimmer | Tool.NativeAot, "https://github.com/dotnet/roslyn/issues/79333", CompilerGeneratedCode = true)]
+#endif
+                static async Task Local1()
+                {
+                    Local2();
+
+                    [ExpectedWarning("IL2090", "T", nameof(DynamicallyAccessedMemberTypes.PublicProperties))]
+                    static void Local2()
+                    {
+                        _ = typeof(T).GetMethods();
+                        _ = typeof(T).GetProperties();
+                    };
+                };
+            }
+        }
+
         class NestedStaticLambda
         {
             public static class Container<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] T>