[release/6.0-rc2] XmlSerializer support for IsDynamicCodeSupported=false (#59507)

* XmlSerializer support for IsDynamicCodeSupported=false

Add more checks to XmlSerializer to check the SerializationMode. Don't try to use Reflection.Emit if the SerializationMode is ReflectionOnly.

These changes were ported from
* dotnet/runtimelab#593
* dotnet/runtimelab#600

Fix #59167

* Fix a bug in XmlSerializer.CanDeserialize when in ReflectionOnly mode.

* Port UAP code for CanDeserialize

* PR feedback

* Add a linker test to ensure linker option '--enable-opt sealer' works when IsDynamicCodeSupported==false.

Co-authored-by: Eric Erhardt <eric.erhardt@microsoft.com>

Author: github-actions[bot]

src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializer.cs

@@ -194,7 +194,10 @@ public XmlSerializer(XmlTypeMapping xmlTypeMapping)
             if (xmlTypeMapping == null)
                 throw new ArgumentNullException(nameof(xmlTypeMapping));
 
-            _tempAssembly = GenerateTempAssembly(xmlTypeMapping);
+            if (Mode != SerializationMode.ReflectionOnly)
+            {
+                _tempAssembly = GenerateTempAssembly(xmlTypeMapping);
+            }
             _mapping = xmlTypeMapping;
         }
 
@@ -218,6 +221,12 @@ public XmlSerializer(Type type, string? defaultNamespace)
                 _primitiveType = type;
                 return;
             }
+
+            if (Mode == SerializationMode.ReflectionOnly)
+            {
+                return;
+            }
+
             _tempAssembly = s_cache[defaultNamespace, type];
             if (_tempAssembly == null)
             {
@@ -270,7 +279,10 @@ public XmlSerializer(Type type, XmlAttributeOverrides? overrides, Type[]? extraT
             DefaultNamespace = defaultNamespace;
             _rootType = type;
             _mapping = GenerateXmlTypeMapping(type, overrides, extraTypes, root, defaultNamespace);
-            _tempAssembly = GenerateTempAssembly(_mapping, type, defaultNamespace, location);
+            if (Mode != SerializationMode.ReflectionOnly)
+            {
+                _tempAssembly = GenerateTempAssembly(_mapping, type, defaultNamespace, location);
+            }
         }
 
         [RequiresUnreferencedCode("calls ImportTypeMapping")]
@@ -530,6 +542,14 @@ public virtual bool CanDeserialize(XmlReader xmlReader)
                 TypeDesc typeDesc = (TypeDesc)TypeScope.PrimtiveTypes[_primitiveType]!;
                 return xmlReader.IsStartElement(typeDesc.DataType!.Name!, string.Empty);
             }
+            else if (ShouldUseReflectionBasedSerialization(_mapping) || _isReflectionBasedSerializer)
+            {
+                // If we should use reflection, we will try to do reflection-based deserialization, without fallback.
+                // Don't check xmlReader.IsStartElement to avoid having to duplicate SOAP deserialization logic here.
+                // It is better to return an incorrect 'true', which will throw during Deserialize than to return an
+                // incorrect 'false', and the caller won't even try to Deserialize when it would succeed.
+                return true;
+            }
             else if (_tempAssembly != null)
             {
                 return _tempAssembly.CanRead(_mapping, xmlReader);

src/libraries/System.Private.Xml/tests/TrimmingTests/System.Private.Xml.TrimmingTests.proj

@@ -4,6 +4,8 @@
   <ItemGroup>
     <TestConsoleAppSourceFiles Include="XmlSchema.Write.cs" />
     <TestConsoleAppSourceFiles Include="XmlSerializer.Deserialize.cs" />
+    <TestConsoleAppSourceFiles Include="XmlSerializer.Deserialize.SealerOpt.cs"
+                               ExtraTrimmerArgs="--enable-opt sealer" />
     <TestConsoleAppSourceFiles Include="XmlSerializer.Serialize.cs" />
     <TestConsoleAppSourceFiles Include="XslCompiledTransformTests.cs" />
   </ItemGroup>

src/libraries/System.Private.Xml/tests/TrimmingTests/XmlSerializer.Deserialize.SealerOpt.cs

@@ -0,0 +1,48 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Reflection;
+
+namespace System.Xml.Serialization.TrimmingTests
+{
+    /// <summary>
+    /// Tests that using XmlSerializer with linker option '--enable-opt sealer' works
+    /// when IsDynamicCodeSupported==false.
+    /// </summary>
+    internal class Program
+    {
+        // Preserve these types until XmlSerializer is fully trim-safe.
+        // see https://github.com/dotnet/runtime/issues/44768
+        [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Response))]
+        public static int Main()
+        {
+            // simulate IsDynamicCodeSupported==false by setting the SerializationMode to ReflectionOnly
+            const int ReflectionOnly = 1;
+            typeof(XmlSerializer).GetField("s_mode", BindingFlags.NonPublic | BindingFlags.Static)
+                .SetValue(null, ReflectionOnly);
+
+            using StringReader stringReader = new StringReader(@"<?xml version=""1.0"" encoding=""UTF-8""?>
+				<Response DataType=""Data"">
+				</Response>");
+
+            Response obj = (Response)new XmlSerializer(typeof(Response)).Deserialize(stringReader);
+            if (obj.DataType == "Data")
+            {
+                return 100;
+            }
+
+            return -1;
+        }
+    }
+
+    [Serializable]
+    [XmlType(AnonymousType = true)]
+    [XmlRoot(Namespace = "", IsNullable = false)]
+    public class Response
+    {
+        [XmlAttribute]
+        public string DataType { get; set; }
+    }
+}