[manual] Merge branch 'release/9.0' => 'release/9.0-staging'

NuGet.config

@@ -10,6 +10,7 @@
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
     <!--  Begin: Package sources from dotnet-emsdk -->
     <add key="darc-pub-dotnet-emsdk-2c27e40" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-emsdk-2c27e405/nuget/v3/index.json" />
+    <add key="darc-pub-dotnet-emsdk-4c9d1b1" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-emsdk-4c9d1b11/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-emsdk -->
     <!--  Begin: Package sources from dotnet-sdk -->
     <add key="darc-pub-dotnet-sdk-a345a00" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-sdk-a345a003/nuget/v3/index.json" />

eng/Versions.props

@@ -1,13 +1,13 @@
 <Project>
   <PropertyGroup>
     <!-- The .NET product branding version -->
-    <ProductVersion>9.0.1</ProductVersion>
+    <ProductVersion>9.0.2</ProductVersion>
     <!-- File version numbers -->
     <MajorVersion>9</MajorVersion>
     <MinorVersion>0</MinorVersion>
-    <PatchVersion>1</PatchVersion>
+    <PatchVersion>2</PatchVersion>
     <SdkBandVersion>9.0.100</SdkBandVersion>
-    <PackageVersionNet8>8.0.11</PackageVersionNet8>
+    <PackageVersionNet8>8.0.$([MSBuild]::Add($(PatchVersion),11))</PackageVersionNet8>
     <PackageVersionNet7>7.0.20</PackageVersionNet7>
     <PackageVersionNet6>6.0.36</PackageVersionNet6>
     <PreReleaseVersionLabel>servicing</PreReleaseVersionLabel>
@@ -164,7 +164,7 @@
     <optimizationPGOCoreCLRVersion>1.0.0-prerelease.24462.2</optimizationPGOCoreCLRVersion>
     <!-- Not auto-updated. -->
     <MicrosoftDiaSymReaderVersion>2.0.0</MicrosoftDiaSymReaderVersion>
-    <MicrosoftDiaSymReaderNativeVersion>17.10.0-beta1.24272.1</MicrosoftDiaSymReaderNativeVersion>
+    <MicrosoftDiaSymReaderNativeVersion>17.12.0-beta1.24603.5</MicrosoftDiaSymReaderNativeVersion>
     <SystemCommandLineVersion>2.0.0-beta4.24324.3</SystemCommandLineVersion>
     <TraceEventVersion>3.1.7</TraceEventVersion>
     <NETStandardLibraryRefVersion>2.1.0</NETStandardLibraryRefVersion>

src/libraries/System.Formats.Nrbf/ref/System.Formats.Nrbf.cs

@@ -9,7 +9,6 @@ namespace System.Formats.Nrbf
     public abstract partial class ArrayRecord : System.Formats.Nrbf.SerializationRecord
     {
         internal ArrayRecord() { }
-        public virtual long FlattenedLength { get { throw null; } }
         public override System.Formats.Nrbf.SerializationRecordId Id { get { throw null; } }
         public abstract System.ReadOnlySpan<int> Lengths { get; }
         public int Rank { get { throw null; } }

src/libraries/System.Formats.Nrbf/src/PACKAGE.md

@@ -54,7 +54,7 @@ There are more than a dozen different serialization [record types](https://learn
   - `PrimitiveTypeRecord<T>` derives from the non-generic [PrimitiveTypeRecord](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.primitivetyperecord), which also exposes a [Value](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.primitivetyperecord.value) property. But on the base class, the value is returned as `object` (which introduces boxing for value types).
 - [ClassRecord](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.classrecord): describes all `class` and `struct` besides the aforementioned  primitive types.
 - [ArrayRecord](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.arrayrecord): describes all array records, including jagged and multi-dimensional arrays.
-- [`SZArrayRecord<T>`](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.szarrayrecord-1): describes single-dimensional, zero-indexed array records, where `T` can be either a primitive type or a `ClassRecord`.
+- [`SZArrayRecord<T>`](https://learn.microsoft.com/dotnet/api/system.formats.nrbf.szarrayrecord-1): describes single-dimensional, zero-indexed array records, where `T` can be either a primitive type or a `SerializationRecord`.
 
 ```csharp
 SerializationRecord rootObject = NrbfDecoder.Decode(payload); // payload is a Stream

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/AllowedRecordType.cs

@@ -28,12 +28,13 @@ internal enum AllowedRecordTypes : uint
     ArraySingleString = 1 << SerializationRecordType.ArraySingleString,
 
     Nulls = ObjectNull | ObjectNullMultiple256 | ObjectNullMultiple,
+    Arrays = ArraySingleObject | ArraySinglePrimitive | ArraySingleString | BinaryArray,
 
     /// <summary>
     /// Any .NET object (a primitive, a reference type, a reference or single null).
     /// </summary>
     AnyObject = MemberPrimitiveTyped
-        | ArraySingleObject | ArraySinglePrimitive | ArraySingleString | BinaryArray
+        | Arrays
         | ClassWithId | ClassWithMembersAndTypes | SystemClassWithMembersAndTypes
         | BinaryObjectString
         | MemberReference

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/ArrayRecord.cs

@@ -4,6 +4,9 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Reflection.Metadata;
 using System.Formats.Nrbf.Utils;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Runtime.Serialization;
 
 namespace System.Formats.Nrbf;
 
@@ -27,12 +30,6 @@ private protected ArrayRecord(ArrayInfo arrayInfo)
     /// <value>A buffer of integers that represent the number of elements in every dimension.</value>
     public abstract ReadOnlySpan<int> Lengths { get; }
 
-    /// <summary>
-    /// When overridden in a derived class, gets the total number of all elements in every dimension.
-    /// </summary>
-    /// <value>A number that represent the total number of all elements in every dimension.</value>
-    public virtual long FlattenedLength => ArrayInfo.FlattenedLength;
-
     /// <summary>
     /// Gets the rank of the array.
     /// </summary>
@@ -118,4 +115,86 @@ private void HandleNext(object value, NextInfo info, int size)
     }
 
     internal abstract (AllowedRecordTypes allowed, PrimitiveType primitiveType) GetAllowedRecordType();
+
+    internal static void Populate(List<SerializationRecord> source, Array destination, int[] lengths, AllowedRecordTypes allowedRecordTypes, bool allowNulls)
+    {
+        int[] indices = new int[lengths.Length];
+        nuint numElementsWritten = 0; // only for debugging; not used in release builds
+
+        foreach (SerializationRecord record in source)
+        {
+            object? value = GetActualValue(record, allowedRecordTypes, out int incrementCount);
+            if (value is not null)
+            {
+                // null is a default element for all array of reference types, so we don't call SetValue for nulls.
+                destination.SetValue(value, indices);
+                Debug.Assert(incrementCount == 1, "IncrementCount other than 1 is allowed only for null records.");
+            }
+            else if (!allowNulls)
+            {
+                ThrowHelper.ThrowArrayContainedNulls();
+            }
+
+            while (incrementCount > 0)
+            {
+                incrementCount--;
+                numElementsWritten++;
+                int dimension = indices.Length - 1;
+                while (dimension >= 0)
+                {
+                    indices[dimension]++;
+                    if (indices[dimension] < lengths[dimension])
+                    {
+                        break;
+                    }
+                    indices[dimension] = 0;
+                    dimension--;
+                }
+
+                if (dimension < 0)
+                {
+                    break;
+                }
+            }
+        }
+
+        Debug.Assert(numElementsWritten == (uint)source.Count, "We should have traversed the entirety of the source records collection.");
+        Debug.Assert(numElementsWritten == (ulong)destination.LongLength, "We should have traversed the entirety of the destination array.");
+    }
+
+    private static object? GetActualValue(SerializationRecord record, AllowedRecordTypes allowedRecordTypes, out int repeatCount)
+    {
+        repeatCount = 1;
+
+        if (record is NullsRecord nullsRecord)
+        {
+            repeatCount = nullsRecord.NullCount;
+            return null;
+        }
+        else if (record.RecordType == SerializationRecordType.MemberReference)
+        {
+            record = ((MemberReferenceRecord)record).GetReferencedRecord();
+        }
+
+        if (allowedRecordTypes == AllowedRecordTypes.BinaryObjectString)
+        {
+            if (record is not BinaryObjectStringRecord stringRecord)
+            {
+                throw new SerializationException(SR.Serialization_InvalidReference);
+            }
+
+            return stringRecord.Value;
+        }
+        else if (allowedRecordTypes == AllowedRecordTypes.Arrays)
+        {
+            if (record is not ArrayRecord arrayRecord)
+            {
+                throw new SerializationException(SR.Serialization_InvalidReference);
+            }
+
+            return arrayRecord;
+        }
+
+        return record;
+    }
 }

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/ArrayRectangularPrimitiveRecord.cs

@@ -0,0 +1,83 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Formats.Nrbf.Utils;
+using System.Linq;
+using System.Reflection.Metadata;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Formats.Nrbf
+{
+    internal sealed class ArrayRectangularPrimitiveRecord<T> : ArrayRecord where T : unmanaged
+    {
+        private readonly int[] _lengths;
+        private readonly IReadOnlyList<T> _values;
+        private TypeName? _typeName;
+
+        internal ArrayRectangularPrimitiveRecord(ArrayInfo arrayInfo, int[] lengths, IReadOnlyList<T> values) : base(arrayInfo)
+        {
+            _lengths = lengths;
+            _values = values;
+            ValuesToRead = 0; // there is nothing to read anymore
+        }
+
+        public override ReadOnlySpan<int> Lengths => _lengths;
+
+        public override SerializationRecordType RecordType => SerializationRecordType.BinaryArray;
+
+        public override TypeName TypeName
+            => _typeName ??= TypeNameHelpers.GetPrimitiveTypeName(TypeNameHelpers.GetPrimitiveType<T>()).MakeArrayTypeName(Rank);
+
+        internal override (AllowedRecordTypes allowed, PrimitiveType primitiveType) GetAllowedRecordType() => throw new InvalidOperationException();
+
+        private protected override void AddValue(object value) => throw new InvalidOperationException();
+
+        [RequiresDynamicCode("May call Array.CreateInstance().")]
+        private protected override Array Deserialize(Type arrayType, bool allowNulls)
+        {
+            Array result =
+#if NET9_0_OR_GREATER
+                Array.CreateInstanceFromArrayType(arrayType, _lengths);
+#else
+                Array.CreateInstance(typeof(T), _lengths);
+#endif
+            int[] indices = new int[_lengths.Length];
+            nuint numElementsWritten = 0; // only for debugging; not used in release builds
+
+            for (int i = 0; i < _values.Count; i++)
+            {
+                result.SetValue(_values[i], indices);
+                numElementsWritten++;
+
+                int dimension = indices.Length - 1;
+                while (dimension >= 0)
+                {
+                    indices[dimension]++;
+                    if (indices[dimension] < Lengths[dimension])
+                    {
+                        break;
+                    }
+                    indices[dimension] = 0;
+                    dimension--;
+                }
+
+                if (dimension < 0)
+                {
+                    break;
+                }
+            }
+
+            Debug.Assert(numElementsWritten == (uint)_values.Count, "We should have traversed the entirety of the source values collection.");
+            Debug.Assert(numElementsWritten == (ulong)result.LongLength, "We should have traversed the entirety of the destination array.");
+
+            return result;
+        }
+    }
+}

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/ArraySingleObjectRecord.cs

@@ -15,9 +15,9 @@ namespace System.Formats.Nrbf;
 /// <remarks>
 /// ArraySingleObject records are described in <see href="https://learn.microsoft.com/openspecs/windows_protocols/ms-nrbf/982b2f50-6367-402a-aaf2-44ee96e2a5e0">[MS-NRBF] 2.4.3.2</see>.
 /// </remarks>
-internal sealed class ArraySingleObjectRecord : SZArrayRecord<object?>
+internal sealed class ArraySingleObjectRecord : SZArrayRecord<SerializationRecord>
 {
-    private ArraySingleObjectRecord(ArrayInfo arrayInfo) : base(arrayInfo) => Records = [];
+    internal ArraySingleObjectRecord(ArrayInfo arrayInfo) : base(arrayInfo) => Records = [];
 
     public override SerializationRecordType RecordType => SerializationRecordType.ArraySingleObject;
 
@@ -27,25 +27,26 @@ public override TypeName TypeName
     private List<SerializationRecord> Records { get; }
 
     /// <inheritdoc/>
-    public override object?[] GetArray(bool allowNulls = true)
-        => (object?[])(allowNulls ? _arrayNullsAllowed ??= ToArray(true) : _arrayNullsNotAllowed ??= ToArray(false));
+    public override SerializationRecord?[] GetArray(bool allowNulls = true)
+        => (SerializationRecord?[])(allowNulls ? _arrayNullsAllowed ??= ToArray(true) : _arrayNullsNotAllowed ??= ToArray(false));
 
-    private object?[] ToArray(bool allowNulls)
+    private SerializationRecord?[] ToArray(bool allowNulls)
     {
-        object?[] values = new object?[Length];
+        SerializationRecord?[] values = new SerializationRecord?[Length];
 
         int valueIndex = 0;
         for (int recordIndex = 0; recordIndex < Records.Count; recordIndex++)
         {
             SerializationRecord record = Records[recordIndex];
 
-            int nullCount = record is NullsRecord nullsRecord ? nullsRecord.NullCount : 0;
-            if (nullCount == 0)
+            if (record is MemberReferenceRecord referenceRecord)
             {
-                // "new object[] { <SELF> }" is special cased because it allows for storing reference to itself.
-                values[valueIndex++] = record is MemberReferenceRecord referenceRecord && referenceRecord.Reference.Equals(Id)
-                    ? values // a reference to self, and a way to get StackOverflow exception ;)
-                    : record.GetValue();
+                record = referenceRecord.GetReferencedRecord();
+            }
+
+            if (record is not NullsRecord nullsRecord)
+            {
+                values[valueIndex++] = record;
                 continue;
             }
 
@@ -54,6 +55,7 @@ public override TypeName TypeName
                 ThrowHelper.ThrowArrayContainedNulls();
             }
 
+            int nullCount = nullsRecord.NullCount;
             do
             {
                 values[valueIndex++] = null;