Merged PR 48946: Backport recent M.E.AI changes from main

Several git cherry-picks with no conflicts or manual changes.

----
#### AI description  (iteration 1)
#### PR Classification
Backport recent changes from the main branch to the current branch.

#### PR Summary
This pull request backports recent changes related to AI functionalities and tests from the main branch.
- `OpenAIChatClientTests.cs`: Added new tests for image content handling in chat messages.
- `ChatResponseExtensions.cs`: Enhanced content coalescing to handle multiple content types.
- `TextReasoningContent.cs` and `TextReasoningContentTests.cs`: Added new class and corresponding tests for text reasoning content.
- `AIFunctionFactory.cs`: Improved result marshalling logic for async methods.
- `DistributedCachingChatClient.cs`: Updated cache key computation to include messages and options.
<!-- GitOpsUserAgent=GitOps.Apps.Server.pullrequestcopilot -->

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -180,53 +180,60 @@ static async Task<ChatResponse> ToChatResponseAsync(
         }
     }
 
-    /// <summary>Coalesces sequential <see cref="TextContent"/> content elements.</summary>
+    /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
     internal static void CoalesceTextContent(List<AIContent> contents)
     {
-        StringBuilder? coalescedText = null;
+        Coalesce<TextContent>(contents, static text => new(text));
+        Coalesce<TextReasoningContent>(contents, static text => new(text));
 
-        // Iterate through all of the items in the list looking for contiguous items that can be coalesced.
-        int start = 0;
-        while (start < contents.Count - 1)
+        // This implementation relies on TContent's ToString returning its exact text.
+        static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent> fromText)
+            where TContent : AIContent
         {
-            // We need at least two TextContents in a row to be able to coalesce.
-            if (contents[start] is not TextContent firstText)
-            {
-                start++;
-                continue;
-            }
-
-            if (contents[start + 1] is not TextContent secondText)
-            {
-                start += 2;
-                continue;
-            }
+            StringBuilder? coalescedText = null;
 
-            // Append the text from those nodes and continue appending subsequent TextContents until we run out.
-            // We null out nodes as their text is appended so that we can later remove them all in one O(N) operation.
-            coalescedText ??= new();
-            _ = coalescedText.Clear().Append(firstText.Text).Append(secondText.Text);
-            contents[start + 1] = null!;
-            int i = start + 2;
-            for (; i < contents.Count && contents[i] is TextContent next; i++)
+            // Iterate through all of the items in the list looking for contiguous items that can be coalesced.
+            int start = 0;
+            while (start < contents.Count - 1)
             {
-                _ = coalescedText.Append(next.Text);
-                contents[i] = null!;
+                // We need at least two TextContents in a row to be able to coalesce.
+                if (contents[start] is not TContent firstText)
+                {
+                    start++;
+                    continue;
+                }
+
+                if (contents[start + 1] is not TContent secondText)
+                {
+                    start += 2;
+                    continue;
+                }
+
+                // Append the text from those nodes and continue appending subsequent TextContents until we run out.
+                // We null out nodes as their text is appended so that we can later remove them all in one O(N) operation.
+                coalescedText ??= new();
+                _ = coalescedText.Clear().Append(firstText).Append(secondText);
+                contents[start + 1] = null!;
+                int i = start + 2;
+                for (; i < contents.Count && contents[i] is TContent next; i++)
+                {
+                    _ = coalescedText.Append(next);
+                    contents[i] = null!;
+                }
+
+                // Store the replacement node. We inherit the properties of the first text node. We don't
+                // currently propagate additional properties from the subsequent nodes. If we ever need to,
+                // we can add that here.
+                var newContent = fromText(coalescedText.ToString());
+                contents[start] = newContent;
+                newContent.AdditionalProperties = firstText.AdditionalProperties?.Clone();
+
+                start = i;
             }
 
-            // Store the replacement node.
-            contents[start] = new TextContent(coalescedText.ToString())
-            {
-                // We inherit the properties of the first text node. We don't currently propagate additional
-                // properties from the subsequent nodes. If we ever need to, we can add that here.
-                AdditionalProperties = firstText.AdditionalProperties?.Clone(),
-            };
-
-            start = i;
+            // Remove all of the null slots left over from the coalescing process.
+            _ = contents.RemoveAll(u => u is null);
         }
-
-        // Remove all of the null slots left over from the coalescing process.
-        _ = contents.RemoveAll(u => u is null);
     }
 
     /// <summary>Finalizes the <paramref name="response"/> object.</summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -12,6 +12,7 @@ namespace Microsoft.Extensions.AI;
 [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: "functionCall")]
 [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: "functionResult")]
 [JsonDerivedType(typeof(TextContent), typeDiscriminator: "text")]
+[JsonDerivedType(typeof(TextReasoningContent), typeDiscriminator: "reasoning")]
 [JsonDerivedType(typeof(UriContent), typeDiscriminator: "uri")]
 [JsonDerivedType(typeof(UsageContent), typeDiscriminator: "usage")]
 public class AIContent

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ErrorContent.cs

@@ -2,8 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
-using System.Text.Json.Serialization;
-using Microsoft.Shared.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Microsoft.Extensions.AI;
 
@@ -16,33 +15,33 @@ namespace Microsoft.Extensions.AI;
 public class ErrorContent : AIContent
 {
     /// <summary>The error message.</summary>
-    private string _message;
+    private string? _message;
 
-    /// <summary>Initializes a new instance of the <see cref="ErrorContent"/> class with the specified message.</summary>
-    /// <param name="message">The message to store in this content.</param>
-    [JsonConstructor]
-    public ErrorContent(string message)
+    /// <summary>Initializes a new instance of the <see cref="ErrorContent"/> class with the specified error message.</summary>
+    /// <param name="message">The error message to store in this content.</param>
+    public ErrorContent(string? message)
     {
-        _message = Throw.IfNull(message);
+        _message = message;
     }
 
     /// <summary>Gets or sets the error message.</summary>
+    [AllowNull]
     public string Message
     {
-        get => _message;
-        set => _message = Throw.IfNull(value);
+        get => _message ?? string.Empty;
+        set => _message = value;
     }
 
-    /// <summary>Gets or sets the error code.</summary>
+    /// <summary>Gets or sets an error code associated with the error.</summary>
     public string? ErrorCode { get; set; }
 
-    /// <summary>Gets or sets the error details.</summary>
+    /// <summary>Gets or sets additional details about the error.</summary>
     public string? Details { get; set; }
 
     /// <summary>Gets a string representing this instance to display in the debugger.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
     private string DebuggerDisplay =>
-        $"Error = {Message}" +
-        (ErrorCode is not null ? $" ({ErrorCode})" : string.Empty) +
-        (Details is not null ? $" - {Details}" : string.Empty);
+        $"Error = \"{Message}\"" +
+        (!string.IsNullOrWhiteSpace(ErrorCode) ? $" ({ErrorCode})" : string.Empty) +
+        (!string.IsNullOrWhiteSpace(Details) ? $" - \"{Details}\"" : string.Empty);
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextReasoningContent.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents text reasoning content in a chat.
+/// </summary>
+/// <remarks>
+/// <see cref="TextReasoningContent"/> is distinct from <see cref="TextContent"/>. <see cref="TextReasoningContent"/>
+/// represents "thinking" or "reasoning" performed by the model and is distinct from the actual output text from
+/// the model, which is represented by <see cref="TextContent"/>. Neither types derives from the other.
+/// </remarks>
+[DebuggerDisplay("{DebuggerDisplay,nq}")]
+public sealed class TextReasoningContent : AIContent
+{
+    private string? _text;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TextReasoningContent"/> class.
+    /// </summary>
+    /// <param name="text">The text reasoning content.</param>
+    public TextReasoningContent(string? text)
+    {
+        _text = text;
+    }
+
+    /// <summary>
+    /// Gets or sets the text reasoning content.
+    /// </summary>
+    [AllowNull]
+    public string Text
+    {
+        get => _text ?? string.Empty;
+        set => _text = value;
+    }
+
+    /// <inheritdoc/>
+    public override string ToString() => Text;
+
+    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
+    private string DebuggerDisplay => $"Reasoning = \"{Text}\"";
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs

@@ -42,30 +42,18 @@ public static partial class AIJsonUtilities
     [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", Justification = "DefaultJsonTypeInfoResolver is only used when reflection-based serialization is enabled")]
     private static JsonSerializerOptions CreateDefaultOptions()
     {
-        // If reflection-based serialization is enabled by default, use it, as it's the most permissive in terms of what it can serialize,
-        // and we want to be flexible in terms of what can be put into the various collections in the object model.
-        // Otherwise, use the source-generated options to enable trimming and Native AOT.
-        JsonSerializerOptions options;
+        // Copy configuration from the source generated context.
+        JsonSerializerOptions options = new(JsonContext.Default.Options)
+        {
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        };
 
         if (JsonSerializer.IsReflectionEnabledByDefault)
         {
-            // Keep in sync with the JsonSourceGenerationOptions attribute on JsonContext below.
-            options = new(JsonSerializerDefaults.Web)
-            {
-                TypeInfoResolver = new DefaultJsonTypeInfoResolver(),
-                Converters = { new JsonStringEnumConverter() },
-                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-                Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-                WriteIndented = true,
-            };
-        }
-        else
-        {
-            options = new(JsonContext.Default.Options)
-            {
-                // Compile-time encoder setting not yet available
-                Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-            };
+            // If reflection-based serialization is enabled by default, use it as a fallback for all other types.
+            // Also turn on string-based enum serialization for all unknown enums.
+            options.TypeInfoResolverChain.Add(new DefaultJsonTypeInfoResolver());
+            options.Converters.Add(new JsonStringEnumConverter());
         }
 
         options.MakeReadOnly();
@@ -83,6 +71,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(SpeechToTextResponseUpdate))]
     [JsonSerializable(typeof(IReadOnlyList<SpeechToTextResponseUpdate>))]
     [JsonSerializable(typeof(IList<ChatMessage>))]
+    [JsonSerializable(typeof(IEnumerable<ChatMessage>))]
+    [JsonSerializable(typeof(ChatMessage[]))]
     [JsonSerializable(typeof(ChatOptions))]
     [JsonSerializable(typeof(EmbeddingGenerationOptions))]
     [JsonSerializable(typeof(ChatClientMetadata))]
@@ -95,14 +85,24 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(JsonDocument))]
     [JsonSerializable(typeof(JsonElement))]
     [JsonSerializable(typeof(JsonNode))]
+    [JsonSerializable(typeof(JsonObject))]
+    [JsonSerializable(typeof(JsonValue))]
+    [JsonSerializable(typeof(JsonArray))]
     [JsonSerializable(typeof(IEnumerable<string>))]
+    [JsonSerializable(typeof(char))]
     [JsonSerializable(typeof(string))]
     [JsonSerializable(typeof(int))]
+    [JsonSerializable(typeof(short))]
     [JsonSerializable(typeof(long))]
+    [JsonSerializable(typeof(uint))]
+    [JsonSerializable(typeof(ushort))]
+    [JsonSerializable(typeof(ulong))]
     [JsonSerializable(typeof(float))]
     [JsonSerializable(typeof(double))]
+    [JsonSerializable(typeof(decimal))]
     [JsonSerializable(typeof(bool))]
     [JsonSerializable(typeof(TimeSpan))]
+    [JsonSerializable(typeof(DateTime))]
     [JsonSerializable(typeof(DateTimeOffset))]
     [JsonSerializable(typeof(Embedding))]
     [JsonSerializable(typeof(Embedding<byte>))]

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.cs

@@ -90,7 +90,7 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
 
         // For cases where the hash may be used as a cache key, we rely on collision resistance for security purposes.
         // If a collision occurs, we'd serve a cached LLM response for a potentially unrelated prompt, leading to information
-        // disclosure. Use of SHA256 is an implementation detail and can be easily swapped in the future if needed, albeit
+        // disclosure. Use of SHA384 is an implementation detail and can be easily swapped in the future if needed, albeit
         // invalidating any existing cache entries.
 #if NET
         IncrementalHashStream? stream = IncrementalHashStream.ThreadStaticInstance;
@@ -107,7 +107,7 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
             stream = new();
         }
 
-        Span<byte> hashData = stackalloc byte[SHA256.HashSizeInBytes];
+        Span<byte> hashData = stackalloc byte[SHA384.HashSizeInBytes];
         try
         {
             foreach (object? value in values)
@@ -133,8 +133,8 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
             JsonSerializer.Serialize(stream, value, jti);
         }
 
-        using var sha256 = SHA256.Create();
-        var hashData = sha256.ComputeHash(stream.GetBuffer(), 0, (int)stream.Length);
+        using var hashAlgorithm = SHA384.Create();
+        var hashData = hashAlgorithm.ComputeHash(stream.GetBuffer(), 0, (int)stream.Length);
 
         return ConvertToHexString(hashData);
 
@@ -185,7 +185,7 @@ private sealed class IncrementalHashStream : Stream
         public static IncrementalHashStream? ThreadStaticInstance;
 
         /// <summary>The <see cref="IncrementalHash"/> used by this instance.</summary>
-        private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
+        private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA384);
 
         /// <summary>Gets the current hash and resets.</summary>
         public void GetHashAndReset(Span<byte> bytes) => _hash.GetHashAndReset(bytes);

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs

@@ -125,6 +125,6 @@ protected override async Task WriteCacheStreamingAsync(
         }
     }
 
-    protected override string GetCacheKey(params ReadOnlySpan<object?> values)
-        => base.GetCacheKey([.. values, .. _cachingKeys]);
+    protected override string GetCacheKey(IEnumerable<ChatMessage> messages, ChatOptions? options, params ReadOnlySpan<object?> additionalValues)
+        => base.GetCacheKey(messages, options, [.. additionalValues, .. _cachingKeys]);
 }

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs

@@ -190,11 +190,11 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
                     break;
 
                 case UriContent uriContent when uriContent.HasTopLevelMediaType("image"):
-                    parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri));
+                    parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)));
                     break;
 
                 case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
-                    parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
+                    parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)));
                     break;
 
                 case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"):
@@ -220,6 +220,21 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
         return parts;
     }
 
+    private static ChatImageDetailLevel? GetImageDetail(AIContent content)
+    {
+        if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true)
+        {
+            return value switch
+            {
+                string detailString => new ChatImageDetailLevel(detailString),
+                ChatImageDetailLevel detail => detail,
+                _ => null
+            };
+        }
+
+        return null;
+    }
+
     private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingChatCompletionAsync(
         IAsyncEnumerable<StreamingChatCompletionUpdate> updates,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)