[release/9.10] Prepare 9.10.2 Release

eng/Versions.props

@@ -2,7 +2,7 @@
   <PropertyGroup Label="Version settings">
     <MajorVersion>9</MajorVersion>
     <MinorVersion>10</MinorVersion>
-    <PatchVersion>1</PatchVersion>
+    <PatchVersion>2</PatchVersion>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
     <PreReleaseVersionIteration>1</PreReleaseVersionIteration>
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>

eng/packages/General.props

@@ -17,7 +17,7 @@
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="OllamaSharp" Version="5.1.9" />
-    <PackageVersion Include="OpenAI" Version="2.5.0" />
+    <PackageVersion Include="OpenAI" Version="2.6.0" />
     <PackageVersion Include="Polly" Version="8.4.2" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />
     <PackageVersion Include="Polly.Extensions" Version="8.4.2" />

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -2,8 +2,18 @@
 
 ## NOT YET RELEASED
 
+- Updated `AIFunctionFactory` to respect `[DisplayName(...)]` on functions as a way to override the function name.
+- Updated `AIFunctionFactory` to respect `[DefaultValue(...)]` on function parameters as a way to specify default values.
+- Added `CodeInterpreterToolCallContent`/`CodeInterpreterToolResultContent` for representing code interpreter tool calls and results.
+- Added `Name`, `MediaType`, and `HasTopLevelMediaType` to `HostedFileContent`.
+- Fixed the serialization/deserialization of variables typed as `UserInputRequestContent`/`UserInputResponseContent`.
+
+## 9.10.1
+
 - Updated `HostedMcpServerTool` to allow for non-`Uri` server addresses, in order to enable built-in names.
 - Updated `HostedMcpServerTool` to replace the header collection with an `AuthorizationToken` property.
+- Fixed `ToChatResponse{Async}` to not discard `TextReasoningContent.ProtectedData` when coalescing messages.
+- Fixed `AIFunctionFactory.Create` to special-case return types of `AIContent` and `IEnumerable<AIContent>` to not automatically JSON serialize them.
 
 ## 9.10.0
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -5,7 +5,11 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
+#if !NET
+using System.Runtime.InteropServices;
+#endif
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -181,7 +185,7 @@ static async Task<ChatResponse> ToChatResponseAsync(
     }
 
     /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
-    internal static void CoalesceTextContent(IList<AIContent> contents)
+    internal static void CoalesceContent(IList<AIContent> contents)
     {
         Coalesce<TextContent>(
             contents,
@@ -215,6 +219,110 @@ internal static void CoalesceTextContent(IList<AIContent> contents)
                 return content;
             });
 
+        Coalesce<DataContent>(
+            contents,
+            mergeSingle: false,
+            canMerge: static (r1, r2) => r1.MediaType == r2.MediaType && r1.HasTopLevelMediaType("text") && r1.Name == r2.Name,
+            static (contents, start, end) =>
+            {
+                Debug.Assert(end - start > 1, "Expected multiple contents to merge");
+
+                MemoryStream ms = new();
+                for (int i = start; i < end; i++)
+                {
+                    var current = (DataContent)contents[i];
+#if NET
+                    ms.Write(current.Data.Span);
+#else
+                    if (!MemoryMarshal.TryGetArray(current.Data, out var segment))
+                    {
+                        segment = new(current.Data.ToArray());
+                    }
+
+                    ms.Write(segment.Array!, segment.Offset, segment.Count);
+#endif
+                }
+
+                var first = (DataContent)contents[start];
+                return new DataContent(new ReadOnlyMemory<byte>(ms.GetBuffer(), 0, (int)ms.Length), first.MediaType) { Name = first.Name };
+            });
+
+        Coalesce<CodeInterpreterToolCallContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolCallContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Inputs is not null)
+                    {
+                        CoalesceContent(firstContent.Inputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? inputs = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (inputs ??= []).AddRange(((CodeInterpreterToolCallContent)contents[i]).Inputs ?? []);
+                }
+
+                if (inputs is not null)
+                {
+                    CoalesceContent(inputs);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Inputs = inputs,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
+        Coalesce<CodeInterpreterToolResultContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolResultContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Outputs is not null)
+                    {
+                        CoalesceContent(firstContent.Outputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? output = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Outputs ?? []);
+                }
+
+                if (output is not null)
+                {
+                    CoalesceContent(output);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Outputs = output,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
         static string MergeText(IList<AIContent> contents, int start, int end)
         {
             Debug.Assert(end - start > 1, "Expected multiple contents to merge");
@@ -318,7 +426,7 @@ private static void FinalizeResponse(ChatResponse response)
         int count = response.Messages.Count;
         for (int i = 0; i < count; i++)
         {
-            CoalesceTextContent((List<AIContent>)response.Messages[i].Contents);
+            CoalesceContent((List<AIContent>)response.Messages[i].Contents);
         }
     }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -30,6 +30,8 @@ namespace Microsoft.Extensions.AI;
 // [JsonDerivedType(typeof(McpServerToolResultContent), typeDiscriminator: "mcpServerToolResult")]
 // [JsonDerivedType(typeof(McpServerToolApprovalRequestContent), typeDiscriminator: "mcpServerToolApprovalRequest")]
 // [JsonDerivedType(typeof(McpServerToolApprovalResponseContent), typeDiscriminator: "mcpServerToolApprovalResponse")]
+// [JsonDerivedType(typeof(CodeInterpreterToolCallContent), typeDiscriminator: "codeInterpreterToolCall")]
+// [JsonDerivedType(typeof(CodeInterpreterToolResultContent), typeDiscriminator: "codeInterpreterToolResult")]
 
 public class AIContent
 {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a code interpreter tool call invocation by a hosted service.
+/// </summary>
+/// <remarks>
+/// This content type represents when a hosted AI service invokes a code interpreter tool.
+/// It is informational only and represents the call itself, not the result.
+/// </remarks>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolCallContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolCallContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolCallContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the inputs to the code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Inputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, or other <see cref="AIContent"/> types as supported
+    /// by the service. Typically <see cref="Inputs"/> includes a <see cref="DataContent"/> with a "text/x-python"
+    /// media type representing the code for execution by the code interpreter tool.
+    /// </remarks>
+    public IList<AIContent>? Inputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents the result of a code interpreter tool invocation by a hosted service.
+/// </summary>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolResultContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolResultContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolResultContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID that this result corresponds to.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the output of code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Outputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, <see cref="TextContent"/> for standard output text,
+    /// or other <see cref="AIContent"/> types as supported by the service.
+    /// </remarks>
+    public IList<AIContent>? Outputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs

@@ -9,6 +9,7 @@
 #endif
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Text;
 #if !NET
 using System.Runtime.InteropServices;
 #endif
@@ -115,6 +116,7 @@ public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string?
     /// <param name="mediaType">The media type (also known as MIME type) represented by the content.</param>
     /// <exception cref="ArgumentNullException"><paramref name="mediaType"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentException"><paramref name="mediaType"/> is empty or composed entirely of whitespace.</exception>
+    /// <exception cref="ArgumentException"><paramref name="mediaType"/> represents an invalid media type.</exception>
     public DataContent(ReadOnlyMemory<byte> data, string mediaType)
     {
         MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType);
@@ -236,6 +238,16 @@ private string DebuggerDisplay
     {
         get
         {
+            if (HasTopLevelMediaType("text"))
+            {
+                return $"MediaType = {MediaType}, Text = \"{Encoding.UTF8.GetString(Data.ToArray())}\"";
+            }
+
+            if ("application/json".Equals(MediaType, StringComparison.OrdinalIgnoreCase))
+            {
+                return $"JSON = {Encoding.UTF8.GetString(Data.ToArray())}";
+            }
+
             const int MaxLength = 80;
 
             string uri = Uri;

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs

@@ -14,11 +14,9 @@ namespace Microsoft.Extensions.AI;
 /// Unlike <see cref="DataContent"/> which contains the data for a file or blob, this class represents a file that is hosted
 /// by the AI service and referenced by an identifier. Such identifiers are specific to the provider.
 /// </remarks>
-[DebuggerDisplay("FileId = {FileId}")]
+[DebuggerDisplay("{DebuggerDisplay,nq}")]
 public sealed class HostedFileContent : AIContent
 {
-    private string _fileId;
-
     /// <summary>
     /// Initializes a new instance of the <see cref="HostedFileContent"/> class.
     /// </summary>
@@ -27,7 +25,7 @@ public sealed class HostedFileContent : AIContent
     /// <exception cref="ArgumentException"><paramref name="fileId"/> is empty or composed entirely of whitespace.</exception>
     public HostedFileContent(string fileId)
     {
-        _fileId = Throw.IfNullOrWhitespace(fileId);
+        FileId = fileId;
     }
 
     /// <summary>
@@ -37,7 +35,58 @@ public HostedFileContent(string fileId)
     /// <exception cref="ArgumentException"><paramref name="value"/> is empty or composed entirely of whitespace.</exception>
     public string FileId
     {
-        get => _fileId;
-        set => _fileId = Throw.IfNullOrWhitespace(value);
+        get => field;
+        set => field = Throw.IfNullOrWhitespace(value);
+    }
+
+    /// <summary>Gets or sets an optional media type (also known as MIME type) associated with the file.</summary>
+    /// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
+    public string? MediaType
+    {
+        get;
+        set => field = value is not null ? DataUriParser.ThrowIfInvalidMediaType(value) : value;
+    }
+
+    /// <summary>Gets or sets an optional name associated with the file.</summary>
+    public string? Name { get; set; }
+
+    /// <summary>
+    /// Determines whether the <see cref="MediaType"/>'s top-level type matches the specified <paramref name="topLevelType"/>.
+    /// </summary>
+    /// <param name="topLevelType">The type to compare against <see cref="MediaType"/>.</param>
+    /// <returns><see langword="true"/> if the type portion of <see cref="MediaType"/> matches the specified value; otherwise, false.</returns>
+    /// <remarks>
+    /// <para>
+    /// A media type is primarily composed of two parts, a "type" and a "subtype", separated by a slash ("/").
+    /// The type portion is also referred to as the "top-level type"; for example,
+    /// "image/png" has a top-level type of "image". <see cref="HasTopLevelMediaType"/> compares
+    /// the specified <paramref name="topLevelType"/> against the type portion of <see cref="MediaType"/>.
+    /// </para>
+    /// <para>
+    /// If <see cref="MediaType"/> is <see langword="null"/>, this method returns <see langword="false"/>.
+    /// </para>
+    /// </remarks>
+    public bool HasTopLevelMediaType(string topLevelType) => MediaType is not null && DataUriParser.HasTopLevelMediaType(MediaType, topLevelType);
+
+    /// <summary>Gets a string representing this instance to display in the debugger.</summary>
+    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
+    private string DebuggerDisplay
+    {
+        get
+        {
+            string display = $"FileId = {FileId}";
+
+            if (MediaType is string mediaType)
+            {
+                display += $", MediaType = {mediaType}";
+            }
+
+            if (Name is string name)
+            {
+                display += $", Name = \"{name}\"";
+            }
+
+            return display;
+        }
     }
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs

@@ -67,6 +67,7 @@ public Uri Uri
     }
 
     /// <summary>Gets or sets the media type (also known as MIME type) for this content.</summary>
+    /// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
     public string MediaType
     {
         get => _mediaType;