Special-case AIContent returned from AIFunctionFactory.Create AIFunctions to not be serialized (dotnet#6935)

* Special-case AIContent returned from AIFunctionFactory.Create AIFunctions to not be serialized

They'll likely end up being serialized, anyway, by leaf IChatClients, but this gives those IChatClients the opportunity to do something different, such as by treating DataContent differently.

* Update XML comments

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs

@@ -99,6 +99,11 @@ public static partial class AIFunctionFactory
     /// <para>
     /// By default, return values are serialized to <see cref="JsonElement"/> using <paramref name="options"/>'s
     /// <see cref="AIFunctionFactoryOptions.SerializerOptions"/> if provided, or else using <see cref="AIJsonUtilities.DefaultOptions"/>.
+    /// However, return values whose declared type is <see cref="AIContent"/>, a derived type of <see cref="AIContent"/>, or
+    /// any type assignable from <see cref="IEnumerable{AIContent}"/> (e.g. <c>AIContent[]</c>, <c>List&lt;AIContent&gt;</c>) are
+    /// special-cased and are not serialized: the created function returns the original instance(s) directly to enable
+    /// callers (such as an <c>IChatClient</c>) to perform type tests and implement specialized handling. If
+    /// <see cref="AIFunctionFactoryOptions.MarshalResult"/> is supplied, that delegate governs the behavior instead.
     /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
     /// </para>
     /// </remarks>
@@ -172,7 +177,9 @@ public static AIFunction Create(Delegate method, AIFunctionFactoryOptions? optio
     /// </para>
     /// <para>
     /// Return values are serialized to <see cref="JsonElement"/> using <paramref name="serializerOptions"/> if provided,
-    /// or else using <see cref="AIJsonUtilities.DefaultOptions"/>.
+    /// or else using <see cref="AIJsonUtilities.DefaultOptions"/>. However, return values whose declared type is <see cref="AIContent"/>, a
+    /// derived type of <see cref="AIContent"/>, or any type assignable from <see cref="IEnumerable{AIContent}"/> are not serialized;
+    /// they are returned as-is to facilitate specialized handling.
     /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
@@ -262,6 +269,8 @@ public static AIFunction Create(Delegate method, string? name = null, string? de
     /// <para>
     /// By default, return values are serialized to <see cref="JsonElement"/> using <paramref name="options"/>'s
     /// <see cref="AIFunctionFactoryOptions.SerializerOptions"/> if provided, or else using <see cref="AIJsonUtilities.DefaultOptions"/>.
+    /// However, return values whose declared type is <see cref="AIContent"/>, a derived type of <see cref="AIContent"/>, or
+    /// any type assignable from <see cref="IEnumerable{AIContent}"/> are not serialized and are instead returned directly.
     /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
     /// </para>
     /// </remarks>
@@ -345,7 +354,9 @@ public static AIFunction Create(MethodInfo method, object? target, AIFunctionFac
     /// </para>
     /// <para>
     /// Return values are serialized to <see cref="JsonElement"/> using <paramref name="serializerOptions"/> if provided,
-    /// or else using <see cref="AIJsonUtilities.DefaultOptions"/>.
+    /// or else using <see cref="AIJsonUtilities.DefaultOptions"/>. However, return values whose declared type is <see cref="AIContent"/>, a
+    /// derived type of <see cref="AIContent"/>, or any type assignable from <see cref="IEnumerable{AIContent}"/> are returned
+    /// without serialization to enable specialized handling.
     /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
@@ -448,6 +459,8 @@ public static AIFunction Create(MethodInfo method, object? target, string? name
     /// <para>
     /// By default, return values are serialized to <see cref="JsonElement"/> using <paramref name="options"/>'s
     /// <see cref="AIFunctionFactoryOptions.SerializerOptions"/> if provided, or else using <see cref="AIJsonUtilities.DefaultOptions"/>.
+    /// However, return values whose declared type is <see cref="AIContent"/>, a derived type of <see cref="AIContent"/>, or any type
+    /// assignable from <see cref="IEnumerable{AIContent}"/> are returned directly without serialization.
     /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
     /// </para>
     /// </remarks>
@@ -720,7 +733,7 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions
             Description = key.Description ?? key.Method.GetCustomAttribute<DescriptionAttribute>(inherit: true)?.Description ?? string.Empty;
             JsonSerializerOptions = serializerOptions;
             ReturnJsonSchema = returnType is null || key.ExcludeResultSchema ? null : AIJsonUtilities.CreateJsonSchema(
-                returnType,
+                NormalizeReturnType(returnType, serializerOptions),
                 description: key.Method.ReturnParameter.GetCustomAttribute<DescriptionAttribute>(inherit: true)?.Description,
                 serializerOptions: serializerOptions,
                 inferenceOptions: schemaOptions);
@@ -978,6 +991,7 @@ static void ThrowNullServices(string parameterName) =>
                     MethodInfo taskResultGetter = GetMethodFromGenericMethodDefinition(returnType, _taskGetResult);
                     returnType = taskResultGetter.ReturnType;
 
+                    // If a MarshalResult delegate is provided, use it.
                     if (marshalResult is not null)
                     {
                         return async (taskObj, cancellationToken) =>
@@ -988,6 +1002,18 @@ static void ThrowNullServices(string parameterName) =>
                         };
                     }
 
+                    // Special-case AIContent results to not be serialized, so that IChatClients can type test and handle them
+                    // specially, such as by returning content to the model/service in a manner appropriate to the content type.
+                    if (IsAIContentRelatedType(returnType))
+                    {
+                        return async (taskObj, cancellationToken) =>
+                        {
+                            await ((Task)ThrowIfNullResult(taskObj)).ConfigureAwait(true);
+                            return ReflectionInvoke(taskResultGetter, taskObj, null);
+                        };
+                    }
+
+                    // For everything else, just serialize the result as-is.
                     returnTypeInfo = serializerOptions.GetTypeInfo(returnType);
                     return async (taskObj, cancellationToken) =>
                     {
@@ -1004,6 +1030,7 @@ static void ThrowNullServices(string parameterName) =>
                     MethodInfo asTaskResultGetter = GetMethodFromGenericMethodDefinition(valueTaskAsTask.ReturnType, _taskGetResult);
                     returnType = asTaskResultGetter.ReturnType;
 
+                    // If a MarshalResult delegate is provided, use it.
                     if (marshalResult is not null)
                     {
                         return async (taskObj, cancellationToken) =>
@@ -1015,6 +1042,19 @@ static void ThrowNullServices(string parameterName) =>
                         };
                     }
 
+                    // Special-case AIContent results to not be serialized, so that IChatClients can type test and handle them
+                    // specially, such as by returning content to the model/service in a manner appropriate to the content type.
+                    if (IsAIContentRelatedType(returnType))
+                    {
+                        return async (taskObj, cancellationToken) =>
+                        {
+                            var task = (Task)ReflectionInvoke(valueTaskAsTask, ThrowIfNullResult(taskObj), null)!;
+                            await task.ConfigureAwait(true);
+                            return ReflectionInvoke(asTaskResultGetter, task, null);
+                        };
+                    }
+
+                    // For everything else, just serialize the result as-is.
                     returnTypeInfo = serializerOptions.GetTypeInfo(returnType);
                     return async (taskObj, cancellationToken) =>
                     {
@@ -1026,13 +1066,21 @@ static void ThrowNullServices(string parameterName) =>
                 }
             }
 
-            // For everything else, just serialize the result as-is.
+            // If a MarshalResult delegate is provided, use it.
             if (marshalResult is not null)
             {
                 Type returnTypeCopy = returnType;
                 return (result, cancellationToken) => marshalResult(result, returnTypeCopy, cancellationToken);
             }
 
+            // Special-case AIContent results to not be serialized, so that IChatClients can type test and handle them
+            // specially, such as by returning content to the model/service in a manner appropriate to the content type.
+            if (IsAIContentRelatedType(returnType))
+            {
+                return static (result, _) => new ValueTask<object?>(result);
+            }
+
+            // For everything else, just serialize the result as-is.
             returnTypeInfo = serializerOptions.GetTypeInfo(returnType);
             return (result, cancellationToken) => SerializeResultAsync(result, returnTypeInfo, cancellationToken);
 
@@ -1069,6 +1117,41 @@ private static MethodInfo GetMethodFromGenericMethodDefinition(Type specializedT
 #endif
         }
 
+        private static bool IsAIContentRelatedType(Type type) =>
+            typeof(AIContent).IsAssignableFrom(type) ||
+            typeof(IEnumerable<AIContent>).IsAssignableFrom(type);
+
+        private static Type NormalizeReturnType(Type type, JsonSerializerOptions? options)
+        {
+            options ??= AIJsonUtilities.DefaultOptions;
+
+            if (options == AIJsonUtilities.DefaultOptions && !options.TryGetTypeInfo(type, out _))
+            {
+                // GetTypeInfo is not polymorphic, so attempts to look up derived types will fail even if the
+                // base type is registered. In some cases, though, we can fall back to using interfaces
+                // we know we have contracts for in AIJsonUtilities.DefaultOptions where the semantics of using
+                // that interface will be reasonable. This should really only affect situations where
+                // reflection-based serialization is disabled.
+
+                if (typeof(IEnumerable<AIContent>).IsAssignableFrom(type))
+                {
+                    return typeof(IEnumerable<AIContent>);
+                }
+
+                if (typeof(IEnumerable<ChatMessage>).IsAssignableFrom(type))
+                {
+                    return typeof(IEnumerable<ChatMessage>);
+                }
+
+                if (typeof(IEnumerable<string>).IsAssignableFrom(type))
+                {
+                    return typeof(IEnumerable<string>);
+                }
+            }
+
+            return type;
+        }
+
         private record struct DescriptorKey(
             MethodInfo Method,
             string? Name,

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs

@@ -75,39 +75,23 @@ private static JsonSerializerOptions CreateDefaultOptions()
         UseStringEnumConverter = true,
         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
         WriteIndented = true)]
-    [JsonSerializable(typeof(SpeechToTextOptions))]
-    [JsonSerializable(typeof(SpeechToTextClientMetadata))]
-    [JsonSerializable(typeof(SpeechToTextResponse))]
-    [JsonSerializable(typeof(SpeechToTextResponseUpdate))]
-    [JsonSerializable(typeof(IReadOnlyList<SpeechToTextResponseUpdate>))]
-    [JsonSerializable(typeof(ImageGenerationOptions))]
-    [JsonSerializable(typeof(ImageGenerationResponse))]
-    [JsonSerializable(typeof(IList<ChatMessage>))]
-    [JsonSerializable(typeof(IEnumerable<ChatMessage>))]
-    [JsonSerializable(typeof(ChatMessage[]))]
-    [JsonSerializable(typeof(ChatOptions))]
-    [JsonSerializable(typeof(EmbeddingGenerationOptions))]
-    [JsonSerializable(typeof(ChatClientMetadata))]
-    [JsonSerializable(typeof(EmbeddingGeneratorMetadata))]
-    [JsonSerializable(typeof(ChatResponse))]
-    [JsonSerializable(typeof(ChatResponseUpdate))]
-    [JsonSerializable(typeof(IReadOnlyList<ChatResponseUpdate>))]
-    [JsonSerializable(typeof(Dictionary<string, object>))]
-    [JsonSerializable(typeof(IDictionary<string, object?>))]
+
+    // JSON
     [JsonSerializable(typeof(JsonDocument))]
     [JsonSerializable(typeof(JsonElement))]
     [JsonSerializable(typeof(JsonNode))]
     [JsonSerializable(typeof(JsonObject))]
     [JsonSerializable(typeof(JsonValue))]
     [JsonSerializable(typeof(JsonArray))]
-    [JsonSerializable(typeof(IEnumerable<string>))]
-    [JsonSerializable(typeof(char))]
+
+    // Primitives
     [JsonSerializable(typeof(string))]
-    [JsonSerializable(typeof(int))]
+    [JsonSerializable(typeof(char))]
     [JsonSerializable(typeof(short))]
-    [JsonSerializable(typeof(long))]
-    [JsonSerializable(typeof(uint))]
     [JsonSerializable(typeof(ushort))]
+    [JsonSerializable(typeof(int))]
+    [JsonSerializable(typeof(uint))]
+    [JsonSerializable(typeof(long))]
     [JsonSerializable(typeof(ulong))]
     [JsonSerializable(typeof(float))]
     [JsonSerializable(typeof(double))]
@@ -116,26 +100,58 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(TimeSpan))]
     [JsonSerializable(typeof(DateTime))]
     [JsonSerializable(typeof(DateTimeOffset))]
-    [JsonSerializable(typeof(Embedding))]
-    [JsonSerializable(typeof(Embedding<byte>))]
-    [JsonSerializable(typeof(Embedding<int>))]
-#if NET
-    [JsonSerializable(typeof(Embedding<Half>))]
-#endif
-    [JsonSerializable(typeof(Embedding<float>))]
-    [JsonSerializable(typeof(Embedding<double>))]
-    [JsonSerializable(typeof(AIContent))]
+
+    // AIFunction
     [JsonSerializable(typeof(AIFunctionArguments))]
 
-    // Temporary workaround:
-    // These should be added in once they're no longer [Experimental] and included via [JsonDerivedType] on AIContent.
+    // IChatClient
+    [JsonSerializable(typeof(IEnumerable<ChatMessage>))]
+    [JsonSerializable(typeof(IList<ChatMessage>))]
+    [JsonSerializable(typeof(ChatMessage[]))]
+    [JsonSerializable(typeof(ChatOptions))]
+    [JsonSerializable(typeof(ChatClientMetadata))]
+    [JsonSerializable(typeof(ChatResponse))]
+    [JsonSerializable(typeof(ChatResponseUpdate))]
+    [JsonSerializable(typeof(IReadOnlyList<ChatResponseUpdate>))]
+    [JsonSerializable(typeof(Dictionary<string, object>))]
+    [JsonSerializable(typeof(IDictionary<string, object?>))]
+    [JsonSerializable(typeof(IEnumerable<string>))]
+    [JsonSerializable(typeof(AIContent))]
+    [JsonSerializable(typeof(IEnumerable<AIContent>))]
+
+    // Temporary workaround: These should be implicitly added in once they're no longer [Experimental]
+    // and are included via [JsonDerivedType] on AIContent.
     [JsonSerializable(typeof(FunctionApprovalRequestContent))]
     [JsonSerializable(typeof(FunctionApprovalResponseContent))]
     [JsonSerializable(typeof(McpServerToolCallContent))]
     [JsonSerializable(typeof(McpServerToolResultContent))]
     [JsonSerializable(typeof(McpServerToolApprovalRequestContent))]
     [JsonSerializable(typeof(McpServerToolApprovalResponseContent))]
     [JsonSerializable(typeof(ResponseContinuationToken))]
+
+    // IEmbeddingGenerator
+    [JsonSerializable(typeof(EmbeddingGenerationOptions))]
+    [JsonSerializable(typeof(EmbeddingGeneratorMetadata))]
+    [JsonSerializable(typeof(Embedding))]
+    [JsonSerializable(typeof(Embedding<byte>))]
+    [JsonSerializable(typeof(Embedding<int>))]
+#if NET
+    [JsonSerializable(typeof(Embedding<Half>))]
+#endif
+    [JsonSerializable(typeof(Embedding<float>))]
+    [JsonSerializable(typeof(Embedding<double>))]
+
+    // ISpeechToTextClient
+    [JsonSerializable(typeof(SpeechToTextOptions))]
+    [JsonSerializable(typeof(SpeechToTextClientMetadata))]
+    [JsonSerializable(typeof(SpeechToTextResponse))]
+    [JsonSerializable(typeof(SpeechToTextResponseUpdate))]
+    [JsonSerializable(typeof(IReadOnlyList<SpeechToTextResponseUpdate>))]
+
+    // IImageGenerator
+    [JsonSerializable(typeof(ImageGenerationOptions))]
+    [JsonSerializable(typeof(ImageGenerationResponse))]
+
     [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead.
     private sealed partial class JsonContext : JsonSerializerContext;
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs

@@ -760,15 +760,27 @@ internal static IEnumerable<ResponseItem> ToOpenAIResponseItems(IEnumerable<Chat
 
                         case FunctionResultContent resultContent:
                             string? result = resultContent.Result as string;
-                            if (result is null && resultContent.Result is not null)
+                            if (result is null && resultContent.Result is { } resultObj)
                             {
-                                try
+                                switch (resultObj)
                                 {
-                                    result = JsonSerializer.Serialize(resultContent.Result, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object)));
-                                }
-                                catch (NotSupportedException)
-                                {
-                                    // If the type can't be serialized, skip it.
+                                    // https://github.com/openai/openai-dotnet/issues/759
+                                    // Once OpenAI supports other forms of tool call outputs, special-case various AIContent types here, e.g.
+                                    // case DataContent
+                                    // case HostedFileContent
+                                    // case IEnumerable<AIContent>
+                                    // etc.
+
+                                    default:
+                                        try
+                                        {
+                                            result = JsonSerializer.Serialize(resultContent.Result, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object)));
+                                        }
+                                        catch (NotSupportedException)
+                                        {
+                                            // If the type can't be serialized, skip it.
+                                        }
+                                        break;
                                 }
                             }
 

test/Libraries/Microsoft.Extensions.AI.Integration.Tests/VerbatimHttpHandler.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.Net.Http;
 using System.Text;
 using System.Text.Json.Nodes;
@@ -85,12 +86,24 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
         return new() { Content = new StringContent(_expectedOutput) };
     }
 
-    public static string? RemoveWhiteSpace(string? text) =>
-        text is null ? null :
-        Regex.Replace(text, @"\s*", string.Empty);
+    [return: NotNullIfNotNull(nameof(text))]
+    public static string? RemoveWhiteSpace(string? text)
+    {
+        if (text is null)
+        {
+            return null;
+        }
+
+        text = text.Replace("\\r", "").Replace("\\n", "").Replace("\\t", "");
+
+        return Regex.Replace(text, @"\s*", string.Empty);
+    }
 
     private static void AssertEqualNormalized(string expected, string actual)
     {
+        expected = RemoveWhiteSpace(expected);
+        actual = RemoveWhiteSpace(actual);
+
         // First try to compare as JSON.
         JsonNode? expectedNode = null;
         JsonNode? actualNode = null;
@@ -114,10 +127,7 @@ private static void AssertEqualNormalized(string expected, string actual)
         }
 
         // Legitimately may not have been JSON. Fall back to whitespace normalization.
-        if (RemoveWhiteSpace(expected) != RemoveWhiteSpace(actual))
-        {
-            FailNotEqual(expected, actual);
-        }
+        FailNotEqual(expected, actual);
     }
 
     private static void FailNotEqual(string expected, string actual) =>