Image generation tool (#6749)

* Prototype of using ImageGenerationTool

* Handle DataContent returned from ImageGen

* React to rename and improve metadata

* Handle image_generation tool content from streaming

* Add handling for combining updates with images

* Add tests for new ChatResponseUpdateExtensions

* Rename ImageGenerationTool to HostedImageGenerationTool

* Remove ChatResponseUpdateCoalescingOptions

* Add ImageGeneratingChatClient

* Fix namespace of tool

* Replace traces of function calling

* More namepsace fix

* Enable editing

* Update to preview OpenAI with image tool support

* Temporary OpenAI feed

* Fix tests

* Add integration tests for ImageGeneratingChatClient

* Remove ChatRole.Tool -> Assistant workaround

* Remove use of private reflection for Image results

* Add ChatResponseUpdate.Clone

* Move all mutable state into RequestState object

* Adjust prompt to improve integration test reliability

* Refactor tool initialization

I verified that the tool creation is cached by ReflectionAIFunctionDescriptor

This change includes a small optimization to avoid additional allocation
around inserting tools into the options.

* Add integration tests for streaming

Fixes the removal of tool content - this was broken for streaming when
I changed removal to be based on callId.  We don't have the CallId yet
in the streaming case so we have to remove by name.

* React to changes and fix tests

* Address feedback

* Fix SkipTestException from ConditionalTheory

* Fix formatting

* Add back image replacement coalescing (removed in merge)

* Fix template tests and use new OpenAI

* Remove use of temporary staging nuget feed

* Address feedback

* Make ImageGeneratingChatClient use ImageGenerationTool*Content

* Remove ApplyUpdates and Coalesce ImageResults instead of DataContent.

* Workaround OpenAI issue where image data is not read for partial images.

openai/openai-dotnet#809

* Improved workaround

* Return ImageGenerationToolCallContent from OpenAI

* Add OpenAI image tool tests with representation of real traffic

* Correct the event sequence for streaming single image

* Fix some docs and refactor for clarity

Author: ericstj

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -184,6 +184,47 @@ static async Task<ChatResponse> ToChatResponseAsync(
         }
     }
 
+    /// <summary>
+    /// Coalesces image result content elements in the provided list of <see cref="AIContent"/> items.
+    /// Unlike other content coalescing methods, this will coalesce non-sequential items based on their Name property, 
+    /// and it will replace earlier items with later ones when duplicates are found.
+    /// </summary>
+    private static void CoalesceImageResultContent(IList<AIContent> contents)
+    {
+        Dictionary<string, int>? imageResultIndexById = null;
+        bool hasRemovals = false;
+
+        for (int i = 0; i < contents.Count; i++)
+        {
+            if (contents[i] is ImageGenerationToolResultContent imageResult && !string.IsNullOrEmpty(imageResult.ImageId))
+            {
+                // Check if there's an existing ImageGenerationToolResultContent with the same ImageId to replace
+                if (imageResultIndexById is null)
+                {
+                    imageResultIndexById = new(StringComparer.Ordinal);
+                }
+
+                if (imageResultIndexById.TryGetValue(imageResult.ImageId!, out int existingIndex))
+                {
+                    // Replace the existing imageResult with the new one
+                    contents[existingIndex] = imageResult;
+                    contents[i] = null!; // Mark the current one for removal, then remove in single o(n) pass
+                    hasRemovals = true;
+                }
+                else
+                {
+                    imageResultIndexById[imageResult.ImageId!] = i;
+                }
+            }
+        }
+
+        // Remove all of the null slots left over from the coalescing process.
+        if (hasRemovals)
+        {
+            RemoveNullContents(contents);
+        }
+    }
+
     /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
     internal static void CoalesceContent(IList<AIContent> contents)
     {
@@ -219,6 +260,8 @@ internal static void CoalesceContent(IList<AIContent> contents)
                 return content;
             });
 
+        CoalesceImageResultContent(contents);
+
         Coalesce<DataContent>(
             contents,
             mergeSingle: false,
@@ -394,29 +437,35 @@ static bool TryAsCoalescable(AIContent content, [NotNullWhen(true)] out TContent
             }
 
             // Remove all of the null slots left over from the coalescing process.
-            if (contents is List<AIContent> contentsList)
-            {
-                _ = contentsList.RemoveAll(u => u is null);
-            }
-            else
-            {
-                int nextSlot = 0;
-                int contentsCount = contents.Count;
-                for (int i = 0; i < contentsCount; i++)
-                {
-                    if (contents[i] is { } content)
-                    {
-                        contents[nextSlot++] = content;
-                    }
-                }
+            RemoveNullContents(contents);
+        }
+    }
 
-                for (int i = contentsCount - 1; i >= nextSlot; i--)
+    private static void RemoveNullContents<T>(IList<T> contents)
+        where T : class
+    {
+        if (contents is List<AIContent> contentsList)
+        {
+            _ = contentsList.RemoveAll(u => u is null);
+        }
+        else
+        {
+            int nextSlot = 0;
+            int contentsCount = contents.Count;
+            for (int i = 0; i < contentsCount; i++)
+            {
+                if (contents[i] is { } content)
                 {
-                    contents.RemoveAt(i);
+                    contents[nextSlot++] = content;
                 }
+            }
 
-                Debug.Assert(nextSlot == contents.Count, "Expected final count to equal list length.");
+            for (int i = contentsCount - 1; i >= nextSlot; i--)
+            {
+                contents.RemoveAt(i);
             }
+
+            Debug.Assert(nextSlot == contents.Count, "Expected final count to equal list length.");
         }
     }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs

@@ -20,7 +20,7 @@ namespace Microsoft.Extensions.AI;
 /// </para>
 /// <para>
 /// The relationship between <see cref="ChatResponse"/> and <see cref="ChatResponseUpdate"/> is
-/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync"/> and
+/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync(IAsyncEnumerable{ChatResponseUpdate}, System.Threading.CancellationToken)"/> and
 /// <see cref="ChatResponse.ToChatResponseUpdates"/>, which enable bidirectional conversions
 /// between the two. Note, however, that the provided conversions might be lossy, for example, if multiple
 /// updates all have different <see cref="RawRepresentation"/> objects whereas there's only one slot for
@@ -58,6 +58,29 @@ public ChatResponseUpdate(ChatRole? role, IList<AIContent>? contents)
         _contents = contents;
     }
 
+    /// <summary>
+    /// Creates a new ChatResponseUpdate instance that is a copy of the current object.
+    /// </summary>
+    /// <remarks>The cloned object is a shallow copy; reference-type properties will reference the same
+    /// objects as the original. Use this method to duplicate the response update for further modification without
+    /// affecting the original instance.</remarks>
+    /// <returns>A new ChatResponseUpdate object with the same property values as the current instance.</returns>
+    public ChatResponseUpdate Clone() =>
+        new()
+        {
+            AdditionalProperties = AdditionalProperties,
+            AuthorName = AuthorName,
+            Contents = Contents,
+            CreatedAt = CreatedAt,
+            ConversationId = ConversationId,
+            FinishReason = FinishReason,
+            MessageId = MessageId,
+            ModelId = ModelId,
+            RawRepresentation = RawRepresentation,
+            ResponseId = ResponseId,
+            Role = Role,
+        };
+
     /// <summary>Gets or sets the name of the author of the response update.</summary>
     public string? AuthorName
     {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolCallContent.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents the invocation of an image generation tool call by a hosted service.
+/// </summary>
+[Experimental("MEAI001")]
+public sealed class ImageGenerationToolCallContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ImageGenerationToolCallContent"/> class.
+    /// </summary>
+    public ImageGenerationToolCallContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the unique identifier of the image generation item.
+    /// </summary>
+    public string? ImageId { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolResultContent.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents an image generation tool call invocation by a hosted service.
+/// </summary>
+/// <remarks>
+/// This content type represents when a hosted AI service invokes an image generation tool.
+/// It is informational only and represents the call itself, not the result.
+/// </remarks>
+[Experimental("MEAI001")]
+public sealed class ImageGenerationToolResultContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ImageGenerationToolResultContent"/> class.
+    /// </summary>
+    public ImageGenerationToolResultContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the unique identifier of the image generation item.
+    /// </summary>
+    public string? ImageId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the generated content items.
+    /// </summary>
+    /// <remarks>
+    /// Content is typically <see cref="DataContent"/> for images streamed from the tool, or <see cref="UriContent"/> for remotely hosted images, but
+    /// can also be provider-specific content types that represent the generated images.
+    /// </remarks>
+    public IList<AIContent>? Outputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationOptions.cs

@@ -81,6 +81,11 @@ protected ImageGenerationOptions(ImageGenerationOptions? other)
     /// </summary>
     public ImageGenerationResponseFormat? ResponseFormat { get; set; }
 
+    /// <summary>
+    /// Gets or sets the number of intermediate streaming images to generate.
+    /// </summary>
+    public int? StreamingCount { get; set; }
+
     /// <summary>Gets or sets any additional properties associated with the options.</summary>
     public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json

@@ -1247,6 +1247,10 @@
           "Member": "Microsoft.Extensions.AI.ChatResponseUpdate.ChatResponseUpdate(Microsoft.Extensions.AI.ChatRole? role, System.Collections.Generic.IList<Microsoft.Extensions.AI.AIContent>? contents);",
           "Stage": "Stable"
         },
+        {
+          "Member": "Microsoft.Extensions.AI.ChatResponseUpdate Microsoft.Extensions.AI.ChatResponseUpdate.Clone();",
+          "Stage": "Stable"
+        },
         {
           "Member": "override string Microsoft.Extensions.AI.ChatResponseUpdate.ToString();",
           "Stage": "Stable"

src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs

@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents a hosted tool that can be specified to an AI service to enable it to perform image generation.</summary>
+/// <remarks>
+/// This tool does not itself implement image generation. It is a marker that can be used to inform a service
+/// that the service is allowed to perform image generation if the service is capable of doing so.
+/// </remarks>
+[Experimental("MEAI001")]
+public class HostedImageGenerationTool : AITool
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="HostedImageGenerationTool"/> class with the specified options.
+    /// </summary>
+    public HostedImageGenerationTool()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the options used to configure image generation.
+    /// </summary>
+    public ImageGenerationOptions? Options { get; set; }
+}