add support for background responses (#6854)

* add support for background responses

Adds a model for supporting background responses (long-running operations), updates the chat completion model to use it, and integrates this functionality into OpenAIResponsesChatClient.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Stephen Toub <stoub@microsoft.com>
Co-authored-by: westey <164392973+westey-m@users.noreply.github.com>

Author: 4 people

src/Libraries/.editorconfig

@@ -2936,7 +2936,7 @@ dotnet_diagnostic.S103.severity = suggestion
 # Title    : Files should not have too many lines of code
 # Category : Major Code Smell
 # Help Link: https://rules.sonarsource.com/csharp/RSPEC-104
-dotnet_diagnostic.S104.severity = warning
+dotnet_diagnostic.S104.severity = none
 
 # Title    : Finalizers should not throw exceptions
 # Category : Blocker Bug

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -5,6 +5,7 @@
 - Added protected copy constructors to options types (e.g. `ChatOptions`).
 - Fixed `EmbeddingGeneratorOptions`/`SpeechToTextOptions` `Clone` methods to correctly copy all properties.
 - Fixed `ToChatResponse` to not overwrite `ChatMessage/ChatResponse.CreatedAt` with older timestamps during coalescing. 
+- Added `[Experimental]` support for background responses, such that non-streaming responses are allowed to be pollable, and such that responses and response updates can be tagged with continuation tokens to support later resumption.
 
 ## 9.9.1
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.Extensions.AI;
@@ -25,8 +26,10 @@ protected ChatOptions(ChatOptions? other)
         }
 
         AdditionalProperties = other.AdditionalProperties?.Clone();
+        AllowBackgroundResponses = other.AllowBackgroundResponses;
         AllowMultipleToolCalls = other.AllowMultipleToolCalls;
         ConversationId = other.ConversationId;
+        ContinuationToken = other.ContinuationToken;
         FrequencyPenalty = other.FrequencyPenalty;
         Instructions = other.Instructions;
         MaxOutputTokens = other.MaxOutputTokens;
@@ -155,6 +158,47 @@ protected ChatOptions(ChatOptions? other)
     [JsonIgnore]
     public IList<AITool>? Tools { get; set; }
 
+    /// <summary>Gets or sets a value indicating whether the background responses are allowed.</summary>
+    /// <remarks>
+    /// <para>
+    /// Background responses allow running long-running operations or tasks asynchronously in the background that can be resumed by streaming APIs
+    /// and polled for completion by non-streaming APIs.
+    /// </para>
+    /// <para>
+    /// When this property is set to true, non-streaming APIs may start a background operation and return an initial
+    /// response with a continuation token. Subsequent calls to the same API should be made in a polling manner with
+    /// the continuation token to get the final result of the operation.
+    /// </para>
+    /// <para>
+    /// When this property is set to true, streaming APIs may also start a background operation and begin streaming
+    /// response updates until the operation is completed. If the streaming connection is interrupted, the
+    /// continuation token obtained from the last update that has one should be supplied to a subsequent call to the same streaming API
+    /// to resume the stream from the point of interruption and continue receiving updates until the operation is completed.
+    /// </para>
+    /// <para>
+    /// This property only takes effect if the implementation it's used with supports background responses.
+    /// If the implementation does not support background responses, this property will be ignored.
+    /// </para>
+    /// </remarks>
+    [Experimental("MEAI001")]
+    [JsonIgnore]
+    public bool? AllowBackgroundResponses { get; set; }
+
+    /// <summary>Gets or sets the continuation token for resuming and getting the result of the chat response identified by this token.</summary>
+    /// <remarks>
+    /// This property is used for background responses that can be activated via the <see cref="AllowBackgroundResponses"/>
+    /// property if the <see cref="IChatClient"/> implementation supports them.
+    /// Streamed background responses, such as those returned by default by <see cref="IChatClient.GetStreamingResponseAsync"/>,
+    /// can be resumed if interrupted. This means that a continuation token obtained from the <see cref="ChatResponseUpdate.ContinuationToken"/> 
+    /// of an update just before the interruption occurred can be passed to this property to resume the stream from the point of interruption.
+    /// Non-streamed background responses, such as those returned by <see cref="IChatClient.GetResponseAsync"/>,
+    /// can be polled for completion by obtaining the token from the <see cref="ChatResponse.ContinuationToken"/> property 
+    /// and passing it to this property on subsequent calls to <see cref="IChatClient.GetResponseAsync"/>.
+    /// </remarks>
+    [Experimental("MEAI001")]
+    [JsonIgnore]
+    public object? ContinuationToken { get; set; }
+
     /// <summary>
     /// Gets or sets a callback responsible for creating the raw representation of the chat options from an underlying implementation.
     /// </summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs

@@ -88,6 +88,23 @@ public IList<ChatMessage> Messages
     /// <summary>Gets or sets usage details for the chat response.</summary>
     public UsageDetails? Usage { get; set; }
 
+    /// <summary>Gets or sets the continuation token for getting result of the background chat response.</summary>
+    /// <remarks>
+    /// <see cref="IChatClient"/> implementations that support background responses will return
+    /// a continuation token if background responses are allowed in <see cref="ChatOptions.AllowBackgroundResponses"/>
+    /// and the result of the response has not been obtained yet. If the response has completed and the result has been obtained,
+    /// the token will be <see langword="null"/>.
+    /// <para>
+    /// This property should be used in conjunction with <see cref="ChatOptions.ContinuationToken"/> to 
+    /// continue to poll for the completion of the response. Pass this token to
+    /// <see cref="ChatOptions.ContinuationToken"/> on subsequent calls to <see cref="IChatClient.GetResponseAsync"/>
+    /// to poll for completion.
+    /// </para>
+    /// </remarks>
+    [Experimental("MEAI001")]
+    [JsonIgnore]
+    public object? ContinuationToken { get; set; }
+
     /// <summary>Gets or sets the raw representation of the chat response from an underlying implementation.</summary>
     /// <remarks>
     /// If a <see cref="ChatResponse"/> is created to represent some underlying object from another object
@@ -143,6 +160,7 @@ public ChatResponseUpdate[] ToChatResponseUpdates()
                 ResponseId = ResponseId,
 
                 CreatedAt = message.CreatedAt ?? CreatedAt,
+                ContinuationToken = ContinuationToken,
             };
         }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs

@@ -136,6 +136,21 @@ public IList<AIContent> Contents
     /// <inheritdoc/>
     public override string ToString() => Text;
 
+    /// <summary>Gets or sets the continuation token for resuming the streamed chat response of which this update is a part.</summary>
+    /// <remarks>
+    /// <see cref="IChatClient"/> implementations that support background responses will return
+    /// a continuation token on each update if background responses are allowed in <see cref="ChatOptions.AllowBackgroundResponses"/>
+    /// except of the last update, for which the token will be <see langword="null"/>.
+    /// <para>
+    /// This property should be used for stream resumption, where the continuation token of the latest received update should be
+    /// passed to <see cref="ChatOptions.ContinuationToken"/> on subsequent calls to <see cref="IChatClient.GetStreamingResponseAsync"/>
+    /// to resume streaming from the point of interruption.
+    /// </para>
+    /// </remarks>
+    [Experimental("MEAI001")]
+    [JsonIgnore]
+    public object? ContinuationToken { get; set; }
+
     /// <summary>Gets a <see cref="AIContent"/> object to display in the debugger display.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
     private AIContent? ContentForDebuggerDisplay

src/Libraries/Microsoft.Extensions.AI.Abstractions/ResponseContinuationToken.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.ComponentModel;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a token used to resume, continue, or rehydrate an operation across multiple scenarios/calls,
+/// such as resuming a streamed response from a specific point or retrieving the result of a background operation.
+/// Subclasses of this class encapsulate all necessary information within the token to facilitate these actions.
+/// </summary>
+[JsonConverter(typeof(Converter))]
+[Experimental("MEAI001")]
+public class ResponseContinuationToken
+{
+    /// <summary>Bytes representing this token.</summary>
+    private readonly ReadOnlyMemory<byte> _bytes;
+
+    /// <summary>Initializes a new instance of the <see cref="ResponseContinuationToken"/> class.</summary>
+    protected ResponseContinuationToken()
+    {
+    }
+
+    /// <summary>Initializes a new instance of the <see cref="ResponseContinuationToken"/> class.</summary>
+    /// <param name="bytes">Bytes to create the token from.</param>
+    protected ResponseContinuationToken(ReadOnlyMemory<byte> bytes)
+    {
+        _bytes = bytes;
+    }
+
+    /// <summary>Create a new instance of <see cref="ResponseContinuationToken"/> from the provided <paramref name="bytes"/>.
+    /// </summary>
+    /// <param name="bytes">Bytes representing the <see cref="ResponseContinuationToken"/>.</param>
+    /// <returns>A <see cref="ResponseContinuationToken"/> equivalent to the one from which
+    /// the original<see cref="ResponseContinuationToken"/> bytes were obtained.</returns>
+    public static ResponseContinuationToken FromBytes(ReadOnlyMemory<byte> bytes) => new(bytes);
+
+    /// <summary>Gets the bytes representing this <see cref="ResponseContinuationToken"/>.</summary>
+    /// <returns>Bytes representing the <see cref="ResponseContinuationToken"/>.</returns>"/>
+    public virtual ReadOnlyMemory<byte> ToBytes() => _bytes;
+
+    /// <summary>Provides a <see cref="JsonConverter{ResponseContinuationToken}"/> for serializing <see cref="ResponseContinuationToken"/> instances.</summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    [Experimental("MEAI001")]
+    public sealed class Converter : JsonConverter<ResponseContinuationToken>
+    {
+        /// <inheritdoc/>
+        public override ResponseContinuationToken Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            return ResponseContinuationToken.FromBytes(reader.GetBytesFromBase64());
+        }
+
+        /// <inheritdoc/>
+        public override void Write(Utf8JsonWriter writer, ResponseContinuationToken value, JsonSerializerOptions options)
+        {
+            _ = Throw.IfNull(writer);
+            _ = Throw.IfNull(value);
+
+            writer.WriteBase64StringValue(value.ToBytes().Span);
+        }
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs

@@ -135,7 +135,7 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(McpServerToolResultContent))]
     [JsonSerializable(typeof(McpServerToolApprovalRequestContent))]
     [JsonSerializable(typeof(McpServerToolApprovalResponseContent))]
-
+    [JsonSerializable(typeof(ResponseContinuationToken))]
     [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead.
     private sealed partial class JsonContext : JsonSerializerContext;
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs

@@ -64,7 +64,7 @@ public static ChatResponse AsChatResponse(this OpenAIResponse response, Response
     /// <exception cref="ArgumentNullException"><paramref name="responseUpdates"/> is <see langword="null"/>.</exception>
     public static IAsyncEnumerable<ChatResponseUpdate> AsChatResponseUpdatesAsync(
         this IAsyncEnumerable<StreamingResponseUpdate> responseUpdates, ResponseCreationOptions? options = null, CancellationToken cancellationToken = default) =>
-        OpenAIResponsesChatClient.FromOpenAIStreamingResponseUpdatesAsync(Throw.IfNull(responseUpdates), options, cancellationToken);
+        OpenAIResponsesChatClient.FromOpenAIStreamingResponseUpdatesAsync(Throw.IfNull(responseUpdates), options, cancellationToken: cancellationToken);
 
     /// <summary>Creates an OpenAI <see cref="OpenAIResponse"/> from a <see cref="ChatResponse"/>.</summary>
     /// <param name="response">The response to convert.</param>