Update AsOpenAIResponseItems to roundtrip User AIContent ResponseItems (dotnet#6931)

* Update AsOpenAIResponseItems to roundtrip User AIContent ResponseItems

For Assistant and Tool messages we're directly roundtripping RawRepresentations that are ResponseItems, but not for User messages. Fix that.

* Ensure ordering of AIContent-to-ResponseItem mapping

Previously a ResponseItem between two TextContents, for example, would end up being yielded before the text content that came before it. Instead, yield a response item for each group between directly-mapped items.

Also fix missing RawRepresentation on McpServerToolApprovalResponseContent.

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs

@@ -202,7 +202,7 @@ internal static IEnumerable<ChatMessage> ToChatMessages(IEnumerable<ResponseItem
                     break;
 
                 case McpToolCallApprovalResponseItem mtcari:
-                    message.Contents.Add(new McpServerToolApprovalResponseContent(mtcari.ApprovalRequestId, mtcari.Approved));
+                    message.Contents.Add(new McpServerToolApprovalResponseContent(mtcari.ApprovalRequestId, mtcari.Approved) { RawRepresentation = mtcari });
                     break;
 
                 case FunctionCallOutputResponseItem functionCallOutputItem:
@@ -663,55 +663,86 @@ internal static IEnumerable<ResponseItem> ToOpenAIResponseItems(IEnumerable<Chat
 
             if (input.Role == ChatRole.User)
             {
-                bool handleEmptyMessage = true; // MCP approval responses (and future cases) yield an item rather than adding a part and we don't want to return an empty user message in that case.
-                List<ResponseContentPart> parts = [];
+                // Some AIContent items may map to ResponseItems directly. Others map to ResponseContentParts that need to be grouped together.
+                // In order to preserve ordering, we yield ResponseItems as we find them, grouping ResponseContentParts between those yielded
+                // items together into their own yielded item.
+
+                List<ResponseContentPart>? parts = null;
+                bool responseItemYielded = false;
+
                 foreach (AIContent item in input.Contents)
                 {
+                    // Items that directly map to a ResponseItem.
+                    ResponseItem? directItem = item switch
+                    {
+                        { RawRepresentation: ResponseItem rawRep } => rawRep,
+                        McpServerToolApprovalResponseContent mcpResp => ResponseItem.CreateMcpApprovalResponseItem(mcpResp.Id, mcpResp.Approved),
+                        _ => null
+                    };
+
+                    if (directItem is not null)
+                    {
+                        // Yield any parts already accumulated.
+                        if (parts is not null)
+                        {
+                            yield return ResponseItem.CreateUserMessageItem(parts);
+                            parts = null;
+                        }
+
+                        // Now yield the directly mapped item.
+                        yield return directItem;
+
+                        responseItemYielded = true;
+                        continue;
+                    }
+
+                    // Items that map into ResponseContentParts and are grouped.
                     switch (item)
                     {
                         case AIContent when item.RawRepresentation is ResponseContentPart rawRep:
-                            parts.Add(rawRep);
+                            (parts ??= []).Add(rawRep);
                             break;
 
                         case TextContent textContent:
-                            parts.Add(ResponseContentPart.CreateInputTextPart(textContent.Text));
+                            (parts ??= []).Add(ResponseContentPart.CreateInputTextPart(textContent.Text));
                             break;
 
                         case UriContent uriContent when uriContent.HasTopLevelMediaType("image"):
-                            parts.Add(ResponseContentPart.CreateInputImagePart(uriContent.Uri));
+                            (parts ??= []).Add(ResponseContentPart.CreateInputImagePart(uriContent.Uri));
                             break;
 
                         case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
-                            parts.Add(ResponseContentPart.CreateInputImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
+                            (parts ??= []).Add(ResponseContentPart.CreateInputImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
                             break;
 
                         case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
-                            parts.Add(ResponseContentPart.CreateInputFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, dataContent.Name ?? $"{Guid.NewGuid():N}.pdf"));
+                            (parts ??= []).Add(ResponseContentPart.CreateInputFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, dataContent.Name ?? $"{Guid.NewGuid():N}.pdf"));
                             break;
 
                         case HostedFileContent fileContent:
-                            parts.Add(ResponseContentPart.CreateInputFilePart(fileContent.FileId));
+                            (parts ??= []).Add(ResponseContentPart.CreateInputFilePart(fileContent.FileId));
                             break;
 
                         case ErrorContent errorContent when errorContent.ErrorCode == nameof(ResponseContentPartKind.Refusal):
-                            parts.Add(ResponseContentPart.CreateRefusalPart(errorContent.Message));
-                            break;
-
-                        case McpServerToolApprovalResponseContent mcpApprovalResponseContent:
-                            handleEmptyMessage = false;
-                            yield return ResponseItem.CreateMcpApprovalResponseItem(mcpApprovalResponseContent.Id, mcpApprovalResponseContent.Approved);
+                            (parts ??= []).Add(ResponseContentPart.CreateRefusalPart(errorContent.Message));
                             break;
                     }
                 }
 
-                if (parts.Count == 0 && handleEmptyMessage)
+                // If we haven't accumulated any parts nor have we yielded any items, manufacture an empty input text part
+                // to guarantee that every user message results in at least one ResponseItem.
+                if (parts is null && !responseItemYielded)
                 {
+                    parts = [];
                     parts.Add(ResponseContentPart.CreateInputTextPart(string.Empty));
+                    responseItemYielded = true;
                 }
 
-                if (parts.Count > 0)
+                // Final yield of any accumulated parts.
+                if (parts is not null)
                 {
                     yield return ResponseItem.CreateUserMessageItem(parts);
+                    parts = null;
                 }
 
                 continue;

test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIConversionTests.cs

@@ -275,6 +275,43 @@ public void AsOpenAIResponseItems_ProducesExpectedOutput()
         Assert.Equal("The answer is 42.", Assert.Single(m5.Content).Text);
     }
 
+    [Fact]
+    public void AsOpenAIResponseItems_RoundtripsRawRepresentation()
+    {
+        List<ChatMessage> messages =
+        [
+            new(ChatRole.User,
+            [
+                new TextContent("Hello, "),
+                new AIContent { RawRepresentation = ResponseItem.CreateWebSearchCallItem() },
+                new AIContent { RawRepresentation = ResponseItem.CreateReferenceItem("123") },
+                new TextContent("World"),
+                new TextContent("!"),
+            ]),
+            new(ChatRole.Assistant,
+            [
+                new TextContent("Hi!"),
+                new AIContent { RawRepresentation = ResponseItem.CreateReasoningItem("text") },
+            ]),
+            new(ChatRole.User,
+            [
+                new AIContent { RawRepresentation = ResponseItem.CreateSystemMessageItem("test") },
+            ]),
+        ];
+
+        var items = messages.AsOpenAIResponseItems().ToArray();
+
+        Assert.Equal(7, items.Length);
+        Assert.Equal("Hello, ", ((MessageResponseItem)items[0]).Content[0].Text);
+        Assert.Same(messages[0].Contents[1].RawRepresentation, items[1]);
+        Assert.Same(messages[0].Contents[2].RawRepresentation, items[2]);
+        Assert.Equal("World", ((MessageResponseItem)items[3]).Content[0].Text);
+        Assert.Equal("!", ((MessageResponseItem)items[3]).Content[1].Text);
+        Assert.Equal("Hi!", ((MessageResponseItem)items[4]).Content[0].Text);
+        Assert.Same(messages[1].Contents[1].RawRepresentation, items[5]);
+        Assert.Same(messages[2].Contents[0].RawRepresentation, items[6]);
+    }
+
     [Fact]
     public void AsChatResponse_ConvertsOpenAIChatCompletion()
     {