forked from microsoft/semantic-kernel
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net Agents - Fix interoperability of assistant generated content (mi…
…crosoft#7415) ### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> The `OpenAIAssistantAgent` produced messages that resulted in an exception when processed by the chat-completion service. - Code-Interpreter Output: `Tool` role implies preceeding function-call - `FileReferenceContent`: Image output being isolated from accompanying text Fixed: microsoft#7414 ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> - Changed role of code-interpreter output to `assistant` - Merged message-content into a single `ChatMessageContent` object ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄
- Loading branch information
Showing
3 changed files
with
254 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| using System.Text; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Agents; | ||
| using Microsoft.SemanticKernel.Agents.OpenAI; | ||
| using Microsoft.SemanticKernel.ChatCompletion; | ||
| using Microsoft.SemanticKernel.Connectors.OpenAI; | ||
| using Resources; | ||
|
|
||
| namespace Agents; | ||
|
|
||
| /// <summary> | ||
| /// Demonstrate <see cref="ChatCompletionAgent"/> agent interacts with | ||
| /// <see cref="OpenAIAssistantAgent"/> when it produces file output. | ||
| /// </summary> | ||
| public class MixedChat_Files(ITestOutputHelper output) : BaseTest(output) | ||
| { | ||
| /// <summary> | ||
| /// Target OpenAI services. | ||
| /// </summary> | ||
| protected override bool ForceOpenAI => true; | ||
|
|
||
| private const string SummaryInstructions = "Summarize the entire conversation for the user in natural language."; | ||
|
|
||
| [Fact] | ||
| public async Task AnalyzeFileAndGenerateReportAsync() | ||
| { | ||
| OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey); | ||
|
|
||
| OpenAIFileReference uploadFile = | ||
| await fileService.UploadContentAsync( | ||
| new BinaryContent(await EmbeddedResource.ReadAllAsync("30-user-context.txt"), mimeType: "text/plain"), | ||
| new OpenAIFileUploadExecutionSettings("30-user-context.txt", OpenAIFilePurpose.Assistants)); | ||
|
|
||
| Console.WriteLine(this.ApiKey); | ||
|
|
||
| // Define the agents | ||
| OpenAIAssistantAgent analystAgent = | ||
| await OpenAIAssistantAgent.CreateAsync( | ||
| kernel: new(), | ||
| config: new(this.ApiKey, this.Endpoint), | ||
| new() | ||
| { | ||
| EnableCodeInterpreter = true, // Enable code-interpreter | ||
| ModelId = this.Model, | ||
| FileIds = [uploadFile.Id] // Associate uploaded file with assistant | ||
| }); | ||
|
|
||
| ChatCompletionAgent summaryAgent = | ||
| new() | ||
| { | ||
| Instructions = SummaryInstructions, | ||
| Kernel = this.CreateKernelWithChatCompletion(), | ||
| }; | ||
|
|
||
| // Create a chat for agent interaction. | ||
| AgentGroupChat chat = new(); | ||
|
|
||
| // Respond to user input | ||
| try | ||
| { | ||
| await InvokeAgentAsync( | ||
| analystAgent, | ||
| """ | ||
| Create a tab delimited file report of the ordered (descending) frequency distribution | ||
| of words in the file '30-user-context.txt' for any words used more than once. | ||
| """); | ||
| await InvokeAgentAsync(summaryAgent); | ||
| } | ||
| finally | ||
| { | ||
| await analystAgent.DeleteAsync(); | ||
| await fileService.DeleteFileAsync(uploadFile.Id); | ||
| } | ||
|
|
||
| // Local function to invoke agent and display the conversation messages. | ||
| async Task InvokeAgentAsync(Agent agent, string? input = null) | ||
| { | ||
| if (!string.IsNullOrWhiteSpace(input)) | ||
| { | ||
| chat.AddChatMessage(new(AuthorRole.User, input)); | ||
| Console.WriteLine($"# {AuthorRole.User}: '{input}'"); | ||
| } | ||
|
|
||
| await foreach (ChatMessageContent content in chat.InvokeAsync(agent)) | ||
| { | ||
| Console.WriteLine($"\n# {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'"); | ||
|
|
||
| foreach (AnnotationContent annotation in content.Items.OfType<AnnotationContent>()) | ||
| { | ||
| Console.WriteLine($"\t* '{annotation.Quote}' => {annotation.FileId}"); | ||
| BinaryContent fileContent = await fileService.GetFileContentAsync(annotation.FileId!); | ||
| byte[] byteContent = fileContent.Data?.ToArray() ?? []; | ||
| Console.WriteLine($"\n{Encoding.Default.GetString(byteContent)}"); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Agents; | ||
| using Microsoft.SemanticKernel.Agents.OpenAI; | ||
| using Microsoft.SemanticKernel.ChatCompletion; | ||
| using Microsoft.SemanticKernel.Connectors.OpenAI; | ||
|
|
||
| namespace Agents; | ||
|
|
||
| /// <summary> | ||
| /// Demonstrate <see cref="ChatCompletionAgent"/> agent interacts with | ||
| /// <see cref="OpenAIAssistantAgent"/> when it produces image output. | ||
| /// </summary> | ||
| public class MixedChat_Images(ITestOutputHelper output) : BaseTest(output) | ||
| { | ||
| /// <summary> | ||
| /// Target OpenAI services. | ||
| /// </summary> | ||
| protected override bool ForceOpenAI => true; | ||
|
|
||
| private const string AnalystName = "Analyst"; | ||
| private const string AnalystInstructions = "Create charts as requested without explanation."; | ||
|
|
||
| private const string SummarizerName = "Summarizer"; | ||
| private const string SummarizerInstructions = "Summarize the entire conversation for the user in natural language."; | ||
|
|
||
| [Fact] | ||
| public async Task AnalyzeDataAndGenerateChartAsync() | ||
| { | ||
| OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey); | ||
|
|
||
| // Define the agents | ||
| OpenAIAssistantAgent analystAgent = | ||
| await OpenAIAssistantAgent.CreateAsync( | ||
| kernel: new(), | ||
| config: new(this.ApiKey, this.Endpoint), | ||
| new() | ||
| { | ||
| Instructions = AnalystInstructions, | ||
| Name = AnalystName, | ||
| EnableCodeInterpreter = true, | ||
| ModelId = this.Model, | ||
| }); | ||
|
|
||
| ChatCompletionAgent summaryAgent = | ||
| new() | ||
| { | ||
| Instructions = SummarizerInstructions, | ||
| Name = SummarizerName, | ||
| Kernel = this.CreateKernelWithChatCompletion(), | ||
| }; | ||
|
|
||
| // Create a chat for agent interaction. | ||
| AgentGroupChat chat = new(); | ||
|
|
||
| // Respond to user input | ||
| try | ||
| { | ||
| await InvokeAgentAsync( | ||
| analystAgent, | ||
| """ | ||
| Graph the percentage of storm events by state using a pie chart: | ||
| State, StormCount | ||
| TEXAS, 4701 | ||
| KANSAS, 3166 | ||
| IOWA, 2337 | ||
| ILLINOIS, 2022 | ||
| MISSOURI, 2016 | ||
| GEORGIA, 1983 | ||
| MINNESOTA, 1881 | ||
| WISCONSIN, 1850 | ||
| NEBRASKA, 1766 | ||
| NEW YORK, 1750 | ||
| """); | ||
|
|
||
| await InvokeAgentAsync(summaryAgent); | ||
| } | ||
| finally | ||
| { | ||
| await analystAgent.DeleteAsync(); | ||
| } | ||
|
|
||
| // Local function to invoke agent and display the conversation messages. | ||
| async Task InvokeAgentAsync(Agent agent, string? input = null) | ||
| { | ||
| if (!string.IsNullOrWhiteSpace(input)) | ||
| { | ||
| chat.AddChatMessage(new(AuthorRole.User, input)); | ||
| Console.WriteLine($"# {AuthorRole.User}: '{input}'"); | ||
| } | ||
|
|
||
| await foreach (ChatMessageContent message in chat.InvokeAsync(agent)) | ||
| { | ||
| if (!string.IsNullOrWhiteSpace(message.Content)) | ||
| { | ||
| Console.WriteLine($"\n# {message.Role} - {message.AuthorName ?? "*"}: '{message.Content}'"); | ||
| } | ||
|
|
||
| foreach (FileReferenceContent fileReference in message.Items.OfType<FileReferenceContent>()) | ||
| { | ||
| Console.WriteLine($"\t* Generated image - @{fileReference.FileId}"); | ||
| BinaryContent fileContent = await fileService.GetFileContentAsync(fileReference.FileId!); | ||
| byte[] byteContent = fileContent.Data?.ToArray() ?? []; | ||
| string filePath = Path.ChangeExtension(Path.GetTempFileName(), ".png"); | ||
| await File.WriteAllBytesAsync($"{filePath}.png", byteContent); | ||
| Console.WriteLine($"\t* Local path - {filePath}"); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters