Added Feature to pass document to Mistral AI's CHAT Models (#10921)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, by providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

I implemented the feature to ask questions on a document provided to
Chat models as content, which was recently added to MistralAI models. It
is supported by all the models.

https://mistral.ai/news/mistral-ocr

```
curl https://api.mistral.ai/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer ${MISTRAL_API_KEY}" \
  -d '{
    "model": "mistral-small-latest",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "what is the last sentence in the document"
          },
          {
            "type": "document_url",
            "document_url": "https://arxiv.org/pdf/1805.04770"
          }
        ]
      }
    ],
    "document_image_limit": 8,
    "document_page_limit": 64
  }'
```

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com>

Author: N-E-W-T-O-N

dotnet/src/Connectors/Connectors.MistralAI.UnitTests/Client/MistralClientTests.cs

@@ -206,6 +206,8 @@ public async Task ValidateChatMessageRequestWithToolsAsync()
         Assert.NotNull(request);
         var chatRequest = JsonSerializer.Deserialize<ChatCompletionRequest>(request);
         Assert.NotNull(chatRequest);
+        Assert.Null(chatRequest.DocumentPageLimit);
+        Assert.Null(chatRequest.DocumentImageLimit);
         Assert.Equal("auto", chatRequest.ToolChoice);
         Assert.NotNull(chatRequest.Tools);
         Assert.Single(chatRequest.Tools);
@@ -622,6 +624,153 @@ public void ValidateCloneMistralAIPromptExecutionSettings()
         Assert.Equal(settings.ResponseFormat, clonedMistralAISettings.ResponseFormat);
     }
 
+    [Fact]
+    public void ToMistralChatMessagesWithArrayOfByteBinaryContentShouldThrow()
+    {
+        // Arrange
+        using var httpClient = new HttpClient();
+        var client = new MistralClient("mistral-large-latest", httpClient, "key");
+        var chatMessage = new ChatMessageContent()
+        {
+            Role = AuthorRole.User,
+            Items =
+            [
+                new BinaryContent(data: new byte[] { 1, 2, 3 }, mimeType: "application/pdf")
+            ],
+        };
+
+        // Act
+        // Assert
+        Assert.Throws<NotSupportedException>(() => client.ToMistralChatMessages(chatMessage, default));
+    }
+
+    [Fact]
+    public void ToMistralChatMessagesWithBase64BinaryContentShouldThrow()
+    {
+        // Arrange
+        using var httpClient = new HttpClient();
+        var client = new MistralClient("mistral-large-latest", httpClient, "key");
+        var chatMessage = new ChatMessageContent()
+        {
+            Role = AuthorRole.User,
+            Items =
+            [
+                new BinaryContent(dataUri: "data:application/pdf:base64,sdfghjyswedfghjjhertgiutdgbg")
+            ],
+        };
+
+        // Act
+        // Assert
+        Assert.Throws<NotSupportedException>(() => client.ToMistralChatMessages(chatMessage, default));
+    }
+
+    [Fact]
+    public void ValidateToMistralChatMessagesWithUrlBinaryContent()
+    {
+        // Arrange
+        using var httpClient = new HttpClient();
+        var client = new MistralClient("mistral-large-latest", httpClient, "key");
+        var chatMessage = new ChatMessageContent()
+        {
+            Role = AuthorRole.User,
+            Items =
+            [
+                new BinaryContent(new Uri("https://arxiv.org/pdf/1805.04770"))
+            ],
+        };
+
+        // Act
+        var message = client.ToMistralChatMessages(chatMessage, default);
+        var contents = message[0].Content as List<ContentChunk>;
+        var content = contents![0] as DocumentUrlChunk;
+
+        // Assert
+        Assert.NotNull(message);
+        Assert.Single(message);
+        Assert.IsType<MistralChatMessage>(message[0]);
+        Assert.Equal("user", message[0].Role);
+
+        Assert.IsType<List<ContentChunk>>(message[0].Content);
+        Assert.NotNull(contents);
+        Assert.Single(contents);
+
+        Assert.IsType<DocumentUrlChunk>(content);
+        Assert.NotNull(content);
+        Assert.Equal("https://arxiv.org/pdf/1805.04770", content.DocumentUrl);
+        Assert.Equal("document_url", content.Type);
+    }
+
+    [Fact]
+    public async Task ValidateToMistralChatMessagesWithDocumentRequestAsync()
+    {
+        // Arrange
+        var client = this.CreateMistralClient("mistral-small-latest", "https://api.mistral.ai/v1/chat/completions", "chat_completions_response_with_document.json");
+
+        var chatHistory = new ChatHistory
+        {
+            new ChatMessageContent(
+                AuthorRole.User,
+                [
+                    new TextContent("Summarize the document for me."),
+                    new BinaryContent(new Uri("https://arxiv.org/pdf/1805.04770"))
+                ]),
+        };
+
+        // Act
+        var executionSettings = new MistralAIPromptExecutionSettings { DocumentPageLimit = 64, DocumentImageLimit = 8 };
+        await client.GetChatMessageContentsAsync(chatHistory, default, executionSettings);
+        var request = this.DelegatingHandler!.RequestContent;
+
+        // Assert
+        Assert.NotNull(request);
+        var chatRequest = JsonSerializer.Deserialize<ChatCompletionRequest>(request);
+        Assert.NotNull(chatRequest);
+        Assert.Equal("mistral-small-latest", chatRequest.Model);
+        Assert.Single(chatRequest.Messages);
+        Assert.Equal("user", chatRequest.Messages[0].Role);
+        Assert.NotNull(chatRequest.Messages[0].Content);
+        Assert.Equal(64, chatRequest.DocumentPageLimit);
+        Assert.Equal(8, chatRequest.DocumentImageLimit);
+
+        // Assert
+        var content = JsonSerializer.Serialize(chatRequest.Messages[0].Content);
+        string json = """[{"text":"Summarize the document for me.","type":"text"},{"document_url":"https://arxiv.org/pdf/1805.04770","type":"document_url"}]""";
+        Assert.Equal(json, content);
+    }
+
+    [Fact]
+    public async Task ValidateToMistralChatMessagesWithDocumentResponseAsync()
+    {
+        // Arrange
+        var client = this.CreateMistralClient("mistral-small-latest", "https://api.mistral.ai/v1/chat/completions", "chat_completions_response_with_document.json");
+
+        var chatHistory = new ChatHistory
+        {
+            new ChatMessageContent(
+                AuthorRole.User,
+                [
+                    new TextContent("Summarize the document for me."),
+                    new BinaryContent(new Uri("https://arxiv.org/pdf/1805.04770"))
+                ]),
+        };
+
+        // Act
+        var executionSettings = new MistralAIPromptExecutionSettings { DocumentPageLimit = 64, DocumentImageLimit = 8 };
+        var response = await client.GetChatMessageContentsAsync(chatHistory, default, executionSettings);
+
+        // Assert
+        Assert.NotNull(response);
+        Assert.Single(response);
+        Assert.Contains("The document titled \"Born-Again Neural Networks\"", response[0].Content);
+        Assert.Equal("mistral-small-latest", response[0].ModelId);
+        Assert.Equal(AuthorRole.Assistant, response[0].Role);
+        Assert.NotNull(response[0].Metadata);
+        Assert.Equal(7, response[0].Metadata?.Count);
+        Assert.NotNull(response[0].Metadata?["Usage"]);
+        Assert.NotNull(response[0].InnerContent);
+        Assert.IsType<MistralChatChoice>(response[0].InnerContent);
+    }
+
     public sealed class WeatherPlugin
     {
         [KernelFunction]

dotnet/src/Connectors/Connectors.MistralAI.UnitTests/TestData/chat_completions_response_with_document.json

@@ -0,0 +1,22 @@
+{
+  "id": "cfa5bc963e1640ebbd6c25f9671e5398",
+  "object": "chat.completion",
+  "created": 1741974042,
+  "model": "mistral-small-latest",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "tool_calls": null,
+        "content": "The document titled \"Born-Again Neural Networks\" explores the concept of Knowledge Distillation."
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 12999,
+    "total_tokens": 13488,
+    "completion_tokens": 489
+  }
+}

dotnet/src/Connectors/Connectors.MistralAI/Client/ChatCompletionRequest.cs

@@ -60,6 +60,14 @@ internal sealed class ChatCompletionRequest
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public IList<string>? Stop { get; set; }
 
+    [JsonPropertyName("document_image_limit")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? DocumentImageLimit { get; set; }
+
+    [JsonPropertyName("document_page_limit")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? DocumentPageLimit { get; set; }
+
     /// <summary>
     /// Construct an instance of <see cref="ChatCompletionRequest"/>.
     /// </summary>

dotnet/src/Connectors/Connectors.MistralAI/Client/ContentChunk.cs

@@ -6,6 +6,7 @@ namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client;
 
 [JsonDerivedType(typeof(TextChunk))]
 [JsonDerivedType(typeof(ImageUrlChunk))]
+[JsonDerivedType(typeof(DocumentUrlChunk))]
 internal abstract class ContentChunk(ContentChunkType type)
 {
     [JsonPropertyName("type")]

dotnet/src/Connectors/Connectors.MistralAI/Client/ContentChunkType.cs

@@ -12,6 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client;
 
     public static ContentChunkType ImageUrl { get; } = new("image_url");
 
+    public static ContentChunkType DocumentUrl { get; } = new("document_url");
+
     public string Type { get; }
 
     /// <summary>

dotnet/src/Connectors/Connectors.MistralAI/Client/DocumentUrlChunk.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client;
+
+internal class DocumentUrlChunk(string documentUrl) : ContentChunk(ContentChunkType.DocumentUrl)
+{
+    [JsonPropertyName("document_url")]
+    public string DocumentUrl { get; set; } = documentUrl;
+}

dotnet/src/Connectors/Connectors.MistralAI/Client/MistralClient.cs

@@ -698,6 +698,8 @@ private ChatCompletionRequest CreateChatCompletionRequest(string modelId, bool s
             FrequencyPenalty = executionSettings.FrequencyPenalty,
             PresencePenalty = executionSettings.PresencePenalty,
             Stop = executionSettings.Stop,
+            DocumentImageLimit = executionSettings.DocumentImageLimit,
+            DocumentPageLimit = executionSettings.DocumentPageLimit
         };
 
         executionSettings.ToolCallBehavior?.ConfigureRequest(kernel, request);
@@ -737,10 +739,12 @@ internal List<MistralChatMessage> ToMistralChatMessages(ChatMessageContent chatM
                 };
                 toolCalls.Add(callRequest.Id, toolCall);
             }
+
             if (toolCalls.Count > 0)
             {
                 message.ToolCalls = [.. toolCalls.Values];
             }
+
             return [message];
         }
 
@@ -764,12 +768,9 @@ internal List<MistralChatMessage> ToMistralChatMessages(ChatMessageContent chatM
                     ToolCallId = resultContent.CallId
                 });
             }
-            if (messages is not null)
-            {
-                return messages;
-            }
 
-            throw new NotSupportedException("No function result provided in the tool message.");
+            return messages
+                ?? throw new NotSupportedException("No function result provided in the tool message.");
         }
 
         if (chatMessage.Items.Count == 1 && chatMessage.Items[0] is TextContent text)
@@ -783,22 +784,31 @@ internal List<MistralChatMessage> ToMistralChatMessages(ChatMessageContent chatM
             if (item is TextContent textContent && !string.IsNullOrEmpty(textContent.Text))
             {
                 content.Add(new TextChunk(textContent.Text!));
+                continue;
             }
-            else if (item is ImageContent imageContent)
+
+            if (item is ImageContent imageContent)
             {
                 if (imageContent.Uri is not null)
                 {
                     content.Add(new ImageUrlChunk(imageContent.Uri.ToString()));
+                    continue;
                 }
-                else if (imageContent.DataUri is not null)
+
+                if (imageContent.DataUri is not null)
                 {
                     content.Add(new ImageUrlChunk(imageContent.DataUri));
+                    continue;
                 }
             }
-            else
+
+            if (item is BinaryContent binaryContent && binaryContent.Uri is not null)
             {
-                throw new NotSupportedException("Invalid message content, only text and image url are supported.");
+                content.Add(new DocumentUrlChunk(binaryContent.Uri.ToString()));
+                continue;
             }
+
+            throw new NotSupportedException("Invalid message content, only text, image url and document url are supported.");
         }
 
         return [new MistralChatMessage(chatMessage.Role.ToString(), content)];

dotnet/src/Connectors/Connectors.MistralAI/MistralAIPromptExecutionSettings.cs

@@ -240,6 +240,38 @@ public double? FrequencyPenalty
         }
     }
 
+    /// <summary>
+    /// Limit Image OCR in document
+    /// </summary>
+    [JsonPropertyName("document_image_limit")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? DocumentImageLimit
+    {
+        get => this._documentImageLimit;
+
+        set
+        {
+            this.ThrowIfFrozen();
+            this._documentImageLimit = value;
+        }
+    }
+
+    /// <summary>
+    /// Limit Pages upto which OCR will be done
+    /// </summary>
+    [JsonPropertyName("document_page_limit")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? DocumentPageLimit
+    {
+        get => this._documentPageLimit;
+
+        set
+        {
+            this.ThrowIfFrozen();
+            this._documentPageLimit = value;
+        }
+    }
+
     /// <inheritdoc/>
     public override void Freeze()
     {
@@ -313,6 +345,8 @@ public static MistralAIPromptExecutionSettings FromExecutionSettings(PromptExecu
     private double? _presencePenalty;
     private double? _frequencyPenalty;
     private IList<string>? _stop;
+    private int? _documentImageLimit;
+    private int? _documentPageLimit;
 
     #endregion
 }