Update to 1.38 of the otel genai standard convention

src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs

@@ -16,14 +16,16 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Shared.Diagnostics;
 
+#pragma warning disable CA1307 // Specify StringComparison for clarity
+#pragma warning disable CA1308 // Normalize strings to uppercase
 #pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter
 #pragma warning disable SA1113 // Comma should be on the same line as previous parameter
 
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
 /// <remarks>
-/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.37, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
+/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.38, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
 /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
 /// </remarks>
 public sealed partial class OpenTelemetryChatClient : DelegatingChatClient
@@ -273,20 +275,35 @@ internal static string SerializeChatMessages(
                         });
                         break;
 
-                    // These are non-standard and are using the "generic" non-text part that provides an extensibility mechanism:
-
-                    case UriContent uc:
-                        m.Parts.Add(new OtelGenericPart { Type = "image", Content = uc.Uri.ToString() });
+                    case DataContent dc:
+                        m.Parts.Add(new OtelBlobPart
+                        {
+                            Content = dc.Base64Data.ToString(),
+                            MimeType = dc.MediaType,
+                            Modality = DeriveModalityFromMediaType(dc.MediaType),
+                        });
                         break;
 
-                    case DataContent dc:
-                        m.Parts.Add(new OtelGenericPart { Type = "image", Content = dc.Uri });
+                    case UriContent uc:
+                        m.Parts.Add(new OtelUriPart
+                        {
+                            Uri = uc.Uri.AbsoluteUri,
+                            MimeType = uc.MediaType,
+                            Modality = DeriveModalityFromMediaType(uc.MediaType),
+                        });
                         break;
 
                     case HostedFileContent fc:
-                        m.Parts.Add(new OtelGenericPart { Type = "file", Content = fc.FileId });
+                        m.Parts.Add(new OtelFilePart
+                        {
+                            FileId = fc.FileId,
+                            MimeType = fc.MediaType,
+                            Modality = DeriveModalityFromMediaType(fc.MediaType),
+                        });
                         break;
 
+                    // These are non-standard and are using the "generic" non-text part that provides an extensibility mechanism:
+
                     case HostedVectorStoreContent vsc:
                         m.Parts.Add(new OtelGenericPart { Type = "vector_store", Content = vsc.VectorStoreId });
                         break;
@@ -329,6 +346,25 @@ internal static string SerializeChatMessages(
         return JsonSerializer.Serialize(output, _defaultOptions.GetTypeInfo(typeof(IList<object>)));
     }
 
+    private static string? DeriveModalityFromMediaType(string? mediaType)
+    {
+        if (mediaType is not null)
+        {
+            int pos = mediaType.IndexOf('/');
+            if (pos >= 0)
+            {
+                ReadOnlySpan<char> topLevel = mediaType.AsSpan(0, pos);
+                return
+                    topLevel.Equals("image", StringComparison.OrdinalIgnoreCase) ? "image" :
+                    topLevel.Equals("audio", StringComparison.OrdinalIgnoreCase) ? "audio" :
+                    topLevel.Equals("video", StringComparison.OrdinalIgnoreCase) ? "video" :
+                    null;
+            }
+        }
+
+        return null;
+    }
+
     /// <summary>Creates an activity for a chat request, or returns <see langword="null"/> if not enabled.</summary>
     private Activity? CreateAndConfigureActivity(ChatOptions? options)
     {
@@ -607,6 +643,30 @@ private sealed class OtelGenericPart
         public object? Content { get; set; } // should be a string when Type == "text"
     }
 
+    private sealed class OtelBlobPart
+    {
+        public string Type { get; set; } = "blob";
+        public string? Content { get; set; } // base64-encoded binary data
+        public string? MimeType { get; set; }
+        public string? Modality { get; set; }
+    }
+
+    private sealed class OtelUriPart
+    {
+        public string Type { get; set; } = "uri";
+        public string? Uri { get; set; }
+        public string? MimeType { get; set; }
+        public string? Modality { get; set; }
+    }
+
+    private sealed class OtelFilePart
+    {
+        public string Type { get; set; } = "file";
+        public string? FileId { get; set; }
+        public string? MimeType { get; set; }
+        public string? Modality { get; set; }
+    }
+
     private sealed class OtelToolCallRequestPart
     {
         public string Type { get; set; } = "tool_call";
@@ -653,6 +713,9 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(IList<object>))]
     [JsonSerializable(typeof(OtelMessage))]
     [JsonSerializable(typeof(OtelGenericPart))]
+    [JsonSerializable(typeof(OtelBlobPart))]
+    [JsonSerializable(typeof(OtelUriPart))]
+    [JsonSerializable(typeof(OtelFilePart))]
     [JsonSerializable(typeof(OtelToolCallRequestPart))]
     [JsonSerializable(typeof(OtelToolCallResponsePart))]
     [JsonSerializable(typeof(IEnumerable<OtelFunction>))]

src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs

@@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a delegating image generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
 /// <remarks>
-/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.37, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
+/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.38, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
 /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
 /// </remarks>
 [Experimental("MEAI001")]

src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs

@@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
 /// <remarks>
-/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.37, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
+/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.38, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
 /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
 /// </remarks>
 /// <typeparam name="TInput">The type of input used to produce embeddings.</typeparam>

src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs

@@ -21,7 +21,7 @@ namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a delegating speech-to-text client that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
 /// <remarks>
-/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.37, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
+/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.38, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
 /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
 /// </remarks>
 [Experimental("MEAI001")]

test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs

@@ -333,6 +333,192 @@ async static IAsyncEnumerable<ChatResponseUpdate> CallbackAsync(
         }
     }
 
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task AllOfficialOtelContentPartTypes_SerializedCorrectly(bool streaming)
+    {
+        var sourceName = Guid.NewGuid().ToString();
+        var activities = new List<Activity>();
+        using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
+            .AddSource(sourceName)
+            .AddInMemoryExporter(activities)
+            .Build();
+
+        using var innerClient = new TestChatClient
+        {
+            GetResponseAsyncCallback = async (messages, options, cancellationToken) =>
+            {
+                await Task.Yield();
+                return new ChatResponse(new ChatMessage(ChatRole.Assistant,
+                [
+                    new TextContent("Assistant response text"),
+                    new TextReasoningContent("This is reasoning"),
+                    new FunctionCallContent("call-123", "GetWeather", new Dictionary<string, object?> { ["location"] = "Seattle" }),
+                    new FunctionResultContent("call-123", "72°F and sunny"),
+                    new DataContent(Convert.FromBase64String("aGVsbG8gd29ybGQ="), "image/png"),
+                    new UriContent(new Uri("https://example.com/image.jpg"), "image/jpeg"),
+                    new HostedFileContent("file-abc123"),
+                ]));
+            },
+            GetStreamingResponseAsyncCallback = CallbackAsync,
+        };
+
+        async static IAsyncEnumerable<ChatResponseUpdate> CallbackAsync(
+            IEnumerable<ChatMessage> messages, ChatOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            await Task.Yield();
+            yield return new(ChatRole.Assistant, "Assistant response text");
+            yield return new() { Contents = [new TextReasoningContent("This is reasoning")] };
+            yield return new() { Contents = [new FunctionCallContent("call-123", "GetWeather", new Dictionary<string, object?> { ["location"] = "Seattle" })] };
+            yield return new() { Contents = [new FunctionResultContent("call-123", "72°F and sunny")] };
+            yield return new() { Contents = [new DataContent(Convert.FromBase64String("aGVsbG8gd29ybGQ="), "image/png")] };
+            yield return new() { Contents = [new UriContent(new Uri("https://example.com/image.jpg"), "image/jpeg")] };
+            yield return new() { Contents = [new HostedFileContent("file-abc123")] };
+        }
+
+        using var chatClient = innerClient
+            .AsBuilder()
+            .UseOpenTelemetry(null, sourceName, configure: instance =>
+            {
+                instance.EnableSensitiveData = true;
+                instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options;
+            })
+            .Build();
+
+        List<ChatMessage> messages =
+        [
+            new(ChatRole.User,
+            [
+                new TextContent("User request text"),
+                new TextReasoningContent("User reasoning"),
+                new DataContent(Convert.FromBase64String("ZGF0YSBjb250ZW50"), "audio/mp3"),
+                new UriContent(new Uri("https://example.com/video.mp4"), "video/mp4"),
+                new HostedFileContent("file-xyz789"),
+            ]),
+            new(ChatRole.Assistant, [new FunctionCallContent("call-456", "SearchFiles")]),
+            new(ChatRole.Tool, [new FunctionResultContent("call-456", "Found 3 files")]),
+        ];
+
+        if (streaming)
+        {
+            await foreach (var update in chatClient.GetStreamingResponseAsync(messages))
+            {
+                await Task.Yield();
+            }
+        }
+        else
+        {
+            await chatClient.GetResponseAsync(messages);
+        }
+
+        var activity = Assert.Single(activities);
+        Assert.NotNull(activity);
+
+        var inputMessages = activity.Tags.First(kvp => kvp.Key == "gen_ai.input.messages").Value;
+        Assert.Equal(ReplaceWhitespace("""
+            [
+              {
+                "role": "user",
+                "parts": [
+                  {
+                    "type": "text",
+                    "content": "User request text"
+                  },
+                  {
+                    "type": "reasoning",
+                    "content": "User reasoning"
+                  },
+                  {
+                    "type": "blob",
+                    "content": "ZGF0YSBjb250ZW50",
+                    "mime_type": "audio/mp3",
+                    "modality": "audio"
+                  },
+                  {
+                    "type": "uri",
+                    "uri": "https://example.com/video.mp4",
+                    "mime_type": "video/mp4",
+                    "modality": "video"
+                  },
+                  {
+                    "type": "file",
+                    "file_id": "file-xyz789"
+                  }
+                ]
+              },
+              {
+                "role": "assistant",
+                "parts": [
+                  {
+                    "type": "tool_call",
+                    "id": "call-456",
+                    "name": "SearchFiles"
+                  }
+                ]
+              },
+              {
+                "role": "tool",
+                "parts": [
+                  {
+                    "type": "tool_call_response",
+                    "id": "call-456",
+                    "response": "Found 3 files"
+                  }
+                ]
+              }
+            ]
+            """), ReplaceWhitespace(inputMessages));
+
+        var outputMessages = activity.Tags.First(kvp => kvp.Key == "gen_ai.output.messages").Value;
+        Assert.Equal(ReplaceWhitespace("""
+            [
+              {
+                "role": "assistant",
+                "parts": [
+                  {
+                    "type": "text",
+                    "content": "Assistant response text"
+                  },
+                  {
+                    "type": "reasoning",
+                    "content": "This is reasoning"
+                  },
+                  {
+                    "type": "tool_call",
+                    "id": "call-123",
+                    "name": "GetWeather",
+                    "arguments": {
+                      "location": "Seattle"
+                    }
+                  },
+                  {
+                    "type": "tool_call_response",
+                    "id": "call-123",
+                    "response": "72°F and sunny"
+                  },
+                  {
+                    "type": "blob",
+                    "content": "aGVsbG8gd29ybGQ=",
+                    "mime_type": "image/png",
+                    "modality": "image"
+                  },
+                  {
+                    "type": "uri",
+                    "uri": "https://example.com/image.jpg",
+                    "mime_type": "image/jpeg",
+                    "modality": "image"
+                  },
+                  {
+                    "type": "file",
+                    "file_id": "file-abc123"
+                  }
+                ]
+              }
+            ]
+            """), ReplaceWhitespace(outputMessages));
+    }
+
     [Fact]
     public async Task UnknownContentTypes_Ignored()
     {

test/Libraries/Microsoft.Extensions.AI.Tests/Image/OpenTelemetryImageGeneratorTests.cs

@@ -125,8 +125,10 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData)
                         "content": "This is the input prompt."
                       },
                       {
-                        "type": "image",
-                        "content": "http://example/input.png"
+                        "type": "uri",
+                        "uri": "http://example/input.png",
+                        "mime_type": "image/png",
+                        "modality": "image"
                       }
                     ]
                   }
@@ -139,12 +141,16 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData)
                     "role": "assistant",
                     "parts": [
                       {
-                        "type": "image",
-                        "content": "http://example/output.png"
+                        "type": "uri",
+                        "uri": "http://example/output.png",
+                        "mime_type": "image/png",
+                        "modality": "image"
                       },
                       {
-                        "type": "image",
-                        "content": "data:image/png;base64,AQIDBA=="
+                        "type": "blob",
+                        "content": "AQIDBA==",
+                        "mime_type": "image/png",
+                        "modality": "image"
                       }
                     ]
                   }