Skip to content

Commit 06d1dca

Browse files
shyamnamboodiripadshyamnamboodiripad
authored
Add model provider information to ChatTurnDetails (#6722)
1 parent 2130f36 commit 06d1dca

File tree

9 files changed

+390
-50
lines changed

9 files changed

+390
-50
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatTurnDetails.cs‎

Lines changed: 96 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -7,76 +7,140 @@
77
// constructor syntax.
88

99
using System;
10+
using System.Text.Json.Serialization;
1011

1112
namespace Microsoft.Extensions.AI.Evaluation.Reporting;
1213

1314
/// <summary>
1415
/// A class that records details related to a particular LLM chat conversation turn involved in the execution of a
1516
/// particular <see cref="ScenarioRun"/>.
1617
/// </summary>
17-
/// <param name="latency">
18-
/// The duration between the time when the request was sent to the LLM and the time when the response was received for
19-
/// the chat conversation turn.
20-
/// </param>
21-
/// <param name="model">
22-
/// The model that was used in the creation of the response for the chat conversation turn. Can be
23-
/// <see langword="null"/> if this information was not available via <see cref="ChatResponse.ModelId"/>.
24-
/// </param>
25-
/// <param name="usage">
26-
/// Usage details for the chat conversation turn (including input and output token counts). Can be
27-
/// <see langword="null"/> if usage details were not available via <see cref="ChatResponse.Usage"/>.
28-
/// </param>
29-
/// <param name="cacheKey">
30-
/// The cache key for the cached model response for the chat conversation turn if response caching was enabled;
31-
/// <see langword="null"/> otherwise.
32-
/// </param>
33-
/// <param name="cacheHit">
34-
/// <see langword="true"/> if response caching was enabled and the model response for the chat conversation turn was
35-
/// retrieved from the cache; <see langword="false"/> if response caching was enabled and the model response was not
36-
/// retrieved from the cache; <see langword="null"/> if response caching was disabled.
37-
/// </param>
38-
public sealed class ChatTurnDetails(
39-
TimeSpan latency,
40-
string? model = null,
41-
UsageDetails? usage = null,
42-
string? cacheKey = null,
43-
bool? cacheHit = null)
18+
public sealed class ChatTurnDetails
4419
{
4520
/// <summary>
4621
/// Gets or sets the duration between the time when the request was sent to the LLM and the time when the response
4722
/// was received for the chat conversation turn.
4823
/// </summary>
49-
public TimeSpan Latency { get; set; } = latency;
24+
public TimeSpan Latency { get; set; }
5025

5126
/// <summary>
5227
/// Gets or sets the model that was used in the creation of the response for the chat conversation turn.
5328
/// </summary>
5429
/// <remarks>
5530
/// Returns <see langword="null"/> if this information was not available via <see cref="ChatResponse.ModelId"/>.
5631
/// </remarks>
57-
public string? Model { get; set; } = model;
32+
public string? Model { get; set; }
33+
34+
/// <summary>
35+
/// Gets or sets the name of the provider for the model identified by <see cref="Model"/>.
36+
/// </summary>
37+
/// <remarks>
38+
/// Returns <see langword="null"/> if this information was not available via the
39+
/// <see cref="ChatClientMetadata.ProviderName"/> property for the <see cref="IChatClient"/>.
40+
/// </remarks>
41+
public string? ModelProvider { get; set; }
5842

5943
/// <summary>
6044
/// Gets or sets usage details for the chat conversation turn (including input and output token counts).
6145
/// </summary>
6246
/// <remarks>
6347
/// Returns <see langword="null"/> if usage details were not available via <see cref="ChatResponse.Usage"/>.
6448
/// </remarks>
65-
public UsageDetails? Usage { get; set; } = usage;
49+
public UsageDetails? Usage { get; set; }
6650

6751
/// <summary>
6852
/// Gets or sets the cache key for the cached model response for the chat conversation turn.
6953
/// </summary>
7054
/// <remarks>
7155
/// Returns <see langword="null"/> if response caching was disabled.
7256
/// </remarks>
73-
public string? CacheKey { get; set; } = cacheKey;
57+
public string? CacheKey { get; set; }
7458

7559
/// <summary>
7660
/// Gets or sets a value indicating whether the model response was retrieved from the cache.
7761
/// </summary>
7862
/// <remarks>
7963
/// Returns <see langword="null"/> if response caching was disabled.
8064
/// </remarks>
81-
public bool? CacheHit { get; set; } = cacheHit;
65+
public bool? CacheHit { get; set; }
66+
67+
/// <summary>
68+
/// Initializes a new instance of the <see cref="ChatTurnDetails"/> class.
69+
/// </summary>
70+
/// <param name="latency">
71+
/// The duration between the time when the request was sent to the LLM and the time when the response was received
72+
/// for the chat conversation turn.
73+
/// </param>
74+
/// <param name="model">
75+
/// The model that was used in the creation of the response for the chat conversation turn. Can be
76+
/// <see langword="null"/> if this information was not available via <see cref="ChatResponse.ModelId"/>.
77+
/// </param>
78+
/// <param name="usage">
79+
/// Usage details for the chat conversation turn (including input and output token counts). Can be
80+
/// <see langword="null"/> if usage details were not available via <see cref="ChatResponse.Usage"/>.
81+
/// </param>
82+
/// <param name="cacheKey">
83+
/// The cache key for the cached model response for the chat conversation turn if response caching was enabled;
84+
/// <see langword="null"/> otherwise.
85+
/// </param>
86+
/// <param name="cacheHit">
87+
/// <see langword="true"/> if response caching was enabled and the model response for the chat conversation turn
88+
/// was retrieved from the cache; <see langword="false"/> if response caching was enabled and the model response
89+
/// was not retrieved from the cache; <see langword="null"/> if response caching was disabled.
90+
/// </param>
91+
public ChatTurnDetails(
92+
TimeSpan latency,
93+
string? model = null,
94+
UsageDetails? usage = null,
95+
string? cacheKey = null,
96+
bool? cacheHit = null)
97+
: this(latency, model, modelProvider: null, usage, cacheKey, cacheHit)
98+
{
99+
}
100+
101+
/// <summary>
102+
/// Initializes a new instance of the <see cref="ChatTurnDetails"/> class.
103+
/// </summary>
104+
/// <param name="latency">
105+
/// The duration between the time when the request was sent to the LLM and the time when the response was received
106+
/// for the chat conversation turn.
107+
/// </param>
108+
/// <param name="model">
109+
/// The model that was used in the creation of the response for the chat conversation turn. Can be
110+
/// <see langword="null"/> if this information was not available via <see cref="ChatResponse.ModelId"/>.
111+
/// </param>
112+
/// <param name="modelProvider">
113+
/// The name of the provider for the model identified by <paramref name="model"/>. Can be
114+
/// <see langword="null"/> if this information was not available via the
115+
/// <see cref="ChatClientMetadata.ProviderName"/> property for the <see cref="IChatClient"/>.
116+
/// </param>
117+
/// <param name="usage">
118+
/// Usage details for the chat conversation turn (including input and output token counts). Can be
119+
/// <see langword="null"/> if usage details were not available via <see cref="ChatResponse.Usage"/>.
120+
/// </param>
121+
/// <param name="cacheKey">
122+
/// The cache key for the cached model response for the chat conversation turn if response caching was enabled;
123+
/// <see langword="null"/> otherwise.
124+
/// </param>
125+
/// <param name="cacheHit">
126+
/// <see langword="true"/> if response caching was enabled and the model response for the chat conversation turn
127+
/// was retrieved from the cache; <see langword="false"/> if response caching was enabled and the model response
128+
/// was not retrieved from the cache; <see langword="null"/> if response caching was disabled.
129+
/// </param>
130+
[JsonConstructor]
131+
public ChatTurnDetails(
132+
TimeSpan latency,
133+
string? model,
134+
string? modelProvider,
135+
UsageDetails? usage = null,
136+
string? cacheKey = null,
137+
bool? cacheHit = null)
138+
{
139+
Latency = latency;
140+
Model = model;
141+
ModelProvider = modelProvider;
142+
Usage = usage;
143+
CacheKey = cacheKey;
144+
CacheHit = cacheHit;
145+
}
82146
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.json‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"Name": "Microsoft.Extensions.AI.Evaluation.Reporting, Version=9.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35",
2+
"Name": "Microsoft.Extensions.AI.Evaluation.Reporting, Version=9.9.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35",
33
"Types": [
44
{
55
"Type": "sealed class Microsoft.Extensions.AI.Evaluation.Reporting.ChatDetails",
@@ -40,15 +40,16 @@
4040
]
4141
},
4242
{
43-
// After generating the baseline, manually edit this file to remove primary constructor portion
44-
// This is needed until ICSharpCode.Decompiler adds support for primary constructors
45-
// See: https://github.com/icsharpcode/ILSpy/issues/829
4643
"Type": "sealed class Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails",
4744
"Stage": "Stable",
4845
"Methods": [
4946
{
5047
"Member": "Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails.ChatTurnDetails(System.TimeSpan latency, string? model = null, Microsoft.Extensions.AI.UsageDetails? usage = null, string? cacheKey = null, bool? cacheHit = null);",
5148
"Stage": "Stable"
49+
},
50+
{
51+
"Member": "Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails.ChatTurnDetails(System.TimeSpan latency, string? model, string? modelProvider, Microsoft.Extensions.AI.UsageDetails? usage = null, string? cacheKey = null, bool? cacheHit = null);",
52+
"Stage": "Stable"
5253
}
5354
],
5455
"Properties": [
@@ -68,6 +69,10 @@
6869
"Member": "string? Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails.Model { get; set; }",
6970
"Stage": "Stable"
7071
},
72+
{
73+
"Member": "string? Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails.ModelProvider { get; set; }",
74+
"Stage": "Stable"
75+
},
7176
{
7277
"Member": "Microsoft.Extensions.AI.UsageDetails? Microsoft.Extensions.AI.Evaluation.Reporting.ChatTurnDetails.Usage { get; set; }",
7378
"Stage": "Stable"

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ internal sealed class ResponseCachingChatClient : DistributedCachingChatClient
1414
{
1515
private readonly ChatDetails _chatDetails;
1616
private readonly ConcurrentDictionary<string, Stopwatch> _stopWatches;
17+
private readonly ChatClientMetadata? _metadata;
1718

1819
internal ResponseCachingChatClient(
1920
IChatClient originalChatClient,
@@ -23,8 +24,10 @@ internal ResponseCachingChatClient(
2324
: base(originalChatClient, cache)
2425
{
2526
CacheKeyAdditionalValues = [.. cachingKeys];
27+
2628
_chatDetails = chatDetails;
2729
_stopWatches = new ConcurrentDictionary<string, Stopwatch>();
30+
_metadata = this.GetService<ChatClientMetadata>();
2831
}
2932

3033
protected override async Task<ChatResponse?> ReadCacheAsync(string key, CancellationToken cancellationToken)
@@ -45,6 +48,7 @@ internal ResponseCachingChatClient(
4548
new ChatTurnDetails(
4649
latency: stopwatch.Elapsed,
4750
model: response.ModelId,
51+
modelProvider: _metadata?.ProviderName,
4852
usage: response.Usage,
4953
cacheKey: key,
5054
cacheHit: true));
@@ -75,6 +79,7 @@ internal ResponseCachingChatClient(
7579
new ChatTurnDetails(
7680
latency: stopwatch.Elapsed,
7781
model: response.ModelId,
82+
modelProvider: _metadata?.ProviderName,
7883
usage: response.Usage,
7984
cacheKey: key,
8085
cacheHit: true));
@@ -95,6 +100,7 @@ protected override async Task WriteCacheAsync(string key, ChatResponse value, Ca
95100
new ChatTurnDetails(
96101
latency: stopwatch.Elapsed,
97102
model: value.ModelId,
103+
modelProvider: _metadata?.ProviderName,
98104
usage: value.Usage,
99105
cacheKey: key,
100106
cacheHit: false));
@@ -117,6 +123,7 @@ protected override async Task WriteCacheStreamingAsync(
117123
new ChatTurnDetails(
118124
latency: stopwatch.Elapsed,
119125
model: response.ModelId,
126+
modelProvider: _metadata?.ProviderName,
120127
usage: response.Usage,
121128
cacheKey: key,
122129
cacheHit: false));

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/SimpleChatClient.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting;
1212
internal sealed class SimpleChatClient : DelegatingChatClient
1313
{
1414
private readonly ChatDetails _chatDetails;
15+
private readonly ChatClientMetadata? _metadata;
1516

1617
internal SimpleChatClient(IChatClient originalChatClient, ChatDetails chatDetails)
1718
: base(originalChatClient)
1819
{
1920
_chatDetails = chatDetails;
21+
_metadata = this.GetService<ChatClientMetadata>();
2022
}
2123

2224
public async override Task<ChatResponse> GetResponseAsync(
@@ -41,6 +43,7 @@ public async override Task<ChatResponse> GetResponseAsync(
4143
new ChatTurnDetails(
4244
latency: stopwatch.Elapsed,
4345
model: response.ModelId,
46+
modelProvider: _metadata?.ProviderName,
4447
usage: response.Usage));
4548
}
4649
}
@@ -78,6 +81,7 @@ public override async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseA
7881
new ChatTurnDetails(
7982
latency: stopwatch.Elapsed,
8083
model: response.ModelId,
84+
modelProvider: _metadata?.ProviderName,
8185
usage: response.Usage));
8286
}
8387
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ChatDetailsSection.tsx‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ export const ChatDetailsSection = ({ chatDetails }: { chatDetails: ChatDetails;
1515

1616
const hasCacheKey = chatDetails.turnDetails.some(turn => turn.cacheKey !== undefined);
1717
const hasCacheStatus = chatDetails.turnDetails.some(turn => turn.cacheHit !== undefined);
18-
const hasModelInfo = chatDetails.turnDetails.some(turn => turn.model !== undefined);
18+
const hasModel = chatDetails.turnDetails.some(turn => turn.model !== undefined);
19+
const hasModelProvider = chatDetails.turnDetails.some(turn => turn.modelProvider !== undefined);
1920
const hasInputTokens = chatDetails.turnDetails.some(turn => turn.usage?.inputTokenCount !== undefined);
2021
const hasOutputTokens = chatDetails.turnDetails.some(turn => turn.usage?.outputTokenCount !== undefined);
2122
const hasTotalTokens = chatDetails.turnDetails.some(turn => turn.usage?.totalTokenCount !== undefined);
@@ -42,13 +43,14 @@ export const ChatDetailsSection = ({ chatDetails }: { chatDetails: ChatDetails;
4243
{isExpanded && (
4344
<div className={classes.sectionContainer}>
4445
<div className={classes.tableContainer}>
45-
<Table>
46+
<Table className={classes.autoWidthTable}>
4647
<TableHeader>
4748
<TableRow>
4849
{hasCacheKey && <TableHeaderCell className={classes.tableHeaderCell}>Cache Key</TableHeaderCell>}
4950
{hasCacheStatus && <TableHeaderCell className={classes.tableHeaderCell}>Cache Status</TableHeaderCell>}
5051
<TableHeaderCell className={classes.tableHeaderCell}>Latency (s)</TableHeaderCell>
51-
{hasModelInfo && <TableHeaderCell className={classes.tableHeaderCell}>Model Used</TableHeaderCell>}
52+
{hasModel && <TableHeaderCell className={classes.tableHeaderCell}>Model</TableHeaderCell>}
53+
{hasModelProvider && <TableHeaderCell className={classes.tableHeaderCell}>Model Provider</TableHeaderCell>}
5254
{hasInputTokens && <TableHeaderCell className={classes.tableHeaderCell}>Input Tokens</TableHeaderCell>}
5355
{hasOutputTokens && <TableHeaderCell className={classes.tableHeaderCell}>Output Tokens</TableHeaderCell>}
5456
{hasTotalTokens && <TableHeaderCell className={classes.tableHeaderCell}>Total Tokens</TableHeaderCell>}
@@ -92,7 +94,8 @@ export const ChatDetailsSection = ({ chatDetails }: { chatDetails: ChatDetails;
9294
</TableCell>
9395
)}
9496
<TableCell>{turn.latency.toFixed(2)}</TableCell>
95-
{hasModelInfo && <TableCell>{turn.model || '-'}</TableCell>}
97+
{hasModel && <TableCell>{turn.model || '-'}</TableCell>}
98+
{hasModelProvider && <TableCell>{turn.modelProvider || '-'}</TableCell>}
9699
{hasInputTokens && <TableCell>{turn.usage?.inputTokenCount || '-'}</TableCell>}
97100
{hasOutputTokens && <TableCell>{turn.usage?.outputTokenCount || '-'}</TableCell>}
98101
{hasTotalTokens && <TableCell>{turn.usage?.totalTokenCount || '-'}</TableCell>}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type ChatDetails = {
3939
type ChatTurnDetails = {
4040
latency: number;
4141
model?: string;
42+
modelProvider?: string;
4243
usage?: UsageDetails;
4344
cacheKey?: string;
4445
cacheHit?: boolean;

0 commit comments

Comments
 (0)