diff --git a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs new file mode 100644 index 000000000000..f7c592ace343 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Diagnostics; + +/// +/// Interface for common telemetry events to track actions across the semantic kernel. +/// +public interface ITelemetryService +{ + /// + /// Creates a telemetry event when a skill function is executed. + /// + /// Name of the skill + /// Skill function name + /// If the skill executed successfully + void TrackSkillFunction(string skillName, string functionName, bool success); +} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs index 3c488ba3f655..d163d08c0339 100644 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs +++ b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs @@ -25,6 +25,7 @@ using SemanticKernel.Service.CopilotChat.Hubs; using SemanticKernel.Service.CopilotChat.Models; using SemanticKernel.Service.CopilotChat.Skills.ChatSkills; +using SemanticKernel.Service.Diagnostics; using SemanticKernel.Service.Models; namespace SemanticKernel.Service.CopilotChat.Controllers; @@ -37,14 +38,16 @@ public class ChatController : ControllerBase, IDisposable { private readonly ILogger _logger; private readonly List _disposables; + private readonly ITelemetryService _telemetryService; private const string ChatSkillName = "ChatSkill"; private const string ChatFunctionName = "Chat"; private const string ReceiveResponseClientCall = "ReceiveResponse"; private const string GeneratingResponseClientCall = "ReceiveBotTypingState"; - public ChatController(ILogger logger) + public ChatController(ILogger logger, ITelemetryService telemetryService) { this._logger = logger; + this._telemetryService = telemetryService; this._disposables = new List(); } @@ -103,7 +106,16 @@ public async Task ChatAsync( } // Run the function. - SKContext result = await kernel.RunAsync(contextVariables, function!); + SKContext? result = null; + try + { + result = await kernel.RunAsync(contextVariables, function!); + } + finally + { + this._telemetryService.TrackSkillFunction(ChatSkillName, ChatFunctionName, (!result?.ErrorOccurred) ?? false); + } + if (result.ErrorOccurred) { if (result.LastException is AIException aiException && aiException.Detail is not null) diff --git a/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs b/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs new file mode 100644 index 000000000000..8c472426111d --- /dev/null +++ b/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace SemanticKernel.Service.Diagnostics; + +/// +/// Interface for common telemetry events to track actions across the semantic kernel. +/// +public interface ITelemetryService +{ + /// + /// Creates a telemetry event when a skill function is executed. + /// + /// Name of the skill + /// Skill function name + /// If the skill executed successfully + void TrackSkillFunction(string skillName, string functionName, bool success); +} diff --git a/samples/apps/copilot-chat-app/webapi/Program.cs b/samples/apps/copilot-chat-app/webapi/Program.cs index e761088fa939..dd707dfc59d4 100644 --- a/samples/apps/copilot-chat-app/webapi/Program.cs +++ b/samples/apps/copilot-chat-app/webapi/Program.cs @@ -3,6 +3,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.Server; @@ -12,6 +14,8 @@ using Microsoft.Extensions.Logging; using SemanticKernel.Service.CopilotChat.Extensions; using SemanticKernel.Service.CopilotChat.Hubs; +using SemanticKernel.Service.Diagnostics; +using SemanticKernel.Service.Services; namespace SemanticKernel.Service; @@ -48,10 +52,20 @@ public static async Task Main(string[] args) // Add SignalR as the real time relay service builder.Services.AddSignalR(); - // Add in the rest of the services. + // Add AppInsights telemetry builder.Services - .AddApplicationInsightsTelemetry() + .AddHttpContextAccessor() + .AddApplicationInsightsTelemetry(options => { options.ConnectionString = builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"]; }) + .AddSingleton() .AddLogging(logBuilder => logBuilder.AddApplicationInsights()) + .AddSingleton(); + +#if DEBUG + TelemetryDebugWriter.IsTracingDisabled = false; +#endif + + // Add in the rest of the services. + builder.Services .AddAuthorization(builder.Configuration) .AddEndpointsApiExplorer() .AddSwaggerGen() diff --git a/samples/apps/copilot-chat-app/webapi/README.md b/samples/apps/copilot-chat-app/webapi/README.md index 08a800fa74db..425bf3dcb35d 100644 --- a/samples/apps/copilot-chat-app/webapi/README.md +++ b/samples/apps/copilot-chat-app/webapi/README.md @@ -112,3 +112,87 @@ Before you get started, make sure you have the following additional requirements docker run --name copilotchat -p 6333:6333 -v "$(pwd)/data/qdrant:/qdrant/storage" qdrant/qdrant ``` > To stop the container, in another terminal window run `docker container stop copilotchat; docker container rm copilotchat;`. + +# (Optional) Enable Application Insights telemetry + +Enabling telemetry on CopilotChatApi allows you to capture data about requests to and from the API, allowing you to monitor the deployment and monitor how the application is being used. + +To use Application Insights, first create an instance in your Azure subscription that you can use for this purpose. + +On the resource overview page, in the top right use the copy button to copy the Connection String and paste this into the `APPLICATIONINSIGHTS_CONNECTION_STRING` setting as either a appsettings value, or add it as a secret. + +In addition to this there are some custom events that can inform you how users are using the service such as `SkillFunction`. + +To access these custom events the suggested method is to use Azure Data Explorer (ADX). To access data from Application Insights in ADX, create a new dashboard and add a new Data Source (use the ellipsis dropdown in the top right). + +In the Cluster URI use the following link: `https://ade.applicationinsights.io/subscriptions/`. The subscription id is shown on the resource page for your Applications Insights instance. You can then select the Database for the Application Insights resource. + +For more info see [Query data in Azure Monitor using Azure Data Explorer](https://learn.microsoft.com/en-us/azure/data-explorer/query-monitor-data). + +CopilotChat specific events are in a table called `customEvents`. + +For example to see the most recent 100 skill function invocations: + +```kql +customEvents +| where timestamp between (_startTime .. _endTime) +| where name == "SkillFunction" +| extend skill = tostring(customDimensions.skillName) +| extend function = tostring(customDimensions.functionName) +| extend success = tobool(customDimensions.success) +| extend userId = tostring(customDimensions.userId) +| extend environment = tostring(customDimensions.AspNetCoreEnvironment) +| extend skillFunction = strcat(skill, '/', function) +| project timestamp, skillFunction, success, userId, environment +| order by timestamp desc +| limit 100 +``` + +Or to report the success rate of skill functions against environments, you can first add a parameter to the dashboard to filter the environment. + +You can use this query to show the environments available by adding the `Source` as this `Query`: + +```kql +customEvents +| where timestamp between (['_startTime'] .. ['_endTime']) // Time range filtering +| extend environment = tostring(customDimensions.AspNetCoreEnvironment) +| distinct environment +``` + +Name the variable `_environment`, select `Multiple Selection` and tick `Add empty "Select all" value`. Finally `Select all` as the `Default value`. + +You can then query the success rate with this query: + +```kql +customEvents +| where timestamp between (_startTime .. _endTime) +| where name == "SkillFunction" +| extend skill = tostring(customDimensions.skillName) +| extend function = tostring(customDimensions.functionName) +| extend success = tobool(customDimensions.success) +| extend environment = tostring(customDimensions.AspNetCoreEnvironment) +| extend skillFunction = strcat(skill, '/', function) +| summarize Total=count(), Success=countif(success) by skillFunction, environment +| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment +| order by SuccessPercentage asc +``` + +You may wish to use the Visual tab to turn on conditional formatting to highlight low success rates or render it as a chart. + +Finally you could render this data over time with a query like this: + +```kql +customEvents +| where timestamp between (_startTime .. _endTime) +| where name == "SkillFunction" +| extend skill = tostring(customDimensions.skillName) +| extend function = tostring(customDimensions.functionName) +| extend success = tobool(customDimensions.success) +| extend environment = tostring(customDimensions.AspNetCoreEnvironment) +| extend skillFunction = strcat(skill, '/', function) +| summarize Total=count(), Success=countif(success) by skillFunction, environment, bin(timestamp,1m) +| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment, timestamp +| order by timestamp asc +``` + +Then use a Time chart on the Visual tab. \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs new file mode 100644 index 000000000000..62b9ab0b0075 --- /dev/null +++ b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Security.Claims; +using Microsoft.ApplicationInsights; +using Microsoft.AspNetCore.Http; +using SemanticKernel.Service.Diagnostics; + +namespace SemanticKernel.Service.Services; + +/// +/// Implementation of the telemetry service interface for Azure Application Insights (AppInsights). +/// +public class AppInsightsTelemetryService : ITelemetryService +{ + private const string UnknownUserId = "unauthenticated"; + + private readonly TelemetryClient _telemetryClient; + private readonly IHttpContextAccessor _httpContextAccessor; + + /// + /// Creates an instance of the app insights telemetry service. + /// This should be injected into the service collection during startup. + /// + /// An AppInsights telemetry client + /// Accessor for the current request's http context + public AppInsightsTelemetryService(TelemetryClient telemetryClient, IHttpContextAccessor httpContextAccessor) + { + this._telemetryClient = telemetryClient; + this._httpContextAccessor = httpContextAccessor; + } + + /// + public void TrackSkillFunction(string skillName, string functionName, bool success) + { + var properties = new Dictionary(this.BuildDefaultProperties()) + { + { "skillName", skillName }, + { "functionName", functionName }, + { "success", success.ToString() }, + }; + + this._telemetryClient.TrackEvent("SkillFunction", properties); + } + + /// + /// Gets the current user's ID from the http context for the current request. + /// + /// The http context accessor + /// + public static string GetUserIdFromHttpContext(IHttpContextAccessor contextAccessor) + { + var context = contextAccessor.HttpContext; + if (context == null) + { + return UnknownUserId; + } + + var user = context.User; + if (user?.Identity?.IsAuthenticated != true) + { + return UnknownUserId; + } + + var userId = user.FindFirst(ClaimTypes.NameIdentifier)?.Value; + + if (userId == null) + { + return UnknownUserId; + } + + return userId; + } + + /// + /// Prepares a list of common properties that all telemetry events should contain. + /// + /// Collection of common properties for all telemetry events + private Dictionary BuildDefaultProperties() + { + string? userId = GetUserIdFromHttpContext(this._httpContextAccessor); + + return new Dictionary + { + { "userId", GetUserIdFromHttpContext(this._httpContextAccessor) } + }; + } +} diff --git a/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs new file mode 100644 index 000000000000..49f3033a5b15 --- /dev/null +++ b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; + +namespace SemanticKernel.Service.Services; + +/// +/// A telemetry initializer used by the TelemetryClient to fill in data for requests. +/// This implementation injects the id of the current authenticated user (if there is one). +/// +public class AppInsightsUserTelemetryInitializerService : ITelemetryInitializer +{ + public AppInsightsUserTelemetryInitializerService(IHttpContextAccessor httpContextAccessor) + { + this._contextAccessor = httpContextAccessor; + } + + /// + public void Initialize(ITelemetry telemetry) + { + if (telemetry is not RequestTelemetry requestTelemetry) + { + return; + } + + var userId = AppInsightsTelemetryService.GetUserIdFromHttpContext(this._contextAccessor); + + telemetry.Context.User.Id = userId; + } + + private readonly IHttpContextAccessor _contextAccessor; +} diff --git a/samples/apps/copilot-chat-app/webapi/appsettings.json b/samples/apps/copilot-chat-app/webapi/appsettings.json index 17042c36cc7e..a48432da495f 100644 --- a/samples/apps/copilot-chat-app/webapi/appsettings.json +++ b/samples/apps/copilot-chat-app/webapi/appsettings.json @@ -188,7 +188,7 @@ // CORS "AllowedOrigins": [ - "http://localhost:3000", + "http://localhost:3000", "https://localhost:3000" ], @@ -220,10 +220,8 @@ // // Application Insights configuration - // - Set "ApplicationInsights:ConnectionString" using dotnet's user secrets (see above) - // (i.e. dotnet user-secrets set "ApplicationInsights:ConnectionString" "MY_APPINS_CONNSTRING") - // - "ApplicationInsights": { - "ConnectionString": "" - } + // - Set "APPLICATIONINSIGHTS_CONNECTION_STRING" using dotnet's user secrets (see above) + // (i.e. dotnet user-secrets set "APPLICATIONINSIGHTS_CONNECTION_STRING" "MY_APPINS_CONNSTRING") + // + "APPLICATIONINSIGHTS_CONNECTION_STRING": null }