diff --git a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs
new file mode 100644
index 000000000000..f7c592ace343
--- /dev/null
+++ b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.SemanticKernel.Diagnostics;
+
+///
+/// Interface for common telemetry events to track actions across the semantic kernel.
+///
+public interface ITelemetryService
+{
+ ///
+ /// Creates a telemetry event when a skill function is executed.
+ ///
+ /// Name of the skill
+ /// Skill function name
+ /// If the skill executed successfully
+ void TrackSkillFunction(string skillName, string functionName, bool success);
+}
diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs
index 3c488ba3f655..d163d08c0339 100644
--- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs
+++ b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs
@@ -25,6 +25,7 @@
using SemanticKernel.Service.CopilotChat.Hubs;
using SemanticKernel.Service.CopilotChat.Models;
using SemanticKernel.Service.CopilotChat.Skills.ChatSkills;
+using SemanticKernel.Service.Diagnostics;
using SemanticKernel.Service.Models;
namespace SemanticKernel.Service.CopilotChat.Controllers;
@@ -37,14 +38,16 @@ public class ChatController : ControllerBase, IDisposable
{
private readonly ILogger _logger;
private readonly List _disposables;
+ private readonly ITelemetryService _telemetryService;
private const string ChatSkillName = "ChatSkill";
private const string ChatFunctionName = "Chat";
private const string ReceiveResponseClientCall = "ReceiveResponse";
private const string GeneratingResponseClientCall = "ReceiveBotTypingState";
- public ChatController(ILogger logger)
+ public ChatController(ILogger logger, ITelemetryService telemetryService)
{
this._logger = logger;
+ this._telemetryService = telemetryService;
this._disposables = new List();
}
@@ -103,7 +106,16 @@ public async Task ChatAsync(
}
// Run the function.
- SKContext result = await kernel.RunAsync(contextVariables, function!);
+ SKContext? result = null;
+ try
+ {
+ result = await kernel.RunAsync(contextVariables, function!);
+ }
+ finally
+ {
+ this._telemetryService.TrackSkillFunction(ChatSkillName, ChatFunctionName, (!result?.ErrorOccurred) ?? false);
+ }
+
if (result.ErrorOccurred)
{
if (result.LastException is AIException aiException && aiException.Detail is not null)
diff --git a/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs b/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs
new file mode 100644
index 000000000000..8c472426111d
--- /dev/null
+++ b/samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.Service.Diagnostics;
+
+///
+/// Interface for common telemetry events to track actions across the semantic kernel.
+///
+public interface ITelemetryService
+{
+ ///
+ /// Creates a telemetry event when a skill function is executed.
+ ///
+ /// Name of the skill
+ /// Skill function name
+ /// If the skill executed successfully
+ void TrackSkillFunction(string skillName, string functionName, bool success);
+}
diff --git a/samples/apps/copilot-chat-app/webapi/Program.cs b/samples/apps/copilot-chat-app/webapi/Program.cs
index e761088fa939..dd707dfc59d4 100644
--- a/samples/apps/copilot-chat-app/webapi/Program.cs
+++ b/samples/apps/copilot-chat-app/webapi/Program.cs
@@ -3,6 +3,8 @@
using System;
using System.Linq;
using System.Threading.Tasks;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.ApplicationInsights.Extensibility.Implementation;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
@@ -12,6 +14,8 @@
using Microsoft.Extensions.Logging;
using SemanticKernel.Service.CopilotChat.Extensions;
using SemanticKernel.Service.CopilotChat.Hubs;
+using SemanticKernel.Service.Diagnostics;
+using SemanticKernel.Service.Services;
namespace SemanticKernel.Service;
@@ -48,10 +52,20 @@ public static async Task Main(string[] args)
// Add SignalR as the real time relay service
builder.Services.AddSignalR();
- // Add in the rest of the services.
+ // Add AppInsights telemetry
builder.Services
- .AddApplicationInsightsTelemetry()
+ .AddHttpContextAccessor()
+ .AddApplicationInsightsTelemetry(options => { options.ConnectionString = builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"]; })
+ .AddSingleton()
.AddLogging(logBuilder => logBuilder.AddApplicationInsights())
+ .AddSingleton();
+
+#if DEBUG
+ TelemetryDebugWriter.IsTracingDisabled = false;
+#endif
+
+ // Add in the rest of the services.
+ builder.Services
.AddAuthorization(builder.Configuration)
.AddEndpointsApiExplorer()
.AddSwaggerGen()
diff --git a/samples/apps/copilot-chat-app/webapi/README.md b/samples/apps/copilot-chat-app/webapi/README.md
index 08a800fa74db..425bf3dcb35d 100644
--- a/samples/apps/copilot-chat-app/webapi/README.md
+++ b/samples/apps/copilot-chat-app/webapi/README.md
@@ -112,3 +112,87 @@ Before you get started, make sure you have the following additional requirements
docker run --name copilotchat -p 6333:6333 -v "$(pwd)/data/qdrant:/qdrant/storage" qdrant/qdrant
```
> To stop the container, in another terminal window run `docker container stop copilotchat; docker container rm copilotchat;`.
+
+# (Optional) Enable Application Insights telemetry
+
+Enabling telemetry on CopilotChatApi allows you to capture data about requests to and from the API, allowing you to monitor the deployment and monitor how the application is being used.
+
+To use Application Insights, first create an instance in your Azure subscription that you can use for this purpose.
+
+On the resource overview page, in the top right use the copy button to copy the Connection String and paste this into the `APPLICATIONINSIGHTS_CONNECTION_STRING` setting as either a appsettings value, or add it as a secret.
+
+In addition to this there are some custom events that can inform you how users are using the service such as `SkillFunction`.
+
+To access these custom events the suggested method is to use Azure Data Explorer (ADX). To access data from Application Insights in ADX, create a new dashboard and add a new Data Source (use the ellipsis dropdown in the top right).
+
+In the Cluster URI use the following link: `https://ade.applicationinsights.io/subscriptions/`. The subscription id is shown on the resource page for your Applications Insights instance. You can then select the Database for the Application Insights resource.
+
+For more info see [Query data in Azure Monitor using Azure Data Explorer](https://learn.microsoft.com/en-us/azure/data-explorer/query-monitor-data).
+
+CopilotChat specific events are in a table called `customEvents`.
+
+For example to see the most recent 100 skill function invocations:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend userId = tostring(customDimensions.userId)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| project timestamp, skillFunction, success, userId, environment
+| order by timestamp desc
+| limit 100
+```
+
+Or to report the success rate of skill functions against environments, you can first add a parameter to the dashboard to filter the environment.
+
+You can use this query to show the environments available by adding the `Source` as this `Query`:
+
+```kql
+customEvents
+| where timestamp between (['_startTime'] .. ['_endTime']) // Time range filtering
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| distinct environment
+```
+
+Name the variable `_environment`, select `Multiple Selection` and tick `Add empty "Select all" value`. Finally `Select all` as the `Default value`.
+
+You can then query the success rate with this query:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| summarize Total=count(), Success=countif(success) by skillFunction, environment
+| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment
+| order by SuccessPercentage asc
+```
+
+You may wish to use the Visual tab to turn on conditional formatting to highlight low success rates or render it as a chart.
+
+Finally you could render this data over time with a query like this:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| summarize Total=count(), Success=countif(success) by skillFunction, environment, bin(timestamp,1m)
+| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment, timestamp
+| order by timestamp asc
+```
+
+Then use a Time chart on the Visual tab.
\ No newline at end of file
diff --git a/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs
new file mode 100644
index 000000000000..62b9ab0b0075
--- /dev/null
+++ b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs
@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Security.Claims;
+using Microsoft.ApplicationInsights;
+using Microsoft.AspNetCore.Http;
+using SemanticKernel.Service.Diagnostics;
+
+namespace SemanticKernel.Service.Services;
+
+///
+/// Implementation of the telemetry service interface for Azure Application Insights (AppInsights).
+///
+public class AppInsightsTelemetryService : ITelemetryService
+{
+ private const string UnknownUserId = "unauthenticated";
+
+ private readonly TelemetryClient _telemetryClient;
+ private readonly IHttpContextAccessor _httpContextAccessor;
+
+ ///
+ /// Creates an instance of the app insights telemetry service.
+ /// This should be injected into the service collection during startup.
+ ///
+ /// An AppInsights telemetry client
+ /// Accessor for the current request's http context
+ public AppInsightsTelemetryService(TelemetryClient telemetryClient, IHttpContextAccessor httpContextAccessor)
+ {
+ this._telemetryClient = telemetryClient;
+ this._httpContextAccessor = httpContextAccessor;
+ }
+
+ ///
+ public void TrackSkillFunction(string skillName, string functionName, bool success)
+ {
+ var properties = new Dictionary(this.BuildDefaultProperties())
+ {
+ { "skillName", skillName },
+ { "functionName", functionName },
+ { "success", success.ToString() },
+ };
+
+ this._telemetryClient.TrackEvent("SkillFunction", properties);
+ }
+
+ ///
+ /// Gets the current user's ID from the http context for the current request.
+ ///
+ /// The http context accessor
+ ///
+ public static string GetUserIdFromHttpContext(IHttpContextAccessor contextAccessor)
+ {
+ var context = contextAccessor.HttpContext;
+ if (context == null)
+ {
+ return UnknownUserId;
+ }
+
+ var user = context.User;
+ if (user?.Identity?.IsAuthenticated != true)
+ {
+ return UnknownUserId;
+ }
+
+ var userId = user.FindFirst(ClaimTypes.NameIdentifier)?.Value;
+
+ if (userId == null)
+ {
+ return UnknownUserId;
+ }
+
+ return userId;
+ }
+
+ ///
+ /// Prepares a list of common properties that all telemetry events should contain.
+ ///
+ /// Collection of common properties for all telemetry events
+ private Dictionary BuildDefaultProperties()
+ {
+ string? userId = GetUserIdFromHttpContext(this._httpContextAccessor);
+
+ return new Dictionary
+ {
+ { "userId", GetUserIdFromHttpContext(this._httpContextAccessor) }
+ };
+ }
+}
diff --git a/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs
new file mode 100644
index 000000000000..49f3033a5b15
--- /dev/null
+++ b/samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs
@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.ApplicationInsights.Channel;
+using Microsoft.ApplicationInsights.DataContracts;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.AspNetCore.Http;
+
+namespace SemanticKernel.Service.Services;
+
+///
+/// A telemetry initializer used by the TelemetryClient to fill in data for requests.
+/// This implementation injects the id of the current authenticated user (if there is one).
+///
+public class AppInsightsUserTelemetryInitializerService : ITelemetryInitializer
+{
+ public AppInsightsUserTelemetryInitializerService(IHttpContextAccessor httpContextAccessor)
+ {
+ this._contextAccessor = httpContextAccessor;
+ }
+
+ ///
+ public void Initialize(ITelemetry telemetry)
+ {
+ if (telemetry is not RequestTelemetry requestTelemetry)
+ {
+ return;
+ }
+
+ var userId = AppInsightsTelemetryService.GetUserIdFromHttpContext(this._contextAccessor);
+
+ telemetry.Context.User.Id = userId;
+ }
+
+ private readonly IHttpContextAccessor _contextAccessor;
+}
diff --git a/samples/apps/copilot-chat-app/webapi/appsettings.json b/samples/apps/copilot-chat-app/webapi/appsettings.json
index 17042c36cc7e..a48432da495f 100644
--- a/samples/apps/copilot-chat-app/webapi/appsettings.json
+++ b/samples/apps/copilot-chat-app/webapi/appsettings.json
@@ -188,7 +188,7 @@
// CORS
"AllowedOrigins": [
- "http://localhost:3000",
+ "http://localhost:3000",
"https://localhost:3000"
],
@@ -220,10 +220,8 @@
//
// Application Insights configuration
- // - Set "ApplicationInsights:ConnectionString" using dotnet's user secrets (see above)
- // (i.e. dotnet user-secrets set "ApplicationInsights:ConnectionString" "MY_APPINS_CONNSTRING")
- //
- "ApplicationInsights": {
- "ConnectionString": ""
- }
+ // - Set "APPLICATIONINSIGHTS_CONNECTION_STRING" using dotnet's user secrets (see above)
+ // (i.e. dotnet user-secrets set "APPLICATIONINSIGHTS_CONNECTION_STRING" "MY_APPINS_CONNSTRING")
+ //
+ "APPLICATIONINSIGHTS_CONNECTION_STRING": null
}