Initial pass at AppInsights sending custom events (#899)

### Motivation and Context

Tracking high level user engagement of features can direct further
development and enable measurements of active user counts. It is
important to note that completions and conversations themselves should
not be logged, just data useful in aggregate.

### Description

This is an initial pass for review by the team.

### Motivation and Context
<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

1. This change is essential for tracking user engagement and creating an
audit trail of user activity.
2. This change adds a basic framework for tracking key telemetry events
3. This scenario enables management of those teams deploying instances
to understand how their deployment is being used by their team.
4. This is not related to an open issue.

### Description
<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

The overall design is to add a service accessible to controllers and
other services that enables telemetry to be fired with minimal context.
For this an abstract ITelemetryService interface was added, allowing
systems to fire events on this from across the kernel without any
dependencies on a specific implementation.

### Contribution Checklist
<!-- Before submitting this PR, please make sure: -->
- [X] The code builds clean without any errors or warnings
- [X] The PR follows SK Contribution Guidelines
(https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
- [X] The code follows the .NET coding conventions
(https://learn.microsoft.com/dotnet/csharp/fundamentals/coding-style/coding-conventions)
verified with `dotnet format`
- [] All unit tests pass, and I have added new tests where possible -
_tests appear to fail locally for config related reasons._
- [ ] I didn't break anyone 😄 - _maybe myself, a bit..._

---------

Co-authored-by: Ian Norris <ianorr@microsoft.com>
Co-authored-by: Adrian Bonar <56417140+adrianwyatt@users.noreply.github.com>
Co-authored-by: Gil LaHaye <gillahaye@microsoft.com>
Co-authored-by: Ben Thomas <ben.thomas@microsoft.com>

dotnet/src/SemanticKernel.Abstractions/Diagnostics/ITelemetryService.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.SemanticKernel.Diagnostics;
+
+/// <summary>
+/// Interface for common telemetry events to track actions across the semantic kernel.
+/// </summary>
+public interface ITelemetryService
+{
+    /// <summary>
+    /// Creates a telemetry event when a skill function is executed.
+    /// </summary>
+    /// <param name="skillName">Name of the skill</param>
+    /// <param name="functionName">Skill function name</param>
+    /// <param name="success">If the skill executed successfully</param>
+    void TrackSkillFunction(string skillName, string functionName, bool success);
+}

samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs

@@ -25,6 +25,7 @@
 using SemanticKernel.Service.CopilotChat.Hubs;
 using SemanticKernel.Service.CopilotChat.Models;
 using SemanticKernel.Service.CopilotChat.Skills.ChatSkills;
+using SemanticKernel.Service.Diagnostics;
 using SemanticKernel.Service.Models;
 
 namespace SemanticKernel.Service.CopilotChat.Controllers;
@@ -37,14 +38,16 @@ public class ChatController : ControllerBase, IDisposable
 {
     private readonly ILogger<ChatController> _logger;
     private readonly List<IDisposable> _disposables;
+    private readonly ITelemetryService _telemetryService;
     private const string ChatSkillName = "ChatSkill";
     private const string ChatFunctionName = "Chat";
     private const string ReceiveResponseClientCall = "ReceiveResponse";
     private const string GeneratingResponseClientCall = "ReceiveBotTypingState";
 
-    public ChatController(ILogger<ChatController> logger)
+    public ChatController(ILogger<ChatController> logger, ITelemetryService telemetryService)
     {
         this._logger = logger;
+        this._telemetryService = telemetryService;
         this._disposables = new List<IDisposable>();
     }
 
@@ -103,7 +106,16 @@ public async Task<IActionResult> ChatAsync(
         }
 
         // Run the function.
-        SKContext result = await kernel.RunAsync(contextVariables, function!);
+        SKContext? result = null;
+        try
+        {
+            result = await kernel.RunAsync(contextVariables, function!);
+        }
+        finally
+        {
+            this._telemetryService.TrackSkillFunction(ChatSkillName, ChatFunctionName, (!result?.ErrorOccurred) ?? false);
+        }
+
         if (result.ErrorOccurred)
         {
             if (result.LastException is AIException aiException && aiException.Detail is not null)

samples/apps/copilot-chat-app/webapi/Diagnostics/ITelemetryService.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.Service.Diagnostics;
+
+/// <summary>
+/// Interface for common telemetry events to track actions across the semantic kernel.
+/// </summary>
+public interface ITelemetryService
+{
+    /// <summary>
+    /// Creates a telemetry event when a skill function is executed.
+    /// </summary>
+    /// <param name="skillName">Name of the skill</param>
+    /// <param name="functionName">Skill function name</param>
+    /// <param name="success">If the skill executed successfully</param>
+    void TrackSkillFunction(string skillName, string functionName, bool success);
+}

samples/apps/copilot-chat-app/webapi/Program.cs

@@ -3,6 +3,8 @@
 using System;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.ApplicationInsights.Extensibility.Implementation;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Hosting.Server;
@@ -12,6 +14,8 @@
 using Microsoft.Extensions.Logging;
 using SemanticKernel.Service.CopilotChat.Extensions;
 using SemanticKernel.Service.CopilotChat.Hubs;
+using SemanticKernel.Service.Diagnostics;
+using SemanticKernel.Service.Services;
 
 namespace SemanticKernel.Service;
 
@@ -48,10 +52,20 @@ public static async Task Main(string[] args)
         // Add SignalR as the real time relay service
         builder.Services.AddSignalR();
 
-        // Add in the rest of the services.
+        // Add AppInsights telemetry
         builder.Services
-            .AddApplicationInsightsTelemetry()
+            .AddHttpContextAccessor()
+            .AddApplicationInsightsTelemetry(options => { options.ConnectionString = builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"]; })
+            .AddSingleton<ITelemetryInitializer, AppInsightsUserTelemetryInitializerService>()
             .AddLogging(logBuilder => logBuilder.AddApplicationInsights())
+            .AddSingleton<ITelemetryService, AppInsightsTelemetryService>();
+
+#if DEBUG
+        TelemetryDebugWriter.IsTracingDisabled = false;
+#endif
+
+        // Add in the rest of the services.
+        builder.Services
             .AddAuthorization(builder.Configuration)
             .AddEndpointsApiExplorer()
             .AddSwaggerGen()

samples/apps/copilot-chat-app/webapi/README.md

@@ -112,3 +112,87 @@ Before you get started, make sure you have the following additional requirements
     docker run --name copilotchat -p 6333:6333 -v "$(pwd)/data/qdrant:/qdrant/storage" qdrant/qdrant
     ```
     > To stop the container, in another terminal window run `docker container stop copilotchat; docker container rm copilotchat;`.
+
+# (Optional) Enable Application Insights telemetry
+
+Enabling telemetry on CopilotChatApi allows you to capture data about requests to and from the API, allowing you to monitor the deployment and monitor how the application is being used.
+
+To use Application Insights, first create an instance in your Azure subscription that you can use for this purpose.
+
+On the resource overview page, in the top right use the copy button to copy the Connection String and paste this into the `APPLICATIONINSIGHTS_CONNECTION_STRING` setting as either a appsettings value, or add it as a secret.
+
+In addition to this there are some custom events that can inform you how users are using the service such as `SkillFunction`.
+
+To access these custom events the suggested method is to use Azure Data Explorer (ADX). To access data from Application Insights in ADX, create a new dashboard and add a new Data Source (use the ellipsis dropdown in the top right).
+
+In the Cluster URI use the following link: `https://ade.applicationinsights.io/subscriptions/<Your subscription Id>`. The subscription id is shown on the resource page for your Applications Insights instance. You can then select the Database for the Application Insights resource.
+
+For more info see [Query data in Azure Monitor using Azure Data Explorer](https://learn.microsoft.com/en-us/azure/data-explorer/query-monitor-data).
+
+CopilotChat specific events are in a table called `customEvents`.
+
+For example to see the most recent 100 skill function invocations:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend userId = tostring(customDimensions.userId)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| project timestamp, skillFunction, success, userId, environment
+| order by timestamp desc
+| limit 100
+```
+
+Or to report the success rate of skill functions against environments, you can first add a parameter to the dashboard to filter the environment.
+
+You can use this query to show the environments available by adding the `Source` as this `Query`:
+
+```kql
+customEvents
+| where timestamp between (['_startTime'] .. ['_endTime']) // Time range filtering
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| distinct environment
+```
+
+Name the variable `_environment`, select `Multiple Selection` and tick `Add empty "Select all" value`. Finally `Select all` as the `Default value`.
+
+You can then query the success rate with this query:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| summarize Total=count(), Success=countif(success) by skillFunction, environment
+| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment
+| order by SuccessPercentage asc
+```
+
+You may wish to use the Visual tab to turn on conditional formatting to highlight low success rates or render it as a chart.
+
+Finally you could render this data over time with a query like this:
+
+```kql
+customEvents
+| where timestamp between (_startTime .. _endTime)
+| where name == "SkillFunction"
+| extend skill = tostring(customDimensions.skillName)
+| extend function = tostring(customDimensions.functionName)
+| extend success = tobool(customDimensions.success)
+| extend environment = tostring(customDimensions.AspNetCoreEnvironment)
+| extend skillFunction = strcat(skill, '/', function)
+| summarize Total=count(), Success=countif(success) by skillFunction, environment, bin(timestamp,1m)
+| project skillFunction, SuccessPercentage = 100.0 * Success/Total, environment, timestamp
+| order by timestamp asc
+```
+
+Then use a Time chart on the Visual tab.

samples/apps/copilot-chat-app/webapi/Services/AppInsightsTelemetryService.cs

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Security.Claims;
+using Microsoft.ApplicationInsights;
+using Microsoft.AspNetCore.Http;
+using SemanticKernel.Service.Diagnostics;
+
+namespace SemanticKernel.Service.Services;
+
+/// <summary>
+/// Implementation of the telemetry service interface for Azure Application Insights (AppInsights).
+/// </summary>
+public class AppInsightsTelemetryService : ITelemetryService
+{
+    private const string UnknownUserId = "unauthenticated";
+
+    private readonly TelemetryClient _telemetryClient;
+    private readonly IHttpContextAccessor _httpContextAccessor;
+
+    /// <summary>
+    /// Creates an instance of the app insights telemetry service.
+    /// This should be injected into the service collection during startup.
+    /// </summary>
+    /// <param name="telemetryClient">An AppInsights telemetry client</param>
+    /// <param name="httpContextAccessor">Accessor for the current request's http context</param>
+    public AppInsightsTelemetryService(TelemetryClient telemetryClient, IHttpContextAccessor httpContextAccessor)
+    {
+        this._telemetryClient = telemetryClient;
+        this._httpContextAccessor = httpContextAccessor;
+    }
+
+    /// <inheritdoc/>
+    public void TrackSkillFunction(string skillName, string functionName, bool success)
+    {
+        var properties = new Dictionary<string, string>(this.BuildDefaultProperties())
+        {
+            { "skillName", skillName },
+            { "functionName", functionName },
+            { "success", success.ToString() },
+        };
+
+        this._telemetryClient.TrackEvent("SkillFunction", properties);
+    }
+
+    /// <summary>
+    /// Gets the current user's ID from the http context for the current request.
+    /// </summary>
+    /// <param name="contextAccessor">The http context accessor</param>
+    /// <returns></returns>
+    public static string GetUserIdFromHttpContext(IHttpContextAccessor contextAccessor)
+    {
+        var context = contextAccessor.HttpContext;
+        if (context == null)
+        {
+            return UnknownUserId;
+        }
+
+        var user = context.User;
+        if (user?.Identity?.IsAuthenticated != true)
+        {
+            return UnknownUserId;
+        }
+
+        var userId = user.FindFirst(ClaimTypes.NameIdentifier)?.Value;
+
+        if (userId == null)
+        {
+            return UnknownUserId;
+        }
+
+        return userId;
+    }
+
+    /// <summary>
+    /// Prepares a list of common properties that all telemetry events should contain.
+    /// </summary>
+    /// <returns>Collection of common properties for all telemetry events</returns>
+    private Dictionary<string, string> BuildDefaultProperties()
+    {
+        string? userId = GetUserIdFromHttpContext(this._httpContextAccessor);
+
+        return new Dictionary<string, string>
+        {
+            { "userId", GetUserIdFromHttpContext(this._httpContextAccessor) }
+        };
+    }
+}

samples/apps/copilot-chat-app/webapi/Services/AppInsightsUserTelemetryInitializerService.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.ApplicationInsights.Channel;
+using Microsoft.ApplicationInsights.DataContracts;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.AspNetCore.Http;
+
+namespace SemanticKernel.Service.Services;
+
+/// <summary>
+/// A telemetry initializer used by the TelemetryClient to fill in data for requests.
+/// This implementation injects the id of the current authenticated user (if there is one).
+/// </summary>
+public class AppInsightsUserTelemetryInitializerService : ITelemetryInitializer
+{
+    public AppInsightsUserTelemetryInitializerService(IHttpContextAccessor httpContextAccessor)
+    {
+        this._contextAccessor = httpContextAccessor;
+    }
+
+    /// <inheritdoc/>
+    public void Initialize(ITelemetry telemetry)
+    {
+        if (telemetry is not RequestTelemetry requestTelemetry)
+        {
+            return;
+        }
+
+        var userId = AppInsightsTelemetryService.GetUserIdFromHttpContext(this._contextAccessor);
+
+        telemetry.Context.User.Id = userId;
+    }
+
+    private readonly IHttpContextAccessor _contextAccessor;
+}

samples/apps/copilot-chat-app/webapi/appsettings.json

@@ -188,7 +188,7 @@
 
   // CORS
   "AllowedOrigins": [
-    "http://localhost:3000", 
+    "http://localhost:3000",
     "https://localhost:3000"
   ],
 
@@ -220,10 +220,8 @@
 
   //
   // Application Insights configuration
-  // - Set "ApplicationInsights:ConnectionString" using dotnet's user secrets (see above)
-  //     (i.e. dotnet user-secrets set "ApplicationInsights:ConnectionString" "MY_APPINS_CONNSTRING")
-  // 
-  "ApplicationInsights": {
-    "ConnectionString": ""
-  }
+  // - Set "APPLICATIONINSIGHTS_CONNECTION_STRING" using dotnet's user secrets (see above)
+  //     (i.e. dotnet user-secrets set "APPLICATIONINSIGHTS_CONNECTION_STRING" "MY_APPINS_CONNSTRING")
+  //
+  "APPLICATIONINSIGHTS_CONNECTION_STRING": null
 }