refactor: Sentry PipelineBehavior

.github/workflows/ci.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - develop
+      - dev
   pull_request:
     branches:
       - main
-      - develop
+      - dev
   workflow_dispatch:
 
 jobs:

README.md

@@ -63,11 +63,11 @@ For detailed information about configuring different models, see our documentati
 
 Cellm is useful for repetitive tasks on both structured and unstructured data:
 
-1. **Text classification** - Categorize survey responses, support tickets, etc.
-2. **Model comparison** - Compare results from different LLMs side by side
-3. **Data cleaning** - Standardize names, fix formatting issues
-4. **Content summarization** - Condense articles, papers, or reports
-5. **Entity extraction** - Pull out names, locations, dates from text
+1. **Text classification:** Categorize survey responses, support tickets, etc.
+2. **Model comparison:** Compare results from different LLMs side by side
+3. **Data cleaning:** Standardize names, fix formatting issues
+4. **Content summarization:** Condense articles, papers, or reports
+5. **Entity recognition:** Pull out names, locations, dates from text
 
 For more use cases and examples, see our [Prompting Guide](https://docs.getcellm.com/usage/prompting).
 
@@ -80,6 +80,24 @@ A friend was writing a systematic review paper and had to compare 7,500 papers a
 
 Cellm enables everyone to automate repetitive tasks with AI to a level that was previously available only to programmers.
 
+## Telemetry
+To help us improve Cellm, we collect limited, anonymous telemetry data:
+
+- **Crash reports:** To help us fix bugs.
+- **Prompts:** To help us understand usage patterns. For example, if you use `=PROMPT(A1:B2, "Extract person names")`, we capture the text "Extract person names" and prompt options. The prompt options are things like the model you use and the temperature setting. We do not capture the data in cells A1:B2. 
+
+We do not collect any data from your spreadsheet and we have no way of associating your prompts with you. You can see for yourself at [Cellm.Models/Behaviors/SentryBehavior.cs](Cellm.Models/Behaviors/SentryBehavior.cs).
+
+You can disable telemetry at any time by creating an `appsettings.json` file in the same folder as `Cellm-AddIn64-packed.xll` with the following contents:
+
+```json
+{
+    "SentryConfiguration": {
+        "IsEnabled": false
+    }
+}
+```
+
 ## License
 
 Fair Core License, Version 1.0, Apache 2.0 Future License

src/Cellm.Models/Behaviors/CacheBehavior.cs

@@ -1,4 +1,4 @@
-using System.Security.Cryptography;
+using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
 using Cellm.Models.Providers;
@@ -25,7 +25,7 @@ internal class CacheBehavior<TRequest, TResponse>(
 
     public async Task<TResponse> Handle(TRequest request, RequestHandlerDelegate<TResponse> next, CancellationToken cancellationToken)
     {
-        if (providerConfiguration.CurrentValue.EnableCache)
+        if (!providerConfiguration.CurrentValue.EnableCache)
         {
             logger.LogDebug("Prompt caching disabled");
             return await next();

src/Cellm.Models/Behaviors/SentryBehavior.cs

@@ -1,22 +1,60 @@
-using MediatR;
+using Cellm.AddIn;
+using MediatR;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Logging;
 
 namespace Cellm.Models.Behaviors;
 
-internal class SentryBehavior<TRequest, TResponse> : IPipelineBehavior<TRequest, TResponse>
-    where TRequest : notnull
+internal class SentryBehavior<TRequest, TResponse>(ILogger<SentryBehavior<TRequest, TResponse>> logger) : IPipelineBehavior<TRequest, TResponse>
+    where TRequest : IModelRequest<TResponse>
 {
     public async Task<TResponse> Handle(TRequest request, RequestHandlerDelegate<TResponse> next, CancellationToken cancellationToken)
     {
-        var transaction = SentrySdk.StartTransaction(typeof(TRequest).Name, typeof(TRequest).Name);
+        if (!SentrySdk.IsEnabled)
+        {
+            logger.LogDebug("Sentry disabled");
+            return await next();
+        }
+
+        logger.LogDebug("Sentry enabled");
+
+        var transaction = SentrySdk.StartTransaction($"{nameof(Cellm)}.{nameof(Models)}", typeof(TRequest).Name);
         SentrySdk.ConfigureScope(scope => scope.Transaction = transaction);
 
         try
         {
+            transaction.Contexts["ChatOptions"] = request.Prompt.Options;
+            transaction.Contexts["UserInstructions"] = GetUserInstructions(request.Prompt.Messages);
+
             return await next();
         }
         finally
         {
             transaction.Finish();
         }
     }
+
+    private static string GetUserInstructions(IList<ChatMessage> messages)
+    {
+        var userMessage = messages
+            .Where(x => x.Role == ChatRole.User)
+            .First()
+            .Text;
+
+        var startIndex = userMessage.IndexOf(ArgumentParser.InstructionsStartTag, StringComparison.OrdinalIgnoreCase) + ArgumentParser.InstructionsStartTag.Length;
+
+        if (startIndex < 0)
+        {
+            return string.Empty;
+        }
+
+        var endIndex = userMessage.IndexOf(ArgumentParser.InstructionsEndTag, startIndex, StringComparison.OrdinalIgnoreCase);
+
+        if (endIndex < 0)
+        {
+            return string.Empty;
+        }
+
+        return userMessage[startIndex..endIndex];
+    }
 }

src/Cellm.Models/Behaviors/ToolBehavior.cs

@@ -3,6 +3,7 @@
 using Cellm.Tools.ModelContextProtocol;
 using MediatR;
 using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using ModelContextProtocol.Client;
 
@@ -11,25 +12,47 @@ namespace Cellm.Models.Tools;
 internal class ToolBehavior<TRequest, TResponse>(
     IOptionsMonitor<ProviderConfiguration> providerConfiguration,
     IOptionsMonitor<ModelContextProtocolConfiguration> modelContextProtocolConfiguration,
-    IEnumerable<AIFunction> functions)
+    IEnumerable<AIFunction> functions,
+    ILogger<ToolBehavior<TRequest, TResponse>> logger,
+    ILoggerFactory loggerFactory)
     : IPipelineBehavior<TRequest, TResponse> where TRequest : IModelRequest<TResponse>
 {
+    // TODO: Use HybridCache (await fix that McpLcientTool can't be serialized/deserialized)
+    private Dictionary<string, IList<McpClientTool>> _poorMansCache = [];
+
     public async Task<TResponse> Handle(TRequest request, RequestHandlerDelegate<TResponse> next, CancellationToken cancellationToken)
     {
         if (providerConfiguration.CurrentValue.EnableTools.Any(t => t.Value))
         {
+            logger.LogDebug("Native tools enabled");
+
             request.Prompt.Options.Tools = GetNativeTools();
         }
+        else
+        {
+            logger.LogDebug("Native tools disabled");
+        }
 
         if (providerConfiguration.CurrentValue.EnableModelContextProtocolServers.Any(t => t.Value))
         {
+            logger.LogDebug("MCP tools enabled");
+
             request.Prompt.Options.Tools ??= [];
 
             await foreach (var tool in GetModelContextProtocolTools(cancellationToken))
             {
                 request.Prompt.Options.Tools.Add(tool);
             }
         }
+        else
+        {
+            logger.LogDebug("MCP tools disabled");
+        }
+
+        if (request.Prompt.Options.Tools is not null && request.Prompt.Options.Tools.Any())
+        {
+            logger.LogDebug("Tools: {tools}", request.Prompt.Options.Tools);
+        }
 
         return await next();
     }
@@ -41,13 +64,8 @@ private List<AITool> GetNativeTools()
                 .ToList<AITool>();
     }
 
-    // TODO:
-    //  - Cache capabilities on a per-server basis.
-    //  - Query servers in parallel.
-    //
-    //  Note: We cannot get list of tools only on startup because user can add/delete/enable/disable servers. But
-    //        with this solution we query servers for capabilities on every model call which is hardly ideal.
-    //        We need to cache on a per-server basis so servers added at runtime will be queried
+    // TODO: Query servers in parallel
+
     private async IAsyncEnumerable<AITool> GetModelContextProtocolTools([EnumeratorCancellation] CancellationToken cancellationToken)
     {
         foreach (var server in modelContextProtocolConfiguration.CurrentValue.Servers)
@@ -57,14 +75,16 @@ private async IAsyncEnumerable<AITool> GetModelContextProtocolTools([EnumeratorC
                 continue;
             }
 
-            var client = await McpClientFactory.CreateAsync(server, cancellationToken: cancellationToken);
+            _poorMansCache.TryGetValue(server.Name, out var tools);
 
-            if (client is null)
+            if (tools is null)
             {
-                continue;
+                var client = await McpClientFactory.CreateAsync(server, loggerFactory: loggerFactory, cancellationToken: cancellationToken);
+                tools = await client.ListToolsAsync(cancellationToken: cancellationToken);
+                _poorMansCache[server.Name] = tools;
             }
 
-            foreach (var tool in await client.ListToolsAsync(cancellationToken: cancellationToken))
+            foreach (var tool in tools)
             {
                 yield return tool;
             }

src/Cellm/AddIn/ArgumentParser.cs

@@ -16,6 +16,11 @@ public class ArgumentParser
     private object? _instructionsOrTemperature;
     private object? _temperature;
 
+    public static readonly string ContextStartTag = "<context>";
+    public static readonly string ContextEndTag = "</context>";
+    public static readonly string InstructionsStartTag = "<instructions>";
+    public static readonly string InstructionsEndTag = "<instructions>";
+
     private readonly IConfiguration _configuration;
 
     public ArgumentParser(IConfiguration configuration)
@@ -211,18 +216,18 @@ private static string GetRowName(int rowNumber)
     private static string RenderCells(string context)
     {
         return new StringBuilder()
-            .AppendLine("<context>")
+            .AppendLine(ContextStartTag)
             .AppendLine(context)
-            .AppendLine("</context>")
+            .AppendLine(ContextEndTag)
             .ToString();
     }
 
     private static string RenderInstructions(string instructions)
     {
         return new StringBuilder()
-            .AppendLine("<instructions>")
+            .AppendLine(InstructionsStartTag)
             .AppendLine(instructions)
-            .AppendLine("</instructions>")
+            .AppendLine(InstructionsEndTag)
             .ToString();
     }
 

src/Cellm/Cellm.csproj

@@ -5,9 +5,7 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
-
-    <!-- Required for GitVersion.MsBuild -->
-    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <Version>0.2.0</Version>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -24,10 +22,6 @@
     <PackageReference Include="Anthropic.SDK" Version="5.1.1" />
     <PackageReference Include="ExcelDna.Addin" Version="1.9.0-alpha3" />
     <PackageReference Include="ExcelDna.Interop" Version="15.0.1" />
-    <PackageReference Include="GitVersion.MsBuild" Version="6.2.0">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
     <PackageReference Include="MediatR" Version="12.5.0" />
     <PackageReference Include="Microsoft.Extensions.AI" Version="9.4.0-preview.1.25207.5" />
     <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.4.0-preview.1.25207.5" />
@@ -60,5 +54,4 @@
   </ItemGroup>
 
   <Import Project="..\Cellm.Models\Cellm.Models.projitems" Label="Shared" />
-
 </Project>

src/Cellm/Services/Configuration/SentryConfiguration.cs

@@ -11,4 +11,6 @@ internal class SentryConfiguration
     public float ProfilesSampleRate { get; init; }
 
     public string Environment { get; init; } = string.Empty;
+
+    public bool Debug { get; init; }
 }

src/Cellm/Services/ServiceLocator.cs

@@ -23,6 +23,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Sentry.Infrastructure;
 
 namespace Cellm.Services;
 
@@ -72,25 +73,23 @@ private static IServiceCollection ConfigureServices(IServiceCollection services)
         services
           .AddLogging(loggingBuilder =>
           {
-              var assembly = Assembly.GetExecutingAssembly();
-              var gitVersionInformationType = assembly.GetType("GitVersionInformation");
-              var assemblyInformationalVersion = gitVersionInformationType?.GetField("AssemblyInformationalVersion");
-
               loggingBuilder
+                  .AddConfiguration(configuration.GetSection("Logging"))
                   .AddConsole()
                   .AddDebug()
                   .AddSentry(sentryLoggingOptions =>
                   {
                       sentryLoggingOptions.InitializeSdk = sentryConfiguration.IsEnabled;
                       sentryLoggingOptions.Release = GetReleaseVersion();
+                      sentryLoggingOptions.Environment = sentryConfiguration.Environment;
                       sentryLoggingOptions.Dsn = sentryConfiguration.Dsn;
-                      sentryLoggingOptions.Debug = cellmConfiguration.Debug;
+                      sentryLoggingOptions.Debug = sentryConfiguration.Debug;
                       sentryLoggingOptions.DiagnosticLevel = SentryLevel.Debug;
+                      sentryLoggingOptions.DiagnosticLogger = new TraceDiagnosticLogger(SentryLevel.Debug);
                       sentryLoggingOptions.TracesSampleRate = sentryConfiguration.TracesSampleRate;
                       sentryLoggingOptions.ProfilesSampleRate = sentryConfiguration.ProfilesSampleRate;
                       sentryLoggingOptions.Environment = sentryConfiguration.Environment;
                       sentryLoggingOptions.AutoSessionTracking = true;
-                      sentryLoggingOptions.IsGlobalModeEnabled = true;
                       sentryLoggingOptions.AddIntegration(new ProfilingIntegration());
                   });
           });
@@ -100,9 +99,9 @@ private static IServiceCollection ConfigureServices(IServiceCollection services)
             .AddMediatR(cfg =>
             {
                 cfg.RegisterServicesFromAssembly(Assembly.GetExecutingAssembly());
-                cfg.AddOpenBehavior(typeof(SentryBehavior<,>));
-                cfg.AddOpenBehavior(typeof(CacheBehavior<,>));
-                cfg.AddOpenBehavior(typeof(ToolBehavior<,>));
+                cfg.AddOpenBehavior(typeof(SentryBehavior<,>), ServiceLifetime.Singleton);
+                cfg.AddOpenBehavior(typeof(ToolBehavior<,>), ServiceLifetime.Singleton);
+                cfg.AddOpenBehavior(typeof(CacheBehavior<,>), ServiceLifetime.Singleton);
             })
             .AddSingleton(configuration)
             .AddTransient<ArgumentParser>()
@@ -152,20 +151,9 @@ private static IServiceCollection ConfigureServices(IServiceCollection services)
 
     public static string GetReleaseVersion()
     {
-        var releaseVersion = "unknown";
-
-        var value = Assembly
-            .GetExecutingAssembly()
-            .GetType("GitVersionInformation")?
-            .GetField("AssemblyInformationalVersion")?
-            .GetValue(null);
-
-        if (value is string valueAsString)
-        {
-            releaseVersion = valueAsString;
-        }
-
-        return releaseVersion;
+        return Assembly.GetExecutingAssembly()?
+            .GetCustomAttribute<AssemblyInformationalVersionAttribute>()?
+            .InformationalVersion ?? "unknown";
     }
 
     public static void Dispose()