Add support for Open Telemetry

schemas/dab.draft.schema.json

@@ -357,11 +357,44 @@
               "required": [
                 "connection-string"
               ]
+            },
+            "open-telemetry": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "endpoint": {
+                  "type": "string",
+                  "description": "Open Telemetry connection string"
+                },
+                "headers": {
+                  "type": "string",
+                  "description": "Open Telemetry headers"
+                },
+                "service-name": {
+                  "type": "string",
+                  "description": "Open Telemetry service name",
+                  "default": "dab"
+                },
+                "exporter-protocol": {
+                  "type": "string",
+                  "description": "Open Telemetry protocol",
+                  "default": "grpc",
+                  "enum": [
+                    "grpc",
+                    "httpprotobuf"
+                  ]
+                },
+                "enabled": {
+                  "type": "boolean",
+                  "description": "Allow enabling/disabling Open Telemetry.",
+                  "default": true
+                }
+              },
+              "required": [
+                "endpoint"
+              ]
             }
-          },
-          "required": [
-            "application-insights"
-          ]
+          }
         }
       }
     },

src/Cli.Tests/AddOpenTelemetryTests.cs

@@ -0,0 +1,166 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Cli.Tests
+{
+    /// <summary>
+    /// Tests for verifying the functionality of adding OpenTelemetry to the config file.
+    /// </summary>
+    [TestClass]
+    public class AddOpenTelemetryTests
+    {
+        public string RUNTIME_SECTION_WITH_OPEN_TELEMETRY_SECTION = GenerateRuntimeSection(TELEMETRY_SECTION_WITH_OPEN_TELEMETRY);
+        public string RUNTIME_SECTION_WITH_EMPTY_TELEMETRY_SECTION = GenerateRuntimeSection(EMPTY_TELEMETRY_SECTION);
+        [TestInitialize]
+        public void TestInitialize()
+        {
+            ILoggerFactory loggerFactory = TestLoggerSupport.ProvisionLoggerFactory();
+
+            ConfigGenerator.SetLoggerForCliConfigGenerator(loggerFactory.CreateLogger<ConfigGenerator>());
+            Utils.SetCliUtilsLogger(loggerFactory.CreateLogger<Utils>());
+        }
+
+        /// <summary>
+        /// Testing to check OpenTelemetry options are correctly added to the config.
+        /// Verifying scenarios such as enabling/disabling OpenTelemetry and providing a valid/empty endpoint.
+        /// </summary>
+        [DataTestMethod]
+        [DataRow(CliBool.True, "", false, DisplayName = "Fail to add OpenTelemetry with empty endpoint.")]
+        [DataRow(CliBool.True, "http://localhost:4317", true, DisplayName = "Successfully adds OpenTelemetry with valid endpoint")]
+        [DataRow(CliBool.False, "http://localhost:4317", true, DisplayName = "Successfully adds OpenTelemetry but disabled")]
+        public void TestAddOpenTelemetry(CliBool isTelemetryEnabled, string endpoint, bool expectSuccess)
+        {
+            MockFileSystem fileSystem = FileSystemUtils.ProvisionMockFileSystem();
+            string configPath = "test-opentelemetry-config.json";
+            fileSystem.AddFile(configPath, new MockFileData(INITIAL_CONFIG));
+
+            // Initial State
+            Assert.IsTrue(fileSystem.FileExists(configPath));
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(fileSystem.File.ReadAllText(configPath), out RuntimeConfig? config));
+            Assert.IsNotNull(config);
+            Assert.IsNotNull(config.Runtime);
+            Assert.IsNull(config.Runtime.Telemetry);
+
+            // Add OpenTelemetry
+            bool isSuccess = ConfigGenerator.TryAddTelemetry(
+                new AddTelemetryOptions(openTelemetryEndpoint: endpoint, openTelemetryEnabled: isTelemetryEnabled, config: configPath),
+                new FileSystemRuntimeConfigLoader(fileSystem),
+                fileSystem);
+
+            // Assert after adding OpenTelemetry
+            Assert.AreEqual(expectSuccess, isSuccess);
+            if (expectSuccess)
+            {
+                Assert.IsTrue(fileSystem.FileExists(configPath));
+                Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(fileSystem.File.ReadAllText(configPath), out config));
+                Assert.IsNotNull(config);
+                Assert.IsNotNull(config.Runtime);
+                Assert.IsNotNull(config.Runtime.Telemetry);
+                TelemetryOptions telemetryOptions = config.Runtime.Telemetry;
+                Assert.IsNotNull(telemetryOptions.OpenTelemetry);
+                Assert.AreEqual(isTelemetryEnabled is CliBool.True ? true : false, telemetryOptions.OpenTelemetry.Enabled);
+                Assert.AreEqual(endpoint, telemetryOptions.OpenTelemetry.Endpoint);
+            }
+        }
+
+        /// <summary>
+        /// Test to verify when Telemetry section is present in the config
+        /// It should add OpenTelemetry if telemetry section is empty
+        /// or overwrite the existing OpenTelemetry with the given OpenTelemetry options.
+        /// </summary>
+        [DataTestMethod]
+        [DataRow(true, DisplayName = "Add OpenTelemetry when telemetry section is empty.")]
+        [DataRow(false, DisplayName = "Overwrite OpenTelemetry when telemetry section already exists.")]
+        public void TestAddOpenTelemetryWhenTelemetryAlreadyExists(bool isEmptyTelemetry)
+        {
+            MockFileSystem fileSystem = FileSystemUtils.ProvisionMockFileSystem();
+            string configPath = "test-opentelemetry-config.json";
+            string runtimeSection = isEmptyTelemetry ? RUNTIME_SECTION_WITH_EMPTY_TELEMETRY_SECTION : RUNTIME_SECTION_WITH_OPEN_TELEMETRY_SECTION;
+            string configData = $"{{{SAMPLE_SCHEMA_DATA_SOURCE},{runtimeSection}}}";
+            fileSystem.AddFile(configPath, new MockFileData(configData));
+
+            // Initial State
+            Assert.IsTrue(fileSystem.FileExists(configPath));
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(fileSystem.File.ReadAllText(configPath), out RuntimeConfig? config));
+            Assert.IsNotNull(config);
+            Assert.IsNotNull(config.Runtime);
+            Assert.IsNotNull(config.Runtime.Telemetry);
+
+            if (isEmptyTelemetry)
+            {
+                Assert.IsNull(config.Runtime.Telemetry.OpenTelemetry);
+            }
+            else
+            {
+                Assert.IsNotNull(config.Runtime.Telemetry.OpenTelemetry);
+                Assert.AreEqual(true, config.Runtime.Telemetry.OpenTelemetry.Enabled);
+                Assert.AreEqual("http://localhost:4317", config.Runtime.Telemetry.OpenTelemetry.Endpoint);
+            }
+
+            // Add OpenTelemetry
+            bool isSuccess = ConfigGenerator.TryAddTelemetry(
+                new AddTelemetryOptions(openTelemetryEndpoint: "http://localhost:4318", openTelemetryEnabled: CliBool.False, config: configPath),
+                new FileSystemRuntimeConfigLoader(fileSystem),
+                fileSystem);
+
+            // Assert after adding OpenTelemetry
+            Assert.IsTrue(isSuccess);
+            Assert.IsTrue(fileSystem.FileExists(configPath));
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(fileSystem.File.ReadAllText(configPath), out config));
+            Assert.IsNotNull(config);
+            Assert.IsNotNull(config.Runtime);
+            Assert.IsNotNull(config.Runtime.Telemetry);
+            Assert.IsNotNull(config.Runtime.Telemetry.OpenTelemetry);
+            Assert.IsFalse(config.Runtime.Telemetry.OpenTelemetry.Enabled);
+            Assert.AreEqual("http://localhost:4318", config.Runtime.Telemetry.OpenTelemetry.Endpoint);
+        }
+
+        /// <summary>
+        /// Generates a JSON string representing a runtime section of the config, with a customizable telemetry section.
+        /// </summary>
+        private static string GenerateRuntimeSection(string telemetrySection)
+        {
+            return $@"
+                ""runtime"": {{
+                    ""rest"": {{
+                        ""path"": ""/api"",
+                        ""enabled"": false
+                    }},
+                    ""graphql"": {{
+                        ""path"": ""/graphql"",
+                        ""enabled"": false,
+                        ""allow-introspection"": true
+                    }},
+                    ""host"": {{
+                        ""mode"": ""development"",
+                        ""cors"": {{
+                            ""origins"": [],
+                            ""allow-credentials"": false
+                        }},
+                        ""authentication"": {{
+                            ""provider"": ""StaticWebApps""
+                        }}
+                    }},
+                    {telemetrySection}
+                }},
+                ""entities"": {{}}";
+        }
+
+        /// <summary>
+        /// Represents a JSON string for the telemetry section of the config, with Application Insights enabled and a specified connection string.
+        /// </summary>
+        private const string TELEMETRY_SECTION_WITH_OPEN_TELEMETRY = @"
+            ""telemetry"": {
+                ""open-telemetry"": {
+                    ""enabled"": true,
+                    ""endpoint"": ""http://localhost:4317""
+                }
+            }";
+
+        /// <summary>
+        /// Represents a JSON string for the empty telemetry section of the config.
+        /// </summary>
+        private const string EMPTY_TELEMETRY_SECTION = @"
+            ""telemetry"": {}";
+    }
+}

src/Cli.Tests/AddTelemetryTests.cs

@@ -45,7 +45,7 @@ public void TestAddApplicationInsightsTelemetry(CliBool isTelemetryEnabled, stri
 
             // Add Telemetry
             bool isSuccess = ConfigGenerator.TryAddTelemetry(
-                new AddTelemetryOptions(appInsightsConnString, isTelemetryEnabled, configPath),
+                new AddTelemetryOptions(appInsightsConnString: appInsightsConnString, appInsightsEnabled: isTelemetryEnabled, config: configPath),
                 new FileSystemRuntimeConfigLoader(fileSystem),
                 fileSystem);
 
@@ -101,7 +101,7 @@ public void TestAddAppInsightsTelemetryWhenTelemetryAlreadyExists(bool isEmptyTe
 
             // Add Telemetry
             bool isSuccess = ConfigGenerator.TryAddTelemetry(
-                new AddTelemetryOptions("InstrumentationKey=11111-1111-111-11-1", CliBool.False, configPath),
+                new AddTelemetryOptions(appInsightsConnString: "InstrumentationKey=11111-1111-111-11-1", appInsightsEnabled: CliBool.False, config: configPath),
                 new FileSystemRuntimeConfigLoader(fileSystem),
                 fileSystem);
 
@@ -119,7 +119,7 @@ public void TestAddAppInsightsTelemetryWhenTelemetryAlreadyExists(bool isEmptyTe
 
         /// <summary>
         /// Generates a JSON string representing a runtime section of the config, with a customizable telemetry section.
-        /// </summary> 
+        /// </summary>
         private static string GenerateRuntimeSection(string telemetrySection)
         {
             return $@"

src/Cli.Tests/EndToEndTests.cs

@@ -291,8 +291,8 @@ public void TestAddTelemetry(string? appInsightsEnabled, string appInsightsConnS
         Assert.IsNotNull(updatedConfig.Runtime.Telemetry);
         Assert.IsNotNull(updatedConfig.Runtime.Telemetry.ApplicationInsights);
 
-        // if --app-insights-enabled is not provided, it will default to true
-        Assert.AreEqual(appInsightsEnabled is null ? true : Boolean.Parse(appInsightsEnabled), updatedConfig.Runtime.Telemetry.ApplicationInsights.Enabled);
+        // if --app-insights-enabled is not provided, it will default to false
+        Assert.AreEqual(appInsightsEnabled is null ? false : Boolean.Parse(appInsightsEnabled), updatedConfig.Runtime.Telemetry.ApplicationInsights.Enabled);
         Assert.AreEqual("InstrumentationKey=00000000", updatedConfig.Runtime.Telemetry.ApplicationInsights.ConnectionString);
     }
 

src/Cli/Commands/AddTelemetryOptions.cs

@@ -8,6 +8,7 @@
 using Cli.Constants;
 using CommandLine;
 using Microsoft.Extensions.Logging;
+using OpenTelemetry.Exporter;
 using static Cli.Utils;
 
 namespace Cli.Commands
@@ -18,20 +19,54 @@ namespace Cli.Commands
     [Verb("add-telemetry", isDefault: false, HelpText = "Add telemetry for Data Api builder Application", Hidden = false)]
     public class AddTelemetryOptions : Options
     {
-        public AddTelemetryOptions(string appInsightsConnString, CliBool appInsightsEnabled, string? config) : base(config)
+        public AddTelemetryOptions(
+            string? appInsightsConnString = null,
+            CliBool? appInsightsEnabled = null,
+            string? openTelemetryEndpoint = null,
+            CliBool? openTelemetryEnabled = null,
+            string? openTelemetryHeaders = null,
+            OtlpExportProtocol? openTelemetryExportProtocol = null,
+            string? openTelemetryServiceName = null,
+            string? config = null) : base(config)
         {
             AppInsightsConnString = appInsightsConnString;
             AppInsightsEnabled = appInsightsEnabled;
+            OpenTelemetryEndpoint = openTelemetryEndpoint;
+            OpenTelemetryEnabled = openTelemetryEnabled;
+            OpenTelemetryHeaders = openTelemetryHeaders;
+            OpenTelemetryExportProtocol = openTelemetryExportProtocol;
+            OpenTelemetryServiceName = openTelemetryServiceName;
         }
 
         // Connection string for the Application Insights resource to which telemetry data should be sent.
-        // This option  is required and must be provided with a valid connection string.
-        [Option("app-insights-conn-string", Required = true, HelpText = "Connection string for the Application Insights resource for telemetry data")]
-        public string AppInsightsConnString { get; }
+        // This option is required and must be provided with a valid connection string when using app insights.
+        [Option("app-insights-conn-string", Required = false, HelpText = "Connection string for the Application Insights resource for telemetry data")]
+        public string? AppInsightsConnString { get; }
 
-        // To specify whether Application Insights telemetry should be enabled. This flag is optional and default value is true.
-        [Option("app-insights-enabled", Default = CliBool.True, Required = false, HelpText = "(Default: true) Enable/Disable Application Insights")]
-        public CliBool AppInsightsEnabled { get; }
+        // To specify whether Application Insights telemetry should be enabled. This flag is optional and default value is false.
+        [Option("app-insights-enabled", Default = CliBool.False, Required = false, HelpText = "(Default: false) Enable/Disable Application Insights")]
+        public CliBool? AppInsightsEnabled { get; }
+
+        // Connection string for the Open Telemetry resource to which telemetry data should be sent.
+        // This option is required and must be provided with a valid connection string when using open telemetry.
+        [Option("otel-endpoint", Required = false, HelpText = "Endpoint for Open Telemetry for telemetry data")]
+        public string? OpenTelemetryEndpoint { get; }
+
+        // To specify whether Open Telemetry telemetry should be enabled. This flag is optional and default value is false.
+        [Option("otel-enabled", Default = CliBool.False, Required = false, HelpText = "(Default: false) Enable/Disable OTEL")]
+        public CliBool? OpenTelemetryEnabled { get; }
+
+        // Headers for the Open Telemetry resource to which telemetry data should be sent.
+        [Option("otel-headers", Required = false, HelpText = "Headers for Open Telemetry for telemetry data")]
+        public string? OpenTelemetryHeaders { get; }
+
+        // Specify the Open Telemetry protocol. This flag is optional and default value is grpc.
+        [Option("otel-protocol", Default = OtlpExportProtocol.Grpc, Required = false, HelpText = "(Default: grpc) Accepted: grpc/httpprotobuf")]
+        public OtlpExportProtocol? OpenTelemetryExportProtocol { get; }
+
+        // Service Name for the Open Telemetry resource to which telemetry data should be sent. This flag is optional and default value is dab.
+        [Option("otel-service-name", Default = "dab", Required = false, HelpText = "(Default: dab) Headers for Open Telemetry for telemetry data")]
+        public string? OpenTelemetryServiceName { get; }
 
         public int Handler(ILogger logger, FileSystemRuntimeConfigLoader loader, IFileSystem fileSystem)
         {

src/Cli/ConfigGenerator.cs

@@ -619,8 +619,8 @@ private static bool TryUpdateConfiguredDataSourceOptions(
 
         /// <summary>
         /// Adds CosmosDB-specific options to the provided database options dictionary.
-        /// This method checks if the CosmosDB-specific options (database, container, and schema) are provided in the 
-        /// configuration options. If they are, it converts their names using the provided naming policy and adds them 
+        /// This method checks if the CosmosDB-specific options (database, container, and schema) are provided in the
+        /// configuration options. If they are, it converts their names using the provided naming policy and adds them
         /// to the database options dictionary.
         /// </summary>
         /// <param name="dbOptions">The dictionary to which the CosmosDB-specific options will be added.</param>
@@ -1939,17 +1939,40 @@ public static bool TryAddTelemetry(AddTelemetryOptions options, FileSystemRuntim
                 return false;
             }
 
-            if (string.IsNullOrWhiteSpace(options.AppInsightsConnString))
+            if (options.AppInsightsEnabled is CliBool.True && string.IsNullOrWhiteSpace(options.AppInsightsConnString))
             {
                 _logger.LogError("Invalid Application Insights connection string provided.");
                 return false;
             }
 
+            if (options.OpenTelemetryEnabled is CliBool.True && string.IsNullOrWhiteSpace(options.OpenTelemetryEndpoint))
+            {
+                _logger.LogError("Invalid OTEL endpoint provided.");
+                return false;
+            }
+
             ApplicationInsightsOptions applicationInsightsOptions = new(
                 Enabled: options.AppInsightsEnabled is CliBool.True ? true : false,
                 ConnectionString: options.AppInsightsConnString
             );
 
+            OpenTelemetryOptions openTelemetryOptions = new(
+                Enabled: options.OpenTelemetryEnabled is CliBool.True ? true : false,
+                Endpoint: options.OpenTelemetryEndpoint,
+                Headers: options.OpenTelemetryHeaders,
+                ExporterProtocol: options.OpenTelemetryExportProtocol,
+                ServiceName: options.OpenTelemetryServiceName
+            );
+
+            runtimeConfig = runtimeConfig with
+            {
+                Runtime = runtimeConfig.Runtime with
+                {
+                    Telemetry = runtimeConfig.Runtime.Telemetry is null
+                        ? new TelemetryOptions(ApplicationInsights: applicationInsightsOptions, OpenTelemetry: openTelemetryOptions)
+                        : runtimeConfig.Runtime.Telemetry with { ApplicationInsights = applicationInsightsOptions, OpenTelemetry = openTelemetryOptions }
+                }
+            };
             runtimeConfig = runtimeConfig with
             {
                 Runtime = runtimeConfig.Runtime with

src/Config/Azure.DataApiBuilder.Config.csproj

@@ -25,6 +25,7 @@
       <PackageReference Include="System.IdentityModel.Tokens.Jwt" />
       <PackageReference Include="Humanizer" />
       <PackageReference Include="Npgsql" />
+      <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"  />
   </ItemGroup>
 
     <ItemGroup>

src/Config/ObjectModel/OpenTelemetryOptions.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using OpenTelemetry.Exporter;
+
+namespace Azure.DataApiBuilder.Config.ObjectModel;
+
+/// <summary>
+/// Represents the options for configuring Open Telemetry.
+/// </summary>
+public record OpenTelemetryOptions(bool Enabled = false, string? Endpoint = null, string? Headers = null, OtlpExportProtocol? ExporterProtocol = null, string? ServiceName = null)
+{ }