[AzureMonitorLiveMetrics] Add known fields to MonitoringDataPoint (Az…

…ure#39861)

* Add known feilds to MonitoringDataPoint

* Sdk Utils + event source.

* PR feedback

* s_liveMetricNameMapping check

* fix identation.

sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/SdkVersionUtils.cs

@@ -5,7 +5,12 @@
 using System.Globalization;
 using System.Linq;
 using System.Reflection;
+#if AZURE_MONITOR_EXPORTER
 using Azure.Monitor.OpenTelemetry.Exporter.Internals.Diagnostics;
+#elif LIVE_METRICS_EXPORTER
+using Azure.Monitor.OpenTelemetry.LiveMetrics;
+using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals.Diagnostics;
+#endif
 using OpenTelemetry;
 
 namespace Azure.Monitor.OpenTelemetry.Exporter.Internals
@@ -53,15 +58,23 @@ internal static bool IsDistro
 
                 if (shortVersion.Length > 20)
                 {
+#if AZURE_MONITOR_EXPORTER
                     AzureMonitorExporterEventSource.Log.VersionStringUnexpectedLength(type.Name, versionString);
+#elif LIVE_METRICS_EXPORTER
+                    LiveMetricsExporterEventSource.Log.VersionStringUnexpectedLength(type.Name, versionString);
+#endif
                     return shortVersion.Substring(0, 20);
                 }
 
                 return shortVersion;
             }
             catch (Exception ex)
             {
+#if AZURE_MONITOR_EXPORTER
                 AzureMonitorExporterEventSource.Log.ErrorInitializingPartOfSdkVersion(type.Name, ex);
+#elif LIVE_METRICS_EXPORTER
+                LiveMetricsExporterEventSource.Log.ErrorInitializingPartOfSdkVersion(type.Name, ex);
+#endif
                 return null;
             }
         }
@@ -73,7 +86,11 @@ internal static bool IsDistro
                 string? sdkVersionPrefix = !string.IsNullOrWhiteSpace(SdkVersionPrefix) ? $"{SdkVersionPrefix}_" : null;
                 string? dotnetSdkVersion = GetVersion(typeof(object));
                 string? otelSdkVersion = GetVersion(typeof(Sdk));
+#if AZURE_MONITOR_EXPORTER
                 string? extensionVersion = GetVersion(typeof(AzureMonitorTraceExporter));
+#elif LIVE_METRICS_EXPORTER
+                string? extensionVersion = GetVersion(typeof(LiveMetricsExporter));
+#endif
 
                 if (IsDistro)
                 {
@@ -84,7 +101,11 @@ internal static bool IsDistro
             }
             catch (Exception ex)
             {
+#if AZURE_MONITOR_EXPORTER
                 AzureMonitorExporterEventSource.Log.SdkVersionCreateFailed(ex);
+#elif LIVE_METRICS_EXPORTER
+                LiveMetricsExporterEventSource.Log.SdkVersionCreateFailed(ex);
+#endif
                 return null;
             }
         }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Azure.Monitor.OpenTelemetry.LiveMetrics.csproj

@@ -20,11 +20,14 @@
   <ItemGroup>
     <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\ConnectionString\*.cs" LinkBase="Internals\ConnectionString" />
     <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\Platform\*.cs" LinkBase="Internals\Platform" />
-    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\AksResourceProcessor.cs" Link="AksResourceProcessor.cs" />
+    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\AksResourceProcessor.cs" Link="Internals" />
     <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\NullableAttributes.cs" LinkBase="Internals" />
     <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\ExceptionExtensions.cs" LinkBase="Internals" />
-    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\SchemaConstants.cs" Link="SchemaConstants.cs" />
-    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\SemanticConventions.cs" Link="SemanticConventions.cs" />
+    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\SchemaConstants.cs" Link="Internals" />
+    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\SdkVersionUtils.cs" Link="Internals" />
+    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\SemanticConventions.cs" Link="Internals" />
+    <Compile Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Internals\StringExtensions.cs" Link="Internals" />
+
   </ItemGroup>
 
   <!-- Shared source from Azure.Core -->

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs

@@ -65,5 +65,44 @@ public void FailedToReadEnvironmentVariables(Exception ex)
 
         [Event(2, Message = "Failed to read environment variables due to an exception. This may prevent the Exporter from initializing. {0}", Level = EventLevel.Warning)]
         public void FailedToReadEnvironmentVariables(string errorMessage) => WriteEvent(2, errorMessage);
+
+        [NonEvent]
+        public void AccessingEnvironmentVariableFailedWarning(string environmentVariable, Exception ex)
+        {
+            if (IsEnabled(EventLevel.Warning))
+            {
+                AccessingEnvironmentVariableFailedWarning(environmentVariable, ex.FlattenException().ToInvariantString());
+            }
+        }
+
+        [Event(3, Message = "Accessing environment variable - {0} failed with exception: {1}.", Level = EventLevel.Warning)]
+        public void AccessingEnvironmentVariableFailedWarning(string environmentVariable, string exceptionMessage) => WriteEvent(3, environmentVariable, exceptionMessage);
+
+        [NonEvent]
+        public void SdkVersionCreateFailed(Exception ex)
+        {
+            if (IsEnabled(EventLevel.Warning))
+            {
+                SdkVersionCreateFailed(ex.FlattenException().ToInvariantString());
+            }
+        }
+
+        [Event(4, Message = "Failed to create an SDK version due to an exception. Not user actionable. {0}", Level = EventLevel.Warning)]
+        public void SdkVersionCreateFailed(string exceptionMessage) => WriteEvent(4, exceptionMessage);
+
+        [Event(5, Message = "Version string exceeds expected length. This is only for internal telemetry and can safely be ignored. Type Name: {0}. Version: {1}", Level = EventLevel.Verbose)]
+        public void VersionStringUnexpectedLength(string typeName, string value) => WriteEvent(5, typeName, value);
+
+        [NonEvent]
+        public void ErrorInitializingPartOfSdkVersion(string typeName, Exception ex)
+        {
+            if (IsEnabled(EventLevel.Warning))
+            {
+                ErrorInitializingPartOfSdkVersion(typeName, ex.FlattenException().ToInvariantString());
+            }
+        }
+
+        [Event(6, Message = "Failed to get Type version while initialize SDK version due to an exception. Not user actionable. Type: {0}. {1}", Level = EventLevel.Warning)]
+        public void ErrorInitializingPartOfSdkVersion(string typeName, string exceptionMessage) => WriteEvent(6, typeName, exceptionMessage);
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/GuidExtensions.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Globalization;
+
+namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Internals
+{
+    internal static class GuidExtensions
+    {
+        /// <summary>
+        /// Overload for Guid.ToString().
+        /// </summary>
+        /// <remarks>
+        /// This method encapsulates the language switch for NetStandard and NetFramework and resolves the error "The behavior of guid.ToString() could vary based on the current user's locale settings"
+        /// </remarks>
+        public static string ToStringInvariant(this Guid guid, string format)
+        {
+            return guid.ToString(format, CultureInfo.InvariantCulture);
+        }
+    }
+}

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricConstants.cs

@@ -17,5 +17,17 @@ internal static class LiveMetricConstants
         internal const string ExceptionsPerSecondInstrumentName = "ExceptionsPerSecondLiveMetric";
         internal const string MemoryCommittedBytesInstrumentName = "CommittedBytesLiveMetric";
         internal const string ProcessorTimeInstrumentName = "ProcessorTimeBytesLiveMetric";
+
+        internal const string RequestDurationMetricIdValue = @"\ApplicationInsights\Request Duration";
+        internal const string RequestsPerSecondMetricIdValue = @"\ApplicationInsights\Requests/Sec";
+        internal const string RequestsSucceededPerSecondMetricIdValue = @"\ApplicationInsights\Requests Succeeded/Sec";
+        internal const string RequestsFailedPerSecondMetricIdValue = @"\ApplicationInsights\Requests Failed/Sec";
+        internal const string DependencyDurationMetricIdValue = @"\ApplicationInsights\Dependency Call Duration";
+        internal const string DependenciesPerSecondMetricIdValue = @"\ApplicationInsights\Dependency Calls/Sec";
+        internal const string DependencySucceededPerSecondMetricIdValue = @"\ApplicationInsights\Dependency Calls Succeeded/Sec";
+        internal const string DependencyFailedPerSecondMetricIdValue = @"\ApplicationInsights\Dependency Calls Failed/Sec";
+        internal const string ExceptionsPerSecondMetricIdValue = @"\ApplicationInsights\Exceptions/Sec";
+        internal const string MemoryCommittedBytesMetricIdValue = @"\Memory\Committed Bytes";
+        internal const string ProcessorTimeMetricIdValue = @"\Processor(_Total)\% Processor Time";
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExporter.cs

@@ -1,6 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System;
+using Azure.Monitor.OpenTelemetry.Exporter.Internals;
+using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals;
+using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals.Diagnostics;
 using Azure.Monitor.OpenTelemetry.LiveMetrics.Models;
 using OpenTelemetry;
 using OpenTelemetry.Metrics;
@@ -9,6 +13,11 @@ namespace Azure.Monitor.OpenTelemetry.LiveMetrics
 {
     internal sealed class LiveMetricsExporter : BaseExporter<Metric>
     {
+        public const int CurrentInvariantVersion = 5;
+        private const string WebSiteEnvironmentVariable = "WEBSITE_SITE_NAME";
+        private const string WebSiteIsolationEnvironmentVariable = "WEBSITE_ISOLATION";
+        internal static bool? s_isAzureWebApp = null;
+        private const string WebSiteIsolationHyperV = "hyperv";
         private readonly string _instrumentationKey;
         private readonly DoubleBuffer _doubleBuffer;
         private LiveMetricsResource? _resource;
@@ -24,14 +33,70 @@ public LiveMetricsExporter(DoubleBuffer doubleBuffer, LiveMetricsExporterOptions
 
         public override ExportResult Export(in Batch<Metric> batch)
         {
-            MonitoringDataPoint monitoringDataPoint = new MonitoringDataPoint();
             DocumentBuffer filledBuffer = _doubleBuffer.FlipDocumentBuffers();
+            MonitoringDataPoint monitoringDataPoint = new()
+            {
+                Version = SdkVersionUtils.s_sdkVersion.Truncate(SchemaConstants.Tags_AiInternalSdkVersion_MaxLength),
+                InvariantVersion = CurrentInvariantVersion,
+                Instance = MetricResource?.RoleInstance,
+                RoleName = MetricResource?.RoleName,
+                MachineName = Environment.MachineName,
+                StreamId = GetStreamId(),
+                Timestamp = DateTime.UtcNow,
+                //TODO: Provide feedback to service team to get this removed, it not a part of AI SDK.
+                // TransmissionTime = DateTime.UtcNow,
+                IsWebApp = IsWebAppRunningInAzure(),
+                PerformanceCollectionSupported = false,
+                // AI SDK relies on PerformanceCounter to collect CPU and Memory metrics.
+                // Follow up with service team to get this removed for OTEL based live metrics.
+                // TopCpuProcesses = null,
+                // TODO: Configuration errors are thrown when filter is applied.
+                // CollectionConfigurationErrors = null,
+            };
 
             foreach (var item in filledBuffer.ReadAllAndClear())
             {
                 monitoringDataPoint.Documents.Add(item);
             }
 
+            foreach (var metric in batch)
+            {
+                if (!LiveMetricsExtractionProcessor.s_liveMetricNameMapping.TryGetValue(metric.Name, out var metricName))
+                {
+                    continue;
+                }
+
+                var liveMetricPoint = new Models.MetricPoint
+                {
+                    Name = metricName
+                };
+
+                foreach (ref readonly var metricPoint in metric.GetMetricPoints())
+                {
+                    switch (metric.MetricType)
+                    {
+                        case MetricType.LongSum:
+                            // potential for minor precision loss implicitly going from long->double
+                            // see: https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/builtin-types/numeric-conversions#implicit-numeric-conversions
+                            liveMetricPoint.Value = metricPoint.GetSumLong();
+                            liveMetricPoint.Weight = 1;
+                            break;
+                        case MetricType.Histogram:
+                            long histogramCount = metricPoint.GetHistogramCount();
+                            // When you convert double to float, the double value is rounded to the nearest float value.
+                            // If the double value is too small or too large to fit into the float type, the result is zero or infinity.
+                            // see: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/builtin-types/numeric-conversions#explicit-numeric-conversions
+                            liveMetricPoint.Value = (float)(metricPoint.GetHistogramSum() / histogramCount);
+                            liveMetricPoint.Weight = histogramCount <= int.MaxValue ? (int?)histogramCount : null;
+                            break;
+                    }
+
+                    monitoringDataPoint.Metrics.Add(liveMetricPoint);
+                }
+            }
+
+            // Send data.
+
             return ExportResult.Success;
         }
 
@@ -48,5 +113,38 @@ protected override void Dispose(bool disposing)
 
             base.Dispose(disposing);
         }
+
+        /// <summary>
+        /// Searches for the environment variable specific to Azure Web App.
+        /// </summary>
+        /// <returns>Boolean, which is true if the current application is an Azure Web App.</returns>
+        internal static bool? IsWebAppRunningInAzure()
+        {
+            if (!s_isAzureWebApp.HasValue)
+            {
+                try
+                {
+                    // Presence of "WEBSITE_SITE_NAME" indicate web apps.
+                    // "WEBSITE_ISOLATION"!="hyperv" indicate premium containers. In this case, perf counters
+                    // can be read using regular mechanism and hence this method retuns false for
+                    // premium containers.
+                    // TODO: switch to platform. Not necessary for POC.
+                    s_isAzureWebApp = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(WebSiteEnvironmentVariable)) &&
+                                    Environment.GetEnvironmentVariable(WebSiteIsolationEnvironmentVariable) != WebSiteIsolationHyperV;
+                }
+                catch (Exception ex)
+                {
+                    LiveMetricsExporterEventSource.Log.AccessingEnvironmentVariableFailedWarning(WebSiteEnvironmentVariable, ex);
+                    return false;
+                }
+            }
+
+            return s_isAzureWebApp;
+        }
+
+        private static string GetStreamId()
+        {
+            return Guid.NewGuid().ToStringInvariant("N");
+        }
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtractionProcessor.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
 using System.Globalization;
@@ -30,6 +31,21 @@ internal sealed class LiveMetricsExtractionProcessor : BaseProcessor<Activity>
         private readonly Counter<long> _exceptionsPerSecond;
         private readonly DoubleBuffer _doubleBuffer;
 
+        internal static readonly IReadOnlyDictionary<string, string> s_liveMetricNameMapping = new Dictionary<string, string>()
+        {
+            [LiveMetricConstants.RequestDurationInstrumentName] = LiveMetricConstants.RequestDurationMetricIdValue,
+            [LiveMetricConstants.RequestsInstrumentName] = LiveMetricConstants.RequestsPerSecondMetricIdValue,
+            [LiveMetricConstants.RequestsSucceededPerSecondInstrumentName] = LiveMetricConstants.RequestsSucceededPerSecondMetricIdValue,
+            [LiveMetricConstants.RequestsFailedPerSecondInstrumentName] = LiveMetricConstants.RequestsFailedPerSecondMetricIdValue,
+            [LiveMetricConstants.DependencyDurationInstrumentName] = LiveMetricConstants.DependencyDurationMetricIdValue,
+            [LiveMetricConstants.DependencyInstrumentName] = LiveMetricConstants.DependenciesPerSecondMetricIdValue,
+            [LiveMetricConstants.DependencySucceededPerSecondInstrumentName] = LiveMetricConstants.DependencySucceededPerSecondMetricIdValue,
+            [LiveMetricConstants.DependencyFailedPerSecondInstrumentName] = LiveMetricConstants.DependencyFailedPerSecondMetricIdValue,
+            [LiveMetricConstants.ExceptionsPerSecondInstrumentName] = LiveMetricConstants.ExceptionsPerSecondMetricIdValue,
+            [LiveMetricConstants.MemoryCommittedBytesInstrumentName] = LiveMetricConstants.MemoryCommittedBytesMetricIdValue,
+            [LiveMetricConstants.ProcessorTimeInstrumentName] = LiveMetricConstants.ProcessorTimeMetricIdValue,
+        };
+
         internal LiveMetricsResource? LiveMetricsResource => _resource ??= ParentProvider?.GetResource().CreateAzureMonitorResource();
 
         internal LiveMetricsExtractionProcessor(DoubleBuffer doubleBuffer, LiveMetricsExporter liveMetricExporter)