[Tracer] Add optional tags to OTLP trace metrics

tracer/src/Datadog.Trace/Agent/OtlpSpanStatsSerializer.cs

@@ -6,6 +6,7 @@
 #nullable enable
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Text;
 using Datadog.Trace.Logging;
@@ -141,6 +142,11 @@ private static void WriteResourceJson(JsonTextWriter writer, StatsBuffer buffer)
 
             WriteStringKvJson(writer, "datadog.runtime_id", Tracer.RuntimeId);
 
+            if (details.ProcessTags?.TagsList is { Count: > 0 } processTags)
+            {
+                WriteStringArrayKvJson(writer, "datadog.process_tags", processTags);
+            }
+
             writer.WriteEndArray();
             writer.WriteEndObject();
         }
@@ -255,6 +261,16 @@ private static void WriteDataPointJson(
                 WriteStringKvJson(writer, "datadog.svc_src", key.ServiceSource);
             }
 
+            if (bucket.Otlp.PeerTags.Count > 0)
+            {
+                WriteStringArrayKvJson(writer, "datadog.peer_tags", bucket.Otlp.PeerTags);
+            }
+
+            foreach (var tag in bucket.Otlp.AdditionalMetricTags)
+            {
+                WriteStringKvJson(writer, tag.Key, tag.Value);
+            }
+
             writer.WriteEndArray();
 
             // uint64 fields are encoded as strings in proto3 JSON
@@ -343,6 +359,31 @@ private static void WriteBoolKvJson(JsonTextWriter writer, string key, bool valu
             writer.WriteEndObject();
         }
 
+        private static void WriteStringArrayKvJson(JsonTextWriter writer, string key, List<string> values)
+        {
+            writer.WriteStartObject();
+            writer.WritePropertyName("key");
+            writer.WriteValue(key);
+            writer.WritePropertyName("value");
+            writer.WriteStartObject();
+            writer.WritePropertyName("arrayValue");
+            writer.WriteStartObject();
+            writer.WritePropertyName("values");
+            writer.WriteStartArray();
+            foreach (var value in values)
+            {
+                writer.WriteStartObject();
+                writer.WritePropertyName("stringValue");
+                writer.WriteValue(value);
+                writer.WriteEndObject();
+            }
+
+            writer.WriteEndArray();
+            writer.WriteEndObject();
+            writer.WriteEndObject();
+            writer.WriteEndObject();
+        }
+
         private static byte[] SerializeResourceMetrics(StatsBuffer buffer, long bucketDurationNs)
         {
             using var stream = new MemoryStream(512);
@@ -387,6 +428,11 @@ private static byte[] SerializeResource(StatsBuffer buffer)
 
             WriteAttribute(writer, "datadog.runtime_id", Tracer.RuntimeId);
 
+            if (details.ProcessTags?.TagsList is { Count: > 0 } processTags)
+            {
+                WriteStringArrayAttribute(writer, "datadog.process_tags", processTags);
+            }
+
             writer.Flush();
             return stream.ToArray();
         }
@@ -509,6 +555,16 @@ private static byte[] SerializeDataPoint(
                 WriteAttribute(writer, "datadog.svc_src", key.ServiceSource, FieldNumbers.HistogramDataPointAttributes);
             }
 
+            if (bucket.Otlp.PeerTags.Count > 0)
+            {
+                WriteStringArrayAttribute(writer, "datadog.peer_tags", bucket.Otlp.PeerTags, FieldNumbers.HistogramDataPointAttributes);
+            }
+
+            foreach (var tag in bucket.Otlp.AdditionalMetricTags)
+            {
+                WriteAttribute(writer, tag.Key, tag.Value, FieldNumbers.HistogramDataPointAttributes);
+            }
+
             WriteTag(writer, FieldNumbers.HistogramDataPointStartTimeUnixNano, WireTypeFixed64);
             writer.Write(startTimeUnixNano);
 
@@ -759,6 +815,41 @@ private static byte[] SerializeBoolKeyValue(string key, bool value)
             return stream.ToArray();
         }
 
+        private static void WriteStringArrayAttribute(BinaryWriter writer, string key, List<string> values, int fieldNumber = FieldNumbers.Attributes)
+        {
+            var arraySize = 0;
+            foreach (var value in values)
+            {
+                var valueSize = GetUtf8StringFieldSize(AnyValueFieldNumbers.StringValue, EncodingHelpers.Utf8NoBom.GetByteCount(value));
+                arraySize += GetLengthDelimitedFieldSize(ArrayValueFieldNumbers.Values, valueSize);
+            }
+
+            WriteStringArrayAttributeHeader(writer, key, arraySize, fieldNumber);
+            foreach (var value in values)
+            {
+                var valueSize = GetUtf8StringFieldSize(AnyValueFieldNumbers.StringValue, EncodingHelpers.Utf8NoBom.GetByteCount(value));
+                WriteTag(writer, ArrayValueFieldNumbers.Values, WireTypeLengthDelimited);
+                WriteVarInt(writer, valueSize);
+                WriteStringField(writer, AnyValueFieldNumbers.StringValue, value);
+            }
+        }
+
+        private static void WriteStringArrayAttributeHeader(BinaryWriter writer, string key, int arraySize, int fieldNumber)
+        {
+            var anyValueSize = GetLengthDelimitedFieldSize(AnyValueFieldNumbers.ArrayValue, arraySize);
+            var keySize = EncodingHelpers.Utf8NoBom.GetByteCount(key);
+            var keyValueSize = GetUtf8StringFieldSize(FieldNumbers.Key, keySize)
+                             + GetLengthDelimitedFieldSize(FieldNumbers.Value, anyValueSize);
+
+            WriteTag(writer, fieldNumber, WireTypeLengthDelimited);
+            WriteVarInt(writer, keyValueSize);
+            WriteStringField(writer, FieldNumbers.Key, key);
+            WriteTag(writer, FieldNumbers.Value, WireTypeLengthDelimited);
+            WriteVarInt(writer, anyValueSize);
+            WriteTag(writer, AnyValueFieldNumbers.ArrayValue, WireTypeLengthDelimited);
+            WriteVarInt(writer, arraySize);
+        }
+
         private static void WriteStringField(BinaryWriter writer, int fieldNumber, string value)
         {
             if (!StringUtil.IsNullOrEmpty(value))
@@ -770,6 +861,23 @@ private static void WriteStringField(BinaryWriter writer, int fieldNumber, strin
             }
         }
 
+        private static int GetUtf8StringFieldSize(int fieldNumber, int valueLength)
+            => valueLength == 0 ? 0 : GetLengthDelimitedFieldSize(fieldNumber, valueLength);
+
+        private static int GetLengthDelimitedFieldSize(int fieldNumber, int valueLength)
+            => GetVarIntSize((fieldNumber << 3) | WireTypeLengthDelimited) + GetVarIntSize(valueLength) + valueLength;
+
+        private static int GetVarIntSize(int value)
+        {
+            var size = 1;
+            while ((value >>= 7) != 0)
+            {
+                size++;
+            }
+
+            return size;
+        }
+
         private static void WriteTag(BinaryWriter writer, int fieldNumber, int wireType)
             => WriteVarInt(writer, (fieldNumber << 3) | wireType);
 
@@ -835,6 +943,12 @@ private static class AnyValueFieldNumbers
             public const int StringValue = 1;
             public const int BoolValue = 2;
             public const int IntValue = 3;
+            public const int ArrayValue = 5;
+        }
+
+        private static class ArrayValueFieldNumbers
+        {
+            public const int Values = 1;
         }
     }
 }

tracer/src/Datadog.Trace/Agent/StatsAggregator.cs

@@ -40,6 +40,10 @@ internal sealed class StatsAggregator : IStatsAggregator
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<StatsAggregator>();
         private static readonly List<byte[]> EmptyTags = [];
         private static readonly List<byte[]> BlockedByTracerSentinelList = ["tracer_blocked_value"u8.ToArray()];
+        private static readonly List<string> EmptyOtlpPeerTags = [];
+        private static readonly List<string> BlockedOtlpPeerTags = [BlockedByTracerSentinel];
+        private static readonly List<KeyValuePair<string, string>> EmptyOtlpAdditionalTags = [];
+        private static readonly List<KeyValuePair<string, string>> BlockedOtlpAdditionalTags = [new(BlockedByTracerSentinel, string.Empty)];
 
         private static readonly StatsAggregationKey OverflowKey = new(
             resource: BlockedByTracerSentinel,
@@ -120,7 +124,10 @@ internal StatsAggregator(IApi api, TracerSettings settings, IDiscoveryService di
 
             // StatsAdditionalTags is already deduplicated, sorted, and capped.
             // Pre-encode the UTF-8 key prefixes so BuildKey can hash without per-call string encoding.
-            var additionalTagCount = settings.StatsAdditionalTags.Length;
+            var additionalTagNames = _isOtlp
+                                         ? settings.StatsAdditionalTags.Where(static tag => !IsBuiltInOtlpDataPointAttribute(tag)).ToArray()
+                                         : settings.StatsAdditionalTags;
+            var additionalTagCount = additionalTagNames.Length;
             if (additionalTagCount == 0)
             {
                 _additionalTagKeys = [];
@@ -130,7 +137,7 @@ internal StatsAggregator(IApi api, TracerSettings settings, IDiscoveryService di
                 _additionalTagKeys = new AdditionalTagKey[additionalTagCount];
                 for (var i = 0; i < additionalTagCount; i++)
                 {
-                    _additionalTagKeys[i] = new AdditionalTagKey(settings.StatsAdditionalTags[i]);
+                    _additionalTagKeys[i] = new AdditionalTagKey(additionalTagNames[i]);
                 }
             }
 
@@ -552,6 +559,34 @@ internal static List<byte[]> GetEncodedPeerTags(Span span, List<PeerTagKey> peer
             return result;
         }
 
+        private static List<string> GetOtlpPeerTags(Span span, List<PeerTagKey> peerTagKeys, in PeerTagResults results)
+        {
+            if (results.BaseService is not null)
+            {
+                return [string.Concat(Tags.BaseService, ":", results.BaseService)];
+            }
+
+            if (results.PeerTagCount == 0)
+            {
+                return EmptyOtlpPeerTags;
+            }
+
+            var result = new List<string>(results.PeerTagCount);
+            foreach (var peerTag in peerTagKeys)
+            {
+                var tagValue = span.GetTag(peerTag.Name);
+                if (string.IsNullOrEmpty(tagValue))
+                {
+                    continue;
+                }
+
+                tagValue = IpAddressObfuscationUtil.QuantizePeerIpAddresses(tagValue);
+                result.Add(string.Concat(peerTag.Name, ":", tagValue));
+            }
+
+            return result;
+        }
+
         /// <summary>
         /// Builds a "key:value" UTF-8 byte array by reusing the pre-encoded <paramref name="utf8KeyPrefix"/>
         /// (e.g. "tagKey:") and encoding only <paramref name="tagValue"/> directly into the destination
@@ -569,6 +604,23 @@ private static byte[] EncodeKeyValue(byte[] utf8KeyPrefix, string tagValue)
             return encoded;
         }
 
+        private static bool IsBuiltInOtlpDataPointAttribute(string key)
+            => key is "service.name"
+                   or "status.code"
+                   or "span.kind"
+                   or "span.name"
+                   or "http.request.method"
+                   or "http.response.status_code"
+                   or "http.route"
+                   or "rpc.response.status_code"
+                   or "datadog.operation.name"
+                   or "datadog.span.type"
+                   or "datadog.span.top_level"
+                   or "datadog.is_trace_root"
+                   or "datadog.origin"
+                   or "datadog.svc_src"
+                   or "datadog.peer_tags";
+
         /// <summary>
         /// Computes the FNV-64 hash of the span-derived additional tags present on the span, in the
         /// configured (pre-sorted) order. Values exceeding <see cref="AdditionalTagMaxValueLength"/>
@@ -649,6 +701,28 @@ private List<byte[]> GetEncodedAdditionalTags(Span span, in AdditionalTagResults
             return result;
         }
 
+        private List<KeyValuePair<string, string>> GetOtlpAdditionalTags(Span span, in AdditionalTagResults results)
+        {
+            if (results.TagCount == 0)
+            {
+                return EmptyOtlpAdditionalTags;
+            }
+
+            var result = new List<KeyValuePair<string, string>>(results.TagCount);
+            foreach (var tagKey in _additionalTagKeys)
+            {
+                var tagValue = span.GetTag(tagKey.Name);
+                if (string.IsNullOrEmpty(tagValue))
+                {
+                    continue;
+                }
+
+                result.Add(new(tagKey.Name, tagValue.Length > AdditionalTagMaxValueLength ? BlockedByTracerSentinel : tagValue));
+            }
+
+            return result;
+        }
+
         internal async Task Flush()
         {
             // Use a do/while loop to still flush once if _processExit is already completed (this makes testing easier)
@@ -763,15 +837,27 @@ private void AddToBuffer(Span span)
                 {
                     // Cold path: encode the peer tags and span-derived primary tags for storage in the new bucket.
                     // The overflow row carries no peer/additional tags; folded fields encode empty/masked.
-                    var encodedPeerTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.PeerTags) == StatsCardinalityLimitedFields.PeerTags
-                                              ? BlockedByTracerSentinelList
-                                              : GetEncodedPeerTags(span, peerTagKeys, in peerTagResults);
-
-                    var encodedAdditionalTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.AdditionalMetricTags) == StatsCardinalityLimitedFields.AdditionalMetricTags
-                                                    ? BlockedByTracerSentinelList
-                                                    : GetEncodedAdditionalTags(span, in additionalTagResults);
+                    if (_isOtlp)
+                    {
+                        var peerTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.PeerTags) == StatsCardinalityLimitedFields.PeerTags
+                                           ? BlockedOtlpPeerTags
+                                           : GetOtlpPeerTags(span, peerTagKeys, in peerTagResults);
+                        var additionalTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.AdditionalMetricTags) == StatsCardinalityLimitedFields.AdditionalMetricTags
+                                                 ? BlockedOtlpAdditionalTags
+                                                 : GetOtlpAdditionalTags(span, in additionalTagResults);
+                        bucket = new StatsBucket(key, new StatsBucket.OtlpTags(peerTags, additionalTags));
+                    }
+                    else
+                    {
+                        var peerTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.PeerTags) == StatsCardinalityLimitedFields.PeerTags
+                                           ? BlockedByTracerSentinelList
+                                           : GetEncodedPeerTags(span, peerTagKeys, in peerTagResults);
+                        var additionalTags = (key.CardinalityLimitedFields & StatsCardinalityLimitedFields.AdditionalMetricTags) == StatsCardinalityLimitedFields.AdditionalMetricTags
+                                                 ? BlockedByTracerSentinelList
+                                                 : GetEncodedAdditionalTags(span, in additionalTagResults);
+                        bucket = new StatsBucket(key, peerTags, additionalTags);
+                    }
 
-                    bucket = new StatsBucket(key, encodedPeerTags, encodedAdditionalTags);
                     buffer.Buckets.Add(key, bucket);
                 }
             }

tracer/src/Datadog.Trace/Agent/StatsBucket.cs

@@ -12,13 +12,27 @@ namespace Datadog.Trace.Agent
 {
     internal sealed class StatsBucket
     {
+        private static readonly List<byte[]> EmptyEncodedTags = [];
+        private static readonly OtlpTags EmptyOtlpTags = new([], []);
+
         public StatsBucket(StatsAggregationKey key, List<byte[]> peerTags, List<byte[]> additionalMetricTags)
+            : this(key, peerTags, additionalMetricTags, EmptyOtlpTags)
+        {
+        }
+
+        public StatsBucket(StatsAggregationKey key, OtlpTags tags)
+            : this(key, EmptyEncodedTags, EmptyEncodedTags, tags)
+        {
+        }
+
+        private StatsBucket(StatsAggregationKey key, List<byte[]> peerTags, List<byte[]> additionalMetricTags, OtlpTags otlpTags)
         {
             Key = key;
             OkSummary = CreateSketch();
             ErrorSummary = CreateSketch();
             PeerTags = peerTags;
             AdditionalMetricTags = additionalMetricTags;
+            Otlp = otlpTags;
         }
 
         public StatsAggregationKey Key { get; }
@@ -46,6 +60,8 @@ public StatsBucket(StatsAggregationKey key, List<byte[]> peerTags, List<byte[]>
 
         public List<byte[]> AdditionalMetricTags { get; }
 
+        public OtlpTags Otlp { get; }
+
         public void Clear()
         {
             Hits = 0;
@@ -67,5 +83,9 @@ private static DDSketch CreateSketch()
                 new CollapsingLowestDenseStore(2048),
                 new CollapsingLowestDenseStore(2048));
         }
+
+        internal sealed record OtlpTags(
+            List<string> PeerTags,
+            List<KeyValuePair<string, string>> AdditionalMetricTags);
     }
 }