Add an IArrayPool<char> implementation for vendored Newtonsoft.JSON

tracer/src/Datadog.Trace/Agent/IApiResponse.cs

@@ -11,6 +11,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Util.Streams;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
@@ -66,7 +67,7 @@ public static async Task<string> ReadAsStringAsync(this IApiResponse apiResponse
                 bufferedStream = new InitiallyBufferedStream(responseStream);
                 // wrap the stream in an "initially buffering" stream, so that if deserialization fails completely, we can get some details
                 using var sr = GetStreamReader(apiResponse, bufferedStream);
-                using var jsonTextReader = new JsonTextReader(sr);
+                using var jsonTextReader = new JsonTextReader(sr) { ArrayPool = JsonArrayPool.Shared };
                 return JsonSerializer.Create().Deserialize<T>(jsonTextReader);
             }
             catch (JsonException ex) when (bufferedStream?.GetBufferedContent() is { } buffered)

tracer/src/Datadog.Trace/Agent/Transports/SerializationHelpers.cs

@@ -9,6 +9,7 @@
 using System.IO.Compression;
 using System.Threading.Tasks;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Serialization;
 
@@ -36,6 +37,7 @@ public static async Task WriteAsJson<T>(Stream requestStream, T payload, JsonSer
         using var streamWriter = new StreamWriter(streamToWriteTo, EncodingHelpers.Utf8NoBom, bufferSize: 1024, leaveOpen: true);
         using var jsonWriter = new JsonTextWriter(streamWriter)
         {
+            ArrayPool = JsonArrayPool.Shared,
             CloseOutput = false
         };
         var serializer = JsonSerializer.Create(serializationSettings);

tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs

@@ -13,6 +13,7 @@
 using Datadog.Trace.AppSec.WafEncoding;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 using Datadog.Trace.Vendors.Serilog.Events;
@@ -95,7 +96,7 @@ private static void LogRuleDetailsIfDebugEnabled(JToken root)
                 }
 
                 using var reader = new StreamReader(stream);
-                using var jsonReader = new JsonTextReader(reader);
+                using var jsonReader = new JsonTextReader(reader) { ArrayPool = JsonArrayPool.Shared };
                 root = JToken.ReadFrom(jsonReader);
                 LogRuleDetailsIfDebugEnabled(root);
             }

tracer/src/Datadog.Trace/Ci/CiEnvironment/GithubActionsEnvironmentValues.cs

@@ -15,6 +15,8 @@
 using Datadog.Trace.Configuration;
 using Datadog.Trace.SourceGenerators;
 using Datadog.Trace.Telemetry.Metrics;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 
@@ -77,7 +79,7 @@ private static bool TryExtractFromJson(string filepath, [NotNullWhen(true)] out
         {
             using var fr = File.OpenRead(filepath);
             using var sr = new StreamReader(fr, Encoding.UTF8);
-            using var reader = new JsonTextReader(sr);
+            using var reader = new JsonTextReader(sr) { ArrayPool = JsonArrayPool.Shared };
 
             // Navigate: job.d[] where k == "check_run_id", get v
             // Structure: { "job": { "d": [ { "k": "check_run_id", "v": 55411116365.0 } ] } }

tracer/src/Datadog.Trace/Ci/Ipc/IpcDualChannel.cs

@@ -6,6 +6,8 @@
 
 using System;
 using System.IO;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Serialization;
 
@@ -50,7 +52,7 @@ private void OnMessageReceived(ArraySegment<byte> data)
     {
         using var memoryStream = new MemoryStream(data.Array!, data.Offset, data.Count);
         using var reader = new StreamReader(memoryStream, Util.EncodingHelpers.Utf8NoBom);
-        using var jsonReader = new JsonTextReader(reader);
+        using var jsonReader = new JsonTextReader(reader) { ArrayPool = JsonArrayPool.Shared };
         var message = _jsonSerializer.Deserialize(jsonReader);
         if (message != null)
         {
@@ -66,7 +68,7 @@ public bool TrySendMessage(object message)
             using var memoryStream = new MemoryStream();
             using (var writer = new StreamWriter(memoryStream, Util.EncodingHelpers.Utf8NoBom))
             {
-                using var jsonWriter = new JsonTextWriter(writer);
+                using var jsonWriter = new JsonTextWriter(writer) { ArrayPool = JsonArrayPool.Shared };
                 _jsonSerializer.Serialize(jsonWriter, message);
             }
 

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/ContextPropagation.cs

@@ -8,11 +8,15 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using Datadog.Trace.DataStreamsMonitoring;
 using Datadog.Trace.DuckTyping;
 using Datadog.Trace.Headers;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Propagators;
+using Datadog.Trace.SourceGenerators;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.AWS.Kinesis
@@ -109,6 +113,7 @@ public static void InjectTraceIntoData<TRecordRequest>(Tracer tracer, TRecordReq
             }
         }
 
+        [TestingAndPrivateOnly]
         internal static Dictionary<string, object>? ParseDataObject(MemoryStream dataStream)
         {
             try
@@ -123,11 +128,13 @@ public static void InjectTraceIntoData<TRecordRequest>(Tracer tracer, TRecordReq
             return null;
         }
 
+        [TestingAndPrivateOnly]
         public static Dictionary<string, object>? MemoryStreamToDictionary(MemoryStream stream)
         {
             // Convert the MemoryStream to a string
-            var streamReader = new StreamReader(stream);
-            var reader = new JsonTextReader(streamReader);
+            // Default values for StreamReader, but with leaveOpen:true
+            using var streamReader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true,  bufferSize: 1024, leaveOpen: true);
+            using var reader = new JsonTextReader(streamReader) { ArrayPool = JsonArrayPool.Shared };
             var serializer = new JsonSerializer();
 
             // Deserialize the JSON string into a Dictionary<string, object>
@@ -136,10 +143,12 @@ public static void InjectTraceIntoData<TRecordRequest>(Tracer tracer, TRecordReq
             return serializer.Deserialize<Dictionary<string, object>>(reader);
         }
 
+        [TestingAndPrivateOnly]
         public static MemoryStream DictionaryToMemoryStream(Dictionary<string, object> dictionary)
         {
             var memoryStream = new MemoryStream();
-            var writer = new StreamWriter(memoryStream);
+            using var streamWriter = new StreamWriter(memoryStream, EncodingHelpers.Utf8NoBom, 1024, leaveOpen: true);
+            using var writer = new JsonTextWriter(streamWriter) { ArrayPool = JsonArrayPool.Shared };
             var serializer = new JsonSerializer();
             serializer.Serialize(writer, dictionary);
             writer.Flush();

tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs

@@ -17,6 +17,8 @@
 using Datadog.Trace.Processors;
 using Datadog.Trace.RemoteConfigurationManagement;
 using Datadog.Trace.Telemetry;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 #nullable enable
@@ -643,7 +645,7 @@ private void WriteStartupDebuggerDiagnosticLog()
             {
                 var stringWriter = new StringWriter();
                 var settings = DebuggerSettings;
-                using (var writer = new JsonTextWriter(stringWriter))
+                using (var writer = new JsonTextWriter(stringWriter) { ArrayPool = JsonArrayPool.Shared })
                 {
                     writer.WriteStartObject();
                     writer.WritePropertyName("dynamic_instrumentation_enabled");

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ThirdParty/ThirdPartyConfigurationReader.cs

@@ -8,6 +8,8 @@
 using System.IO;
 using System.Reflection;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 #nullable enable
@@ -40,7 +42,7 @@ private static ImmutableHashSet<string> SafeGetModules()
             }
 
             using var sr = new StreamReader(stream!);
-            using var jsonReader = new JsonTextReader(sr);
+            using var jsonReader = new JsonTextReader(sr) { ArrayPool = JsonArrayPool.Shared };
 
             var modules = new HashSet<string>();
             while (jsonReader.Read())

tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionParser.cs

@@ -11,6 +11,8 @@
 using Datadog.Trace.Debugger.Helpers;
 using Datadog.Trace.Debugger.Models;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 using static Datadog.Trace.Debugger.Expressions.ProbeExpressionParserHelper;
@@ -558,7 +560,7 @@ private ExpressionBodyAndParameters ParseProbeExpression(string expressionJson,
         var result = Expression.Variable(typeof(T), "$dd_el_result");
         scopeMembers.Add(result);
 
-        var reader = new JsonTextReader(new StringReader(expressionJson));
+        using var reader = new JsonTextReader(new StringReader(expressionJson)) { ArrayPool = JsonArrayPool.Shared };
         SetReaderAtExpressionStart(reader);
 
         var finalExpr = ParseRoot(reader, scopeMembers);

tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs

@@ -16,6 +16,7 @@
 using Datadog.Trace.Debugger.Helpers;
 using Datadog.Trace.Debugger.Models;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using ProbeLocation = Datadog.Trace.Debugger.Expressions.ProbeLocation;
 
@@ -60,7 +61,7 @@ public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool
             _isFullSnapshot = isFullSnapshot;
             _probeLocation = location;
             _jsonUnderlyingString = StringBuilderCache.Acquire();
-            JsonWriter = new JsonTextWriter(new StringWriter(_jsonUnderlyingString));
+            JsonWriter = new JsonTextWriter(new StringWriter(_jsonUnderlyingString)) { ArrayPool = JsonArrayPool.Shared };
             MethodScopeMembers = default;
             _captureBehaviour = CaptureBehaviour.Capture;
             _errors = null;

tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs

@@ -21,6 +21,7 @@
 using Datadog.Trace.Logging;
 using Datadog.Trace.RemoteConfigurationManagement;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using OperationCanceledException = System.OperationCanceledException;
 
@@ -372,7 +373,7 @@ private bool TrySerializeClass(
             out int newTotalBytes)
         {
             pooledWriter.Reset();
-            using (var jsonWriter = new JsonTextWriter(pooledWriter) { CloseOutput = false })
+            using (var jsonWriter = new JsonTextWriter(pooledWriter) { CloseOutput = false, ArrayPool = JsonArrayPool.Shared })
             {
                 serializer.Serialize(jsonWriter, classScope);
                 jsonWriter.Flush();

tracer/src/Datadog.Trace/Debugger/Upload/SymbolUploadApi.cs

@@ -16,6 +16,7 @@
 using Datadog.Trace.Debugger.Models;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.Debugger.Upload
@@ -72,7 +73,7 @@ internal static ArraySegment<byte> CreateEventMetadata(string serviceName, strin
             const int bufferSize = 256;
             using var stream = new MemoryStream(capacity: bufferSize);
             using (var streamWriter = new StreamWriter(stream, EncodingHelpers.Utf8NoBom, bufferSize: bufferSize, leaveOpen: true))
-            using (var jsonWriter = new JsonTextWriter(streamWriter))
+            using (var jsonWriter = new JsonTextWriter(streamWriter) { ArrayPool = JsonArrayPool.Shared })
             {
                 jsonWriter.CloseOutput = false;
                 jsonWriter.Formatting = Formatting.None;

tracer/src/Datadog.Trace/Iast/SensitiveData/JsonTokenizer.cs

@@ -9,6 +9,8 @@
 using System.Linq;
 using System.Text.RegularExpressions;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 #nullable enable
@@ -37,7 +39,7 @@ public List<Range> GetTokens(Evidence evidence, IntegrationId? integrationId = n
 
         var redactedRanges = new List<Range>();
         using var sr = new StringReader(value);
-        using var reader = new JsonTextReader(sr);
+        using var reader = new JsonTextReader(sr) { ArrayPool = JsonArrayPool.Shared };
 
         while (reader.Read())
         {

tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs

@@ -15,6 +15,7 @@
 using Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.Logging.DirectSubmission.Formatting
@@ -276,11 +277,11 @@ internal static void WriteValue(JsonWriter writer, object? value)
         }
 
         internal static JsonTextWriter GetJsonWriter(StringBuilder builder)
-        {
-            var writer = new JsonTextWriter(new StringWriter(builder));
-            writer.Formatting = Vendors.Newtonsoft.Json.Formatting.None;
-            return writer;
-        }
+            => new JsonTextWriter(new StringWriter(builder))
+            {
+                ArrayPool = JsonArrayPool.Shared,
+                Formatting = Vendors.Newtonsoft.Json.Formatting.None,
+            };
 
         /// <summary>
         /// Format the log, based on <see cref="Datadog.Trace.Vendors.Serilog.Formatting.Json.JsonFormatter"/>

tracer/src/Datadog.Trace/Logging/TracerFlare/TracerFlareManager.cs

@@ -17,6 +17,7 @@
 using Datadog.Trace.SourceGenerators;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Util;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 
@@ -419,7 +420,7 @@ private void HandleConfigUpdate(AgentConfiguration config)
         {
             using var stream = new MemoryStream(contents);
             using var streamReader = new StreamReader(stream);
-            using var jsonReader = new JsonTextReader(streamReader);
+            using var jsonReader = new JsonTextReader(streamReader) { ArrayPool = JsonArrayPool.Shared };
             return JObject.Load(jsonReader);
         }
         catch (Exception ex)

tracer/src/Datadog.Trace/RemoteConfigurationManagement/Json/TufRootBase64Converter.cs

@@ -7,6 +7,8 @@
 using System.IO;
 using System.Text;
 using Datadog.Trace.RemoteConfigurationManagement.Protocol.Tuf;
+using Datadog.Trace.Util.Json;
+using Datadog.Trace.Util.Streams;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.RemoteConfigurationManagement.Json
@@ -24,7 +26,10 @@ public override TufRoot ReadJson(JsonReader reader, Type objectType, TufRoot exi
 
             using var stream = new MemoryStream(contentDecode);
             using var streamReader = new StreamReader(stream);
-            using var jsonReader = new JsonTextReader(streamReader);
+            using var jsonReader = new JsonTextReader(streamReader)
+            {
+                ArrayPool = JsonArrayPool.Shared,
+            };
             return serializer.Deserialize<TufRoot>(jsonReader);
         }
 

tracer/src/Datadog.Trace/RemoteConfigurationManagement/NamedRawFile.cs

@@ -6,6 +6,7 @@
 #nullable enable
 
 using System.IO;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.RemoteConfigurationManagement
@@ -26,7 +27,11 @@ public NamedRawFile(RemoteConfigurationPath path, byte[] value)
         {
             using var stream = new MemoryStream(RawFile);
             using var streamReader = new StreamReader(stream);
-            using var jsonReader = new JsonTextReader(streamReader);
+            using var jsonReader = new JsonTextReader(streamReader)
+            {
+                ArrayPool = JsonArrayPool.Shared,
+            };
+
             return new NamedTypedFile<T?>(Path.Path, JsonSerializer.CreateDefault().Deserialize<T>(jsonReader));
         }
     }

tracer/src/Datadog.Trace/TracerManager.cs

@@ -33,6 +33,7 @@
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Util;
 using Datadog.Trace.Util.Http;
+using Datadog.Trace.Util.Json;
 using Datadog.Trace.Vendors.Newtonsoft.Json;
 using Datadog.Trace.Vendors.StatsdClient;
 
@@ -361,7 +362,7 @@ private static async Task WriteDiagnosticLog(TracerManager instance, MutableSett
 
                 var stringWriter = new StringWriter();
 
-                using (var writer = new JsonTextWriter(stringWriter))
+                using (var writer = new JsonTextWriter(stringWriter) { ArrayPool = JsonArrayPool.Shared })
                 {
                     void WriteDictionary(IReadOnlyDictionary<string, string> dictionary)
                     {