Upgrade to Newtonsoft.Json 13.0.1

eng/Versions.props

@@ -38,7 +38,7 @@
     <MoqVersion>4.16.1</MoqVersion>
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
     <McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
-    <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+    <NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
     <MicrosoftBclJsonSourcesVersion>4.6.0-preview4.19202.2</MicrosoftBclJsonSourcesVersion>
     <NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
     <NuGetVersion>5.3.0</NuGetVersion>

samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj

@@ -11,6 +11,6 @@
     <!-- / CVE-2017-11770 -->
 
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 </Project>

scripts/build.ps1

@@ -641,11 +641,12 @@ function Publish-Package {
     }
 
     # Copy dependency of Microsoft.TestPlatform.TestHostRuntimeProvider
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\net45\Newtonsoft.Json.dll"
+    $newtonsoftJsonVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.NewtonsoftJsonVersion
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\net45\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $fullCLRPackage451Dir -Force"
     Copy-Item $newtonsoft $fullCLRPackage451Dir -Force
 
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\netstandard1.0\Newtonsoft.Json.dll"
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\netstandard1.0\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $coreCLR20PackageDir -Force"
     Copy-Item $newtonsoft $coreCLR20PackageDir -Force
 

scripts/build.sh

@@ -150,6 +150,7 @@ TPB_LocalizedBuild=$DISABLE_LOCALIZED_BUILD
 TPB_Verbose=$VERBOSE
 TPB_EXTERNALS_VERSION=$(grep TestPlatformExternalsVersion $TP_ROOT_DIR/scripts/build/TestPlatform.Dependencies.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 TPB_CC_EXTERNALS_VERSION=$(grep MicrosoftInternalCodeCoverageVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
+TPB_NEWTONSOFT_JSON_VERSION=$(grep NewtonsoftJsonVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 
 TPB_BRANCH="$(git -C "." rev-parse --abbrev-ref HEAD 2>/dev/null)" || TPB_BRANCH="LOCALBRANCH" # detached HEAD
 TPB_COMMIT="$(git -C "." rev-parse HEAD 2>/dev/null)"              || TPB_COMMIT="LOCALBUILD" # detached HEAD
@@ -436,7 +437,7 @@ function publish_package()
         done
         #*************************************************************************************************************#
 
-        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/9.0.1/lib/netstandard1.0/Newtonsoft.Json.dll
+        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/$TPB_NEWTONSOFT_JSON_VERSION/lib/netstandard1.0/Newtonsoft.Json.dll
         cp $newtonsoft $packageDir
     done
 

scripts/build/TestPlatform.Dependencies.props

@@ -57,7 +57,7 @@
     <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
     <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
-    <JsonNetVersion>9.0.1</JsonNetVersion>
+    <JsonNetVersion>13.0.1</JsonNetVersion>
 
     <!-- We cannot reuse $(msbuildvar) because we need the text value inside build.ps1 -->
     <TestPlatformExternalsVersion>17.3.32621.448</TestPlatformExternalsVersion>

src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs

@@ -96,7 +96,7 @@ public Message DeserializeMessage(string rawMessage)
             // PERF: This is slow, we deserialize the message, and the payload into JToken just to get the header. We then
             // deserialize the data from the JToken, but that is twice as expensive as deserializing the whole object directly into the final object type.
             // We need this for backward compatibility though.
-            return Deserialize<VersionedMessage>(rawMessage);
+            return Deserialize<VersionedMessage>(rawMessage)!;
         }
 
         // PERF: Try grabbing the version and message type from the string directly, we are pretty certain how the message is serialized
@@ -105,7 +105,7 @@ public Message DeserializeMessage(string rawMessage)
         {
             // PERF: If the fast path fails, deserialize into header object that does not have any Payload. When the message type info
             // is at the start of the message, this is also pretty fast. Again, this won't touch the payload.
-            MessageHeader header = JsonConvert.DeserializeObject<MessageHeader>(rawMessage, JsonSettings);
+            MessageHeader header = JsonConvert.DeserializeObject<MessageHeader>(rawMessage, JsonSettings)!;
             version = header.Version;
             messageType = header.MessageType;
         }
@@ -161,21 +161,27 @@ public Message DeserializeMessage(string rawMessage)
         var messageWithRawMessage = (VersionedMessageWithRawMessage)message;
         var rawMessage = messageWithRawMessage.RawMessage;
 
+        if (rawMessage == null)
+        {
+            return default;
+        }
+
         // The deserialized message can still have a version (0 or 1), that should use the old deserializer
         if (payloadSerializer == PayloadSerializerV2)
         {
             // PERF: Fast path is compatibile only with protocol versions that use serializer_2,
             // and this is faster than deserializing via deserializer_2.
             var messageWithPayload = JsonConvert.DeserializeObject<PayloadedMessage<T>>(rawMessage, FastJsonSettings);
-            return messageWithPayload.Payload;
+
+            return messageWithPayload == null ? default : messageWithPayload.Payload;
         }
         else
         {
             // PERF: When payloadSerializer1 was resolved we need to deserialize JToken, and then deserialize that.
             // This is still better than deserializing the JToken in DeserializeMessage because here we know that the payload
             // will actually be used.
             TPDebug.Assert(rawMessage is not null, "rawMessage should not be null");
-            var rawMessagePayload = Deserialize<Message>(rawMessage).Payload;
+            var rawMessagePayload = Deserialize<Message>(rawMessage)?.Payload;
             TPDebug.Assert(rawMessagePayload is not null, "rawMessagePayload should not be null");
             return Deserialize<T>(payloadSerializer, rawMessagePayload);
         }
@@ -286,7 +292,7 @@ private static bool TryGetSubstringUntilDelimiter(string rawMessage, int start,
     /// <param name="version">Version of serializer to be used.</param>
     /// <typeparam name="T">Target type to deserialize.</typeparam>
     /// <returns>An instance of <see cref="T"/>.</returns>
-    public T Deserialize<T>(string json, int version = 1)
+    public T? Deserialize<T>(string json, int version = 1)
     {
         var payloadSerializer = GetPayloadSerializer(version);
         return Deserialize<T>(payloadSerializer, json);
@@ -327,6 +333,9 @@ public string SerializePayload(string? messageType, object? payload, int version
         // so when we resolved the old serializer we should use non-fast path.
         if (DisableFastJson || payloadSerializer == PayloadSerializerV1)
         {
+            if (payload == null)
+                return string.Empty;
+
             var serializedPayload = JToken.FromObject(payload, payloadSerializer);
 
             return version > 1 ?
@@ -387,7 +396,7 @@ private static string Serialize<T>(JsonSerializer serializer, T data)
     /// <param name="serializer">Serializer.</param>
     /// <param name="data">Data to be deserialized.</param>
     /// <returns>Deserialized data.</returns>
-    private static T Deserialize<T>(JsonSerializer serializer, string data)
+    private static T? Deserialize<T>(JsonSerializer serializer, string data)
     {
         using var stringReader = new StringReader(data);
         using var jsonReader = new JsonTextReader(stringReader);
@@ -403,7 +412,7 @@ private static T Deserialize<T>(JsonSerializer serializer, string data)
     /// <returns>Deserialized data.</returns>
     private static T Deserialize<T>(JsonSerializer serializer, JToken jToken)
     {
-        return jToken.ToObject<T>(serializer);
+        return jToken.ToObject<T>(serializer)!;
     }
 
     private static JsonSerializer GetPayloadSerializer(int? version)

src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Shipped.txt

@@ -179,7 +179,7 @@ Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.MessageRec
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.MessageReceivedEventArgs.MessageReceivedEventArgs() -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.Clone<T>(T? obj) -> T?
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.Deserialize<T>(string! json, int version = 1) -> T
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.Deserialize<T>(string! json, int version = 1) -> T?
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.DeserializeMessage(string! rawMessage) -> Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Message!
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.DeserializePayload<T>(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Message? message) -> T?
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.Serialize<T>(T data, int version = 1) -> string!
@@ -361,20 +361,20 @@ Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.VersionedMessage.Vers
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Message.ToString() -> string!
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.DefaultTestPlatformContractResolver.CreateContract(System.Type! objectType) -> Newtonsoft.Json.Serialization.JsonContract!
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestCaseConverter.CanConvert(System.Type! objectType) -> bool
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestCaseConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object! existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestCaseConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object! value, Newtonsoft.Json.JsonSerializer! serializer) -> void
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestCaseConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object? existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object?
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestCaseConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object? value, Newtonsoft.Json.JsonSerializer! serializer) -> void
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.CanConvert(System.Type! objectType) -> bool
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.CanRead.get -> bool
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.CanWrite.get -> bool
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object! existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object! value, Newtonsoft.Json.JsonSerializer! serializer) -> void
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object? existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestObjectConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object? value, Newtonsoft.Json.JsonSerializer! serializer) -> void
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestPlatformContractResolver1.CreateContract(System.Type! objectType) -> Newtonsoft.Json.Serialization.JsonContract!
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestResultConverter.CanConvert(System.Type! objectType) -> bool
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestResultConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object! existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestResultConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object! value, Newtonsoft.Json.JsonSerializer! serializer) -> void
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestResultConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object? existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestResultConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object? value, Newtonsoft.Json.JsonSerializer! serializer) -> void
 override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestRunStatisticsConverter.CanConvert(System.Type! objectType) -> bool
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestRunStatisticsConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object! existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object!
-override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestRunStatisticsConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object! value, Newtonsoft.Json.JsonSerializer! serializer) -> void
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestRunStatisticsConverter.ReadJson(Newtonsoft.Json.JsonReader! reader, System.Type! objectType, object? existingValue, Newtonsoft.Json.JsonSerializer! serializer) -> object?
+override Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization.TestRunStatisticsConverter.WriteJson(Newtonsoft.Json.JsonWriter! writer, object? value, Newtonsoft.Json.JsonSerializer! serializer) -> void
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollectionTestCaseEventSender.Create() -> Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollectionTestCaseEventSender!
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollectionTestCaseEventSender.Instance.get -> Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollectionTestCaseEventSender?
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer.Instance.get -> Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.JsonDataSerializer!

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestCaseConverter.cs

@@ -22,7 +22,7 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         var testCase = new TestCase();
 
@@ -38,11 +38,22 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         // key value pairs.
         foreach (var property in properties.Values<JToken>())
         {
-            var testProperty = property["Key"].ToObject<TestProperty>(serializer);
+            var testProperty = property?["Key"]?.ToObject<TestProperty>(serializer);
+
+            if (testProperty == null)
+            {
+                return null;
+            }
 
             // Let the null values be passed in as null data
-            var token = property["Value"];
+            var token = property?["Value"];
             string? propertyData = null;
+
+            if (token == null)
+            {
+                return null;
+            }
+
             if (token.Type != JTokenType.Null)
             {
                 // If the property is already a string. No need to convert again.
@@ -86,8 +97,13 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
+        if (value == null)
+        {
+            return;
+        }
+
         // P2 to P1
         var testCase = (TestCase)value;