Upgrade to Newtonsoft.Json 13.0.1 (microsoft#3815)

eng/Versions.props

@@ -38,7 +38,7 @@
     <MoqVersion>4.16.1</MoqVersion>
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
     <McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
-    <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+    <NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
     <MicrosoftBclJsonSourcesVersion>4.6.0-preview4.19202.2</MicrosoftBclJsonSourcesVersion>
     <NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
     <NuGetVersion>5.3.0</NuGetVersion>

samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj

@@ -7,8 +7,6 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
-    <PackageReference Include="Newtonsoft.Json">
-      <Version>9.0.1</Version>
-    </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 </Project>

scripts/build.ps1

@@ -631,11 +631,12 @@ function Publish-Package {
     }
 
     # Copy dependency of Microsoft.TestPlatform.TestHostRuntimeProvider
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\net45\Newtonsoft.Json.dll"
+    $newtonsoftJsonVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.NewtonsoftJsonVersion
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\net45\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $fullCLRPackage451Dir -Force"
     Copy-Item $newtonsoft $fullCLRPackage451Dir -Force
 
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\netstandard1.0\Newtonsoft.Json.dll"
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\netstandard1.0\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $coreCLR20PackageDir -Force"
     Copy-Item $newtonsoft $coreCLR20PackageDir -Force
 

scripts/build.sh

@@ -150,6 +150,7 @@ TPB_LocalizedBuild=$DISABLE_LOCALIZED_BUILD
 TPB_Verbose=$VERBOSE
 TPB_EXTERNALS_VERSION=$(grep TestPlatformExternalsVersion $TP_ROOT_DIR/scripts/build/TestPlatform.Dependencies.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 TPB_CC_EXTERNALS_VERSION=$(grep MicrosoftInternalCodeCoverageVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
+TPB_NEWTONSOFT_JSON_VERSION=$(grep NewtonsoftJsonVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 
 TPB_BRANCH="$(git -C "." rev-parse --abbrev-ref HEAD 2>/dev/null)" || TPB_BRANCH="LOCALBRANCH" # detached HEAD
 TPB_COMMIT="$(git -C "." rev-parse HEAD 2>/dev/null)"              || TPB_COMMIT="LOCALBUILD" # detached HEAD
@@ -436,7 +437,7 @@ function publish_package()
         done
         #*************************************************************************************************************#
 
-        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/9.0.1/lib/netstandard1.0/Newtonsoft.Json.dll
+        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/$TPB_NEWTONSOFT_JSON_VERSION/lib/netstandard1.0/Newtonsoft.Json.dll
         cp $newtonsoft $packageDir
     done
 

scripts/build/TestPlatform.Dependencies.props

@@ -57,7 +57,7 @@
     <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
     <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
-    <JsonNetVersion>9.0.1</JsonNetVersion>
+    <JsonNetVersion>13.0.1</JsonNetVersion>
 
     <TestPlatformExternalsVersion>17.1.0-preview-2-31925-026</TestPlatformExternalsVersion>
     <!-- <TestPlatformMSDiaVersion>$(TestPlatformExternalsVersion)</TestPlatformMSDiaVersion> -->

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs

@@ -23,13 +23,13 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         return serializer.Deserialize<TestRunStatistics>(reader);
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
         serializer.Serialize(writer, value);
     }

src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs

@@ -157,7 +157,8 @@ public async Task<ICollection<AttachmentSet>> ProcessAttachmentSetsAsync(XmlElem
         cancellationToken.Register(() => _wrapper.CancelProcessAttachment());
         _processAttachmentSetsLogger = logger;
         _progressReporter = progressReporter;
-        return JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false));
+        var result = await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false);
+        return JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(result)!;
     }
 
     public void Dispose()

src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorWrapper.cs

@@ -62,7 +62,7 @@ public string ProcessAttachment(
     {
         var doc = new XmlDocument();
         doc.LoadXml(configurationElement);
-        AttachmentSet[] attachmentSets = JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(attachments);
+        AttachmentSet[] attachmentSets = JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(attachments)!;
         SynchronousProgress progress = new(Report);
         _processAttachmentCts = new CancellationTokenSource();
 

src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs

@@ -733,8 +733,8 @@ private string GetTestHostPath(string runtimeConfigDevPath, string depsFilePath,
                 using (JsonTextReader reader = new(file))
                 {
                     JObject context = (JObject)JToken.ReadFrom(reader);
-                    JObject runtimeOptions = (JObject)context.GetValue("runtimeOptions");
-                    JToken additionalProbingPaths = runtimeOptions.GetValue("additionalProbingPaths");
+                    JObject runtimeOptions = (JObject)context.GetValue("runtimeOptions")!;
+                    JToken additionalProbingPaths = runtimeOptions.GetValue("additionalProbingPaths")!;
                     foreach (var x in additionalProbingPaths)
                     {
                         EqtTrace.Verbose("DotnetTestHostmanager: Looking for path {0} in folder {1}", testHostPath, x.ToString());

src/package/ThirdPartyNotices.txt

@@ -8,7 +8,7 @@ and the licenses under which Microsoft received such components are set forth be
 informational purposes only. Microsoft reserves all rights not expressly granted herein, whether by 
 implication, estoppel or otherwise.
 
-1.	Newtonsoft version 9.0.1 (https://github.com/JamesNK/Newtonsoft.Json)
+1.	Newtonsoft version 13.0.1 (https://github.com/JamesNK/Newtonsoft.Json)
 2.	Mono.Cecil version 0.11.3 (https://github.com/jbevain/cecil)
 
 

src/testhost.arm64/app.config

@@ -46,6 +46,11 @@
         <assemblyIdentity name="Microsoft.VisualStudio.QualityTools.Tips.UnitTest.ObjectModel" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
         <bindingRedirect oldVersion="10.0.0.0-16.0.0.0" newVersion="17.0.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <!-- Microsoft.Extensions.DependencyModel depends on version 9.0.0, and newer versions of that nuget don't support all the frameworks we need anymore. -->
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="9.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <system.diagnostics>
@@ -61,4 +66,4 @@
     <!-- This flag is added to support test execution for net35 tests through TMI adapter. -->
     <add key="TestProjectRetargetTo35Allowed" value="true" />
   </appSettings>
-</configuration>
+</configuration>

src/testhost.x86/app.config

@@ -46,6 +46,11 @@
         <assemblyIdentity name="Microsoft.VisualStudio.QualityTools.Tips.UnitTest.ObjectModel" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
         <bindingRedirect oldVersion="10.0.0.0-16.0.0.0" newVersion="17.0.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <!-- Microsoft.Extensions.DependencyModel depends on version 9.0.0, and newer versions of that nuget don't support all the frameworks we need anymore. -->
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="9.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <system.diagnostics>

src/testhost/app.config

@@ -46,6 +46,11 @@
         <assemblyIdentity name="Microsoft.VisualStudio.QualityTools.Tips.UnitTest.ObjectModel" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
         <bindingRedirect oldVersion="10.0.0.0-16.0.0.0" newVersion="17.0.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <!-- Microsoft.Extensions.DependencyModel depends on version 9.0.0, and newer versions of that nuget don't support all the frameworks we need anymore. -->
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="9.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <system.diagnostics>
@@ -61,4 +66,4 @@
     <!-- This flag is added to support test execution for net35 tests through TMI adapter. -->
     <add key="TestProjectRetargetTo35Allowed" value="true" />
   </appSettings>
-</configuration>
+</configuration>

src/vstest.console/app.config

@@ -20,6 +20,11 @@
         <assemblyIdentity name="Microsoft.VisualStudio.QualityTools.Tips.UnitTest.ObjectModel" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
         <bindingRedirect oldVersion="10.0.0.0-16.0.0.0" newVersion="17.0.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <!-- Microsoft.Extensions.DependencyModel depends on version 9.0.0, and newer versions of that nuget don't support all the frameworks we need anymore. -->
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="9.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <appSettings>
@@ -54,4 +59,4 @@
     </sharedListeners>
   </system.diagnostics> -->
 
-</configuration>
+</configuration>

test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs

@@ -35,7 +35,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur
         string sdkVersion = GetLatestSdkVersion(dotnetPath);
         string runtimeConfigFile = Path.Combine(dotnetRunnerPath.FullName, "vstest.console.runtimeconfig.json");
         JObject patchRuntimeConfig = JObject.Parse(File.ReadAllText(runtimeConfigFile));
-        patchRuntimeConfig["runtimeOptions"]["framework"]["version"] = sdkVersion;
+        patchRuntimeConfig!["runtimeOptions"]!["framework"]!["version"] = sdkVersion;
         File.WriteAllText(runtimeConfigFile, patchRuntimeConfig.ToString());
 
         var environmentVariables = new Dictionary<string, string>

test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs

@@ -80,7 +80,7 @@ public void DeserializeShouldDeserializeAnObjectWhichHadSelfReferencingLoopBefor
         var json = _jsonDataSerializer.SerializePayload("dummy", classWithSelfReferencingLoop);
 
         // This line should deserialize properly
-        var result = _jsonDataSerializer.Deserialize<ClassWithSelfReferencingLoop>(json, 1);
+        var result = _jsonDataSerializer.Deserialize<ClassWithSelfReferencingLoop>(json, 1)!;
 
         Assert.AreEqual(typeof(ClassWithSelfReferencingLoop), result.GetType());
         Assert.IsNull(result.InfiniteRefernce);

test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestCaseSerializationTests.cs

@@ -260,7 +260,7 @@ private static string Serialize<T>(T data, int version = 1)
 
     private static T Deserialize<T>(string json, int version = 1)
     {
-        return JsonDataSerializer.Instance.Deserialize<T>(json, version);
+        return JsonDataSerializer.Instance.Deserialize<T>(json, version)!;
     }
 
     private void VerifyDummyPropertyIsRegistered()

test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestObjectConverterTests.cs

@@ -164,6 +164,6 @@ private static string Serialize<T>(T data)
 
     private static T Deserialize<T>(string json)
     {
-        return JsonDataSerializer.Instance.Deserialize<T>(json);
+        return JsonDataSerializer.Instance.Deserialize<T>(json)!;
     }
 }

test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestResultSerializationTests.cs

@@ -296,7 +296,7 @@ private static string Serialize<T>(T data, int version)
 
     private static T Deserialize<T>(string json, int version)
     {
-        return JsonDataSerializer.Instance.Deserialize<T>(json, version);
+        return JsonDataSerializer.Instance.Deserialize<T>(json, version)!;
     }
 
     private static void VerifyDummyPropertyIsRegistered()

test/Microsoft.TestPlatform.ObjectModel.UnitTests/Client/DiscoveryCriteriaTests.cs

@@ -44,7 +44,7 @@ public void DiscoveryCriteriaShouldBeDeserializable()
     {
         var json = "{\"Sources\":[\"sampleTest.dll\"],\"AdapterSourceMap\":{\"_none_\":[\"sampleTest.dll\"]},\"FrequencyOfDiscoveredTestsEvent\":100,\"DiscoveredTestEventTimeout\":\"10675199.02:48:05.4775807\",\"RunSettings\":\"<RunConfiguration></RunConfiguration>\",\"TestCaseFilter\":\"TestFilter\"}";
 
-        var criteria = JsonConvert.DeserializeObject<DiscoveryCriteria>(json, Settings);
+        var criteria = JsonConvert.DeserializeObject<DiscoveryCriteria>(json, Settings)!;
 
         Assert.AreEqual(TimeSpan.MaxValue, criteria.DiscoveredTestEventTimeout);
         Assert.AreEqual(100, criteria.FrequencyOfDiscoveredTestsEvent);

test/Microsoft.TestPlatform.PerformanceTests/ProtocolV1Tests.cs

@@ -121,6 +121,6 @@ private static string SerializeV1<T>(T data)
 
     private static T DeserializeV1<T>(string json)
     {
-        return JsonDataSerializer.Instance.Deserialize<T>(json, version: 1);
+        return JsonDataSerializer.Instance.Deserialize<T>(json, version: 1)!;
     }
 }

test/Microsoft.TestPlatform.PerformanceTests/ProtocolV2Tests.cs

@@ -119,6 +119,6 @@ private static string SerializeV2<T>(T data)
 
     private static T DeserializeV2<T>(string json)
     {
-        return JsonDataSerializer.Instance.Deserialize<T>(json, version: 2);
+        return JsonDataSerializer.Instance.Deserialize<T>(json, version: 2)!;
     }
 }

test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs

@@ -34,7 +34,7 @@ public void VerifyHostArchitecture(string architecture)
         string sdkVersion = GetLatestSdkVersion(dotnetPath);
         string runtimeConfigFile = Path.Combine(dotnetRunnerPath.FullName, "vstest.console.runtimeconfig.json");
         JObject patchRuntimeConfig = JObject.Parse(File.ReadAllText(runtimeConfigFile));
-        patchRuntimeConfig["runtimeOptions"]["framework"]["version"] = sdkVersion;
+        patchRuntimeConfig!["runtimeOptions"]!["framework"]!["version"] = sdkVersion;
         File.WriteAllText(runtimeConfigFile, patchRuntimeConfig.ToString());
 
         var environmentVariables = new Dictionary<string, string>

test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs

@@ -319,7 +319,7 @@ public void GetTestHostProcessStartInfoShouldUseTestHostExeFromNugetIfNotFoundIn
             ""microsoft.testplatform.testhost/15.0.0-Dev"": {
                 ""dependencies"": {
                     ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"",
-                    ""Newtonsoft.Json"": ""9.0.1""
+                    ""Newtonsoft.Json"": ""13.0.1""
                 },
                 ""runtime"": {
                     ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { },
@@ -389,7 +389,7 @@ public void GetTestHostProcessStartInfoShouldUseTestHostX86ExeFromNugetIfNotFoun
             ""microsoft.testplatform.testhost/15.0.0-Dev"": {
                 ""dependencies"": {
                     ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"",
-                    ""Newtonsoft.Json"": ""9.0.1""
+                    ""Newtonsoft.Json"": ""13.0.1""
                 },
                 ""runtime"": {
                     ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { },
@@ -709,7 +709,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromDepsFile()
             ""microsoft.testplatform.testhost/15.0.0-Dev"": {
                 ""dependencies"": {
                     ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"",
-                    ""Newtonsoft.Json"": ""9.0.1""
+                    ""Newtonsoft.Json"": ""13.0.1""
                 },
                 ""runtime"": {
                     ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { },
@@ -774,7 +774,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect
             ""microsoft.testplatform.testhost/15.0.0-Dev"": {
                 ""dependencies"": {
                     ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"",
-                    ""Newtonsoft.Json"": ""9.0.1""
+                    ""Newtonsoft.Json"": ""13.0.1""
                 },
                 ""runtime"": {
                     ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { },
@@ -842,7 +842,7 @@ public void GetTestHostProcessStartInfoShouldSkipInvalidAdditionalProbingPaths()
             ""microsoft.testplatform.testhost/15.0.0-Dev"": {
                 ""dependencies"": {
                     ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"",
-                    ""Newtonsoft.Json"": ""9.0.1""
+                    ""Newtonsoft.Json"": ""13.0.1""
                 },
                 ""runtime"": {
                     ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { },

test/TestAssets/NewtonSoftDependency/App.config

@@ -4,7 +4,7 @@
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
         <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
+        <bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
       </dependentAssembly>
     </assemblyBinding>
   </runtime>

test/TestAssets/NewtonSoftDependency/NewtonSoftDependency.csproj

@@ -26,7 +26,9 @@
   <ItemGroup>
     <PackageReference Include="MSTest.TestAdapter" Version="$(MSTestAdapterVersion)" />
     <PackageReference Include="MSTest.TestFramework" Version="$(MSTestAdapterVersion)" />
-    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+    <!-- TODO: This version would ideally be higher than the newtonsoft version we use in TP to show that the higher version is used.
+    Currently there is no higher stable version. -->
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 
   <ItemGroup>