Add initial commit for package validation (#16992)

* Add intitial version of package Validation

* Add some basic tests

* addressing feedback

* fix and some more tests

* logging the errors on the fly , add GetLastStableVersion, removing duplicate tuples.

* fix test builds

* add some feedback

* adding a new test logger to test the output for tests

* addressing some mroe feedback around  running targets

* Restore fixes, baseline conditional validation

* Use full namespace in  task invocations

* React to ericstj's feedback

* address feedback

* merge checker with diagnostic bag

* use hashset for diaglist and add comments in the package

* fix failing test on ci and resx files

* add left and right to resx

Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>

Author: Anipik

sdk.sln

@@ -345,13 +345,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Compatibility", "Compatibil
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.ApiCompatibility", "src\Compatibility\Microsoft.DotNet.ApiCompatibility\Microsoft.DotNet.ApiCompatibility.csproj", "{3F5A028C-C51B-434A-8C10-37680CD2635C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DotNet.ApiCompatibility.Tests", "src\Tests\Microsoft.DotNet.ApiCompatibility.Tests\Microsoft.DotNet.ApiCompatibility.Tests.csproj", "{24F084ED-35BB-401E-89F5-63E5E22C3B3B}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.ApiCompatibility.Tests", "src\Tests\Microsoft.DotNet.ApiCompatibility.Tests\Microsoft.DotNet.ApiCompatibility.Tests.csproj", "{24F084ED-35BB-401E-89F5-63E5E22C3B3B}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Win32.Msi", "src\Microsoft.Win32.Msi\Microsoft.Win32.Msi.csproj", "{3D002392-6308-41DF-8BD5-224CCC5B049F}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Win32.Msi.Tests", "src\Tests\Microsoft.Win32.Msi.Tests\Microsoft.Win32.Msi.Tests.csproj", "{80932949-B8B2-4163-B325-76F8FDBE3897}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Win32.Msi.Manual.Tests", "src\Tests\Microsoft.Win32.Msi.Manual.Tests\Microsoft.Win32.Msi.Manual.Tests.csproj", "{EEF4C7DD-CDC9-44B6-8B4F-725647D54ED8}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.PackageValidation", "src\Compatibility\Microsoft.DotNet.PackageValidation\Microsoft.DotNet.PackageValidation.csproj", "{E56BEA9A-B52A-4781-9FF4-217439923319}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.PackageValidation.Tests", "src\Tests\Microsoft.DotNet.PackageValidation.Tests\Microsoft.DotNet.PackageValidation.Tests.csproj", "{69C03400-12AC-4E4D-B970-6A880616BF68}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -646,6 +651,14 @@ Global
 		{EEF4C7DD-CDC9-44B6-8B4F-725647D54ED8}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EEF4C7DD-CDC9-44B6-8B4F-725647D54ED8}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EEF4C7DD-CDC9-44B6-8B4F-725647D54ED8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E56BEA9A-B52A-4781-9FF4-217439923319}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E56BEA9A-B52A-4781-9FF4-217439923319}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E56BEA9A-B52A-4781-9FF4-217439923319}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E56BEA9A-B52A-4781-9FF4-217439923319}.Release|Any CPU.Build.0 = Release|Any CPU
+		{69C03400-12AC-4E4D-B970-6A880616BF68}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{69C03400-12AC-4E4D-B970-6A880616BF68}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{69C03400-12AC-4E4D-B970-6A880616BF68}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{69C03400-12AC-4E4D-B970-6A880616BF68}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -765,6 +778,8 @@ Global
 		{3D002392-6308-41DF-8BD5-224CCC5B049F} = {22AB674F-ED91-4FBC-BFEE-8A1E82F9F05E}
 		{80932949-B8B2-4163-B325-76F8FDBE3897} = {580D1AE7-AA8F-4912-8B76-105594E00B3B}
 		{EEF4C7DD-CDC9-44B6-8B4F-725647D54ED8} = {580D1AE7-AA8F-4912-8B76-105594E00B3B}
+		{E56BEA9A-B52A-4781-9FF4-217439923319} = {AF683E5C-421E-4DE0-ADD7-9841E5D12BFA}
+		{69C03400-12AC-4E4D-B970-6A880616BF68} = {580D1AE7-AA8F-4912-8B76-105594E00B3B}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FB8F26CE-4DE6-433F-B32A-79183020BBD6}

src/Compatibility/Microsoft.DotNet.ApiCompatibility/DiagnosticBag.cs

@@ -15,28 +15,34 @@ public class DiagnosticBag<T> where T : IDiagnostic
     {
         private readonly Dictionary<string, HashSet<string>> _ignore;
         private readonly HashSet<string> _noWarn;
-
         private readonly List<T> _differences = new();
 
         /// <summary>
         /// Instantiate an diagnostic bag with the provided settings to ignore diagnostics.
         /// </summary>
         /// <param name="noWarn">Comma separated list of diagnostic IDs to ignore.</param>
         /// <param name="ignoredDifferences">An array of differences to ignore based on diagnostic ID and reference ID.</param>
-        public DiagnosticBag(string noWarn, (string diagnosticId, string referenceId)[] ignoredDifferences)
+        public DiagnosticBag(string noWarn, (string diagnosticId, string referenceId)[] ignoredDifferences) : this (noWarn?.Split(';'), ignoredDifferences)
+        {
+        }
+
+        public DiagnosticBag(IEnumerable<string> noWarn, (string diagnosticId, string referenceId)[] ignoredDifferences)
         {
-            _noWarn = new HashSet<string>(noWarn?.Split(';'));
+            _noWarn = new HashSet<string>(noWarn);
             _ignore = new Dictionary<string, HashSet<string>>();
 
-            foreach ((string diagnosticId, string referenceId) in ignoredDifferences)
+            if (ignoredDifferences != null)
             {
-                if (!_ignore.TryGetValue(diagnosticId, out HashSet<string> members))
+                foreach ((string diagnosticId, string referenceId) in ignoredDifferences)
                 {
-                    members = new HashSet<string>();
-                    _ignore.Add(diagnosticId, members);
-                }
+                    if (!_ignore.TryGetValue(diagnosticId, out HashSet<string> members))
+                    {
+                        members = new HashSet<string>();
+                        _ignore.Add(diagnosticId, members);
+                    }
 
-                members.Add(referenceId);
+                    members.Add(referenceId);
+                }
             }
         }
 
@@ -50,24 +56,37 @@ public void AddRange(IEnumerable<T> differences)
                 Add(difference);
         }
 
+        public void Add(DiagnosticBag<T> bag)
+        {
+            AddRange(bag.Differences);
+        }
+
         /// <summary>
         /// Adds a difference to the diagnostic bag if they are not found in the exclusion settings.
         /// </summary>
         /// <param name="difference">The difference to add.</param>
         public void Add(T difference)
         {
-            if (_noWarn.Contains(difference.DiagnosticId))
-                return;
+            if (!Filter(difference.DiagnosticId, difference.ReferenceId))
+            {
+                _differences.Add(difference);
+            }
+        }
+
+        public bool Filter(string diagnosticId, string referenceId)
+        {
+            if (_noWarn.Contains(diagnosticId))
+                return true;
 
-            if (_ignore.TryGetValue(difference.DiagnosticId, out HashSet<string> members))
+            if (_ignore.TryGetValue(diagnosticId, out HashSet<string> members))
             {
-                if (members.Contains(difference.ReferenceId))
+                if (members.Contains(referenceId))
                 {
-                    return;
+                    return true;
                 }
             }
 
-            _differences.Add(difference);
+            return false;
         }
 
         /// <summary>

src/Compatibility/Microsoft.DotNet.PackageValidation/ApiCompatRunner.cs

@@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using Microsoft.CodeAnalysis;
+using Microsoft.DotNet.ApiCompatibility;
+using Microsoft.DotNet.ApiCompatibility.Abstractions;
+using NuGet.Common;
+
+namespace Microsoft.DotNet.PackageValidation
+{
+    /// <summary>
+    /// Runs ApiCompat over different assembly tuples.
+    /// </summary>
+    public class ApiCompatRunner
+    {
+        private List<(string leftAssemblyPackagePath, string leftAssemblyRelativePath, string rightAssemblyPackagePath, string rightAssemblyRelativePath, string assemblyName, string compatibilityReason, string header)> _queue = new();
+        private readonly ILogger _log;
+        private readonly ApiComparer _differ = new();
+
+        public ApiCompatRunner(string noWarn, (string, string)[] ignoredDifferences, ILogger log)
+        {
+            _log = log;
+            _differ.NoWarn = noWarn;
+            _differ.IgnoredDifferences = ignoredDifferences;
+        }
+
+        /// <summary>
+        /// Runs the api compat for the tuples in the queue.
+        /// </summary>
+        public void RunApiCompat()
+        {
+            foreach (var apicompatTuples in _queue.Distinct())
+            {
+                // TODO: Add Assembly version check.
+                // TODO: Add optimisations tuples.
+                // TODO: Run it Asynchronously.
+                using (Stream leftAssemblyStream = GetFileStreamFromPackage(apicompatTuples.leftAssemblyPackagePath, apicompatTuples.leftAssemblyRelativePath))
+                using (Stream rightAssemblyStream = GetFileStreamFromPackage(apicompatTuples.rightAssemblyPackagePath, apicompatTuples.rightAssemblyRelativePath))
+                {
+                    IAssemblySymbol leftSymbols = new AssemblySymbolLoader().LoadAssembly(apicompatTuples.assemblyName, leftAssemblyStream);
+                    IAssemblySymbol rightSymbols = new AssemblySymbolLoader().LoadAssembly(apicompatTuples.assemblyName, rightAssemblyStream);
+
+                    IEnumerable<CompatDifference> differences = _differ.GetDifferences(leftSymbols, rightSymbols);
+
+                    if (differences.Any())
+                    {
+                        _log.LogError(apicompatTuples.compatibilityReason);
+                        _log.LogError(apicompatTuples.header);
+                    }
+
+                    foreach (CompatDifference difference in differences)
+                    {
+                        _log.LogError(difference.ToString());
+                    }
+                }
+            }
+            _queue.Clear();
+        }
+
+        /// <summary>
+        /// Queues the api compat for 2 assemblies.
+        /// </summary>
+        /// <param name="leftPackagePath">Path to package containing left assembly.</param>
+        /// <param name="leftRelativePath">Relative left assembly path in package.</param>
+        /// <param name="rightPackagePath">Path to package containing right assembly.</param>
+        /// <param name="rightRelativePath">Relative right assembly path in package.</param>
+        /// <param name="assemblyName">The name of the assembly.</param>
+        /// <param name="compatibiltyReason">The reason for assembly compatibilty.</param>
+        /// <param name="header">The header for the api compat diagnostics.</param>
+        public void QueueApiCompat(string leftPackagePath, string leftRelativePath, string rightPackagePath, string rightRelativePath, string assemblyName, string compatibiltyReason, string header)
+        {
+            _queue.Add((leftPackagePath, leftRelativePath, rightPackagePath, rightRelativePath, assemblyName, compatibiltyReason, header));
+        }
+
+        private static Stream GetFileStreamFromPackage(string packagePath, string entry)
+        {
+            MemoryStream ms = new MemoryStream();
+            using (FileStream stream = File.OpenRead(packagePath))
+            {
+                var zipFile = new ZipArchive(stream);
+                zipFile.GetEntry(entry).Open().CopyTo(ms);
+                ms.Seek(0, SeekOrigin.Begin);
+            }
+            return ms;
+        }
+    }
+}

src/Compatibility/Microsoft.DotNet.PackageValidation/BaselinePackageValidator.cs

@@ -0,0 +1,126 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.DotNet.ApiCompatibility;
+using Microsoft.DotNet.ApiCompatibility.Abstractions;
+using NuGet.Common;
+using NuGet.ContentModel;
+using NuGet.Frameworks;
+
+namespace Microsoft.DotNet.PackageValidation
+{
+    /// <summary>
+    /// Validates that no target framework / rid support is dropped in the latest package.
+    /// Reports all the breaking changes in the latest package.
+    /// </summary>
+    public class BaselinePackageValidator
+    {
+        private static HashSet<string> s_diagList = new HashSet<string>{ DiagnosticIds.TargetFrameworkDropped, DiagnosticIds.TargetFrameworkAndRidPairDropped };
+
+        private readonly Package _baselinePackage;
+        private readonly bool _runApiCompat;
+        private readonly ApiCompatRunner _apiCompatRunner;
+        private readonly ILogger _log;
+        private readonly DiagnosticBag<IDiagnostic> _diagnosticBag;
+
+        public BaselinePackageValidator(Package baselinePackage, string noWarn, (string, string)[] ignoredDifferences, bool runApiCompat, ILogger log)
+        {
+            _baselinePackage = baselinePackage;
+            _runApiCompat = runApiCompat;
+            _log = log;
+            _apiCompatRunner = new(noWarn, ignoredDifferences, _log);
+            _diagnosticBag = new(noWarn?.Split(';')?.Where(t => s_diagList.Contains(t)), ignoredDifferences);
+        }
+
+        /// <summary>
+        /// Validates the latest nuget package doesnot drop any target framework/rid and does not introduce any breaking changes.
+        /// </summary>
+        /// <param name="package">Nuget Package that needs to be validated.</param>
+        public void Validate(Package package)
+        {
+            foreach (ContentItem baselineCompileTimeAsset in _baselinePackage.CompileAssets)
+            {
+                NuGetFramework baselineTargetFramework = (NuGetFramework)baselineCompileTimeAsset.Properties["tfm"];
+                ContentItem latestCompileTimeAsset = package.FindBestCompileAssetForFramework(baselineTargetFramework);
+                if (latestCompileTimeAsset == null)
+                {
+                    if (!_diagnosticBag.Filter(DiagnosticIds.TargetFrameworkDropped, baselineTargetFramework.ToString()))
+                    {
+                        string message = string.Format(Resources.MissingTargetFramework, baselineTargetFramework.ToString());
+                        _log.LogError(DiagnosticIds.TargetFrameworkDropped + " " + message);
+                    }
+                }
+                else if (_runApiCompat)
+                {
+                    _apiCompatRunner.QueueApiCompat(_baselinePackage.PackagePath,
+                        baselineCompileTimeAsset.Path,
+                        package.PackagePath,
+                        latestCompileTimeAsset.Path,
+                        Path.GetFileName(package.PackagePath),
+                        Resources.BaselineVersionValidatorHeader,
+                        string.Format(Resources.ApiCompatibilityHeader, baselineCompileTimeAsset.Path, latestCompileTimeAsset.Path));
+                }
+            }
+
+            foreach (ContentItem baselineRuntimeAsset in _baselinePackage.RuntimeAssets)
+            {
+                NuGetFramework baselineTargetFramework = (NuGetFramework)baselineRuntimeAsset.Properties["tfm"];
+                ContentItem latestRuntimeAsset = package.FindBestRuntimeAssetForFramework(baselineTargetFramework);
+                if (latestRuntimeAsset == null)
+                {
+                    if (!_diagnosticBag.Filter(DiagnosticIds.TargetFrameworkDropped, baselineTargetFramework.ToString()))
+                    {
+                        string message = string.Format(Resources.MissingTargetFramework, baselineTargetFramework.ToString());
+                        _log.LogError(DiagnosticIds.TargetFrameworkDropped + " " + message);
+                    }
+                }
+                else
+                {
+                    if (_runApiCompat)
+                    {
+                        _apiCompatRunner.QueueApiCompat(_baselinePackage.PackagePath, 
+                            baselineRuntimeAsset.Path,
+                            package.PackagePath, 
+                            latestRuntimeAsset.Path,
+                            Path.GetFileName(package.PackagePath),
+                            Resources.BaselineVersionValidatorHeader,
+                            string.Format(Resources.ApiCompatibilityHeader, baselineRuntimeAsset.Path, latestRuntimeAsset.Path));
+                    }
+                }
+            }
+
+            foreach (ContentItem baselineRuntimeSpecificAsset in _baselinePackage.RuntimeSpecificAssets)
+            {
+                NuGetFramework baselineTargetFramework = (NuGetFramework)baselineRuntimeSpecificAsset.Properties["tfm"];
+                string baselineRid = (string)baselineRuntimeSpecificAsset.Properties["rid"];
+                ContentItem latestRuntimeSpecificAsset = package.FindBestRuntimeAssetForFrameworkAndRuntime(baselineTargetFramework, baselineRid);
+                if (latestRuntimeSpecificAsset == null)
+                {
+                    if (!_diagnosticBag.Filter(DiagnosticIds.TargetFrameworkDropped, baselineTargetFramework.ToString() + "-" + baselineRid))
+                    {
+                        string message = string.Format(Resources.MissingTargetFrameworkAndRid, baselineTargetFramework.ToString(), baselineRid);
+                        _log.LogError(DiagnosticIds.TargetFrameworkAndRidPairDropped + " " + message);
+                    }
+                }
+                else
+                {
+                    if (_runApiCompat)
+                    {
+                        _apiCompatRunner.QueueApiCompat(_baselinePackage.PackagePath, 
+                            baselineRuntimeSpecificAsset.Path,
+                            package.PackagePath, 
+                            latestRuntimeSpecificAsset.Path,
+                            Path.GetFileName(package.PackagePath),
+                            Resources.BaselineVersionValidatorHeader,
+                            string.Format(Resources.ApiCompatibilityHeader, baselineRuntimeSpecificAsset.Path, latestRuntimeSpecificAsset.Path));
+                    }
+                }
+            }
+            
+            _apiCompatRunner.RunApiCompat();
+        }
+    }
+}