New parameter to control automatic assembly exclusion (#1392)

New parameter to control automatic assembly exclusion

Author: daveMueller

Documentation/Changelog.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Breaking changes
+- New parameter `ExcludeAssembliesWithoutSources` to control automatic assembly exclusion [1164](https://github.com/coverlet-coverage/coverlet/issues/1164). The parameter `InstrumentModulesWithoutLocalSources` has been removed. since it can be handled by setting `ExcludeAssembliesWithoutSources` to `None`.
+- The default heuristics for determining whether to instrument an assembly has been changed. In previous versions any missing source file was taken as a signal that it was a third-party project that shouldn't be instrumented, with exceptions for some common file name patterns for source generators. Now only assemblies where no source files at all can be found are excluded from instrumentation, and the code for detecting source generator files have been removed. To get back to the behaviour that at least one missing file is sufficient to exclude an assembly, set `ExcludeAssembliesWithoutSources` to `MissingAny`, or use assembly exclusion filters for more fine-grained control.
+
 ## Release date 2022-10-29
 ### Packages  
 coverlet.msbuild 3.2.0  

Documentation/GlobalTool.md

@@ -14,31 +14,31 @@ Cross platform .NET Core code coverage tool 3.0.0.0
 Usage: coverlet [arguments] [options]
 
 Arguments:
-  <ASSEMBLY|DIRECTORY>         Path to the test assembly or application directory.
+  <ASSEMBLY|DIRECTORY>                  Path to the test assembly or application directory.
 
 Options:
-  -h|--help                    Show help information
-  -v|--version                 Show version information
-  -t|--target                  Path to the test runner application.
-  -a|--targetargs              Arguments to be passed to the test runner.
-  -o|--output                  Output of the generated coverage report
-  -v|--verbosity               Sets the verbosity level of the command. Allowed values are quiet, minimal, normal, detailed.
-  -f|--format                  Format of the generated coverage report.
-  --threshold                  Exits with error if the coverage % is below value.
-  --threshold-type             Coverage type to apply the threshold to.
-  --threshold-stat             Coverage statistic used to enforce the threshold value.
-  --exclude                    Filter expressions to exclude specific modules and types.
-  --include                    Filter expressions to include only specific modules and types.
-  --exclude-by-file            Glob patterns specifying source files to exclude.
-  --include-directory          Include directories containing additional assemblies to be instrumented.
-  --exclude-by-attribute       Attributes to exclude from code coverage.
-  --include-test-assembly      Specifies whether to report code coverage of the test assembly.
-  --single-hit                 Specifies whether to limit code coverage hit reporting to a single hit for each location
-  --skipautoprops              Neither track nor record auto-implemented properties.
-  --merge-with                 Path to existing coverage result to merge.
-  --use-source-link            Specifies whether to use SourceLink URIs in place of file system paths.
-  --does-not-return-attribute  Attributes that mark methods that do not return.
-  --instrument-modules-without-local-sources Specifies whether modules should be instrumented even if the sources from the PDBs can't be found locally.
+  -h|--help                             Show help information
+  -v|--version                          Show version information
+  -t|--target                           Path to the test runner application.
+  -a|--targetargs                       Arguments to be passed to the test runner.
+  -o|--output                           Output of the generated coverage report
+  -v|--verbosity                        Sets the verbosity level of the command. Allowed values are quiet, minimal, normal, detailed.
+  -f|--format                           Format of the generated coverage report.
+  --threshold                           Exits with error if the coverage % is below value.
+  --threshold-type                      Coverage type to apply the threshold to.
+  --threshold-stat                      Coverage statistic used to enforce the threshold value.
+  --exclude                             Filter expressions to exclude specific modules and types.
+  --include                             Filter expressions to include only specific modules and types.
+  --exclude-by-file                     Glob patterns specifying source files to exclude.
+  --include-directory                   Include directories containing additional assemblies to be instrumented.
+  --exclude-by-attribute                Attributes to exclude from code coverage.
+  --include-test-assembly               Specifies whether to report code coverage of the test assembly.
+  --single-hit                          Specifies whether to limit code coverage hit reporting to a single hit for each location
+  --skipautoprops                       Neither track nor record auto-implemented properties.
+  --merge-with                          Path to existing coverage result to merge.
+  --use-source-link                     Specifies whether to use SourceLink URIs in place of file system paths.
+  --does-not-return-attribute           Attributes that mark methods that do not return.
+  --exclude-assemblies-without-sources  Specifies behaviour of heuristic to ignore assemblies with missing source documents.
 ```
 
 NB. For a [multiple value] options you have to specify values multiple times i.e.

Documentation/MSBuildIntegration.md

@@ -228,3 +228,19 @@ To generate deterministc report the parameter is:
 ```
 /p:DeterministicReport=true
 ```
+
+## Exclude assemblies without sources from coverage
+
+The heuristic coverlet uses to determine if an assembly is a third-party dependency is based on the matching of the assembly`s source documents and the corresponding source files.
+This parameter has three different values to control the automatic assembly exclusion. 
+
+| Parameter | Description |
+|-----------|-------------|
+| MissingAll | Includes the assembly if at least one document is matching. In case the `ExcludeAssembliesWithoutSources` parameter is not specified the default value is `MissingAll`. |
+| MissingAny | Includes the assembly only if all documents can be matched to corresponding source files. |
+| None | No assembly is excluded. |
+
+Here is an example of how to specifiy the parameter:
+```
+/p:ExcludeAssembliesWithoutSources="MissingAny"
+```

Documentation/ReleasePlan.md

@@ -23,9 +23,9 @@ We release 3 components as NuGet packages:
 
 | Package               | Version |
 |:----------------------|:--------|
-|**coverlet.msbuild**   | 3.1.2   | 
-|**coverlet.console**   | 3.1.2   |
-|**coverlet.collector** | 3.1.2   |
+|**coverlet.msbuild**   | 3.2.0   | 
+|**coverlet.console**   | 3.2.0   |
+|**coverlet.collector** | 3.2.0   |
 
 
 | Release Date      | coverlet.msbuild | coverlet.console  | coverlet.collector| commit hash                              | notes                          |

Documentation/VSTestIntegration.md

@@ -97,8 +97,8 @@ These are a list of options that are supported by coverlet. These can be specifi
 | IncludeTestAssembly      | Include coverage of the test assembly.                                                                                                                          |
 | SkipAutoProps            | Neither track nor record auto-implemented properties.                                                                                                           |
 | DoesNotReturnAttribute   | Methods marked with these attributes are known not to return, statements following them will be excluded from coverage                                          |
-| DeterministicReport      | Generates deterministic report in context of deterministic build. Take a look at [documentation](DeterministicBuild.md) for further informations.               | 
-| InstrumentModulesWithoutLocalSources      | Specifies whether modules should be instrumented even if the sources from the PDBs can't be found locally.               | 
+| DeterministicReport      | Generates deterministic report in context of deterministic build. Take a look at [documentation](DeterministicBuild.md) for further informations.
+| ExcludeAssembliesWithoutSources |  Specifies whether to exclude assemblies without source. Options are either MissingAll, MissingAny or None. Default is MissingAll.|
 
 How to specify these options via runsettings?
 
@@ -120,7 +120,7 @@ How to specify these options via runsettings?
           <IncludeTestAssembly>true</IncludeTestAssembly>
           <SkipAutoProps>true</SkipAutoProps>
           <DeterministicReport>false</DeterministicReport>
-          <InstrumentModulesWithoutLocalSources>false</InstrumentModulesWithoutLocalSources>
+          <ExcludeAssembliesWithoutSources>MissingAll,MissingAny,None</ExcludeAssembliesWithoutSources>
         </Configuration>
       </DataCollector>
     </DataCollectors>

src/coverlet.collector/DataCollection/CoverageWrapper.cs

@@ -34,7 +34,7 @@ public Coverage CreateCoverage(CoverletSettings settings, ILogger coverletLogger
                 SkipAutoProps = settings.SkipAutoProps,
                 DoesNotReturnAttributes = settings.DoesNotReturnAttributes,
                 DeterministicReport = settings.DeterministicReport,
-                InstrumentModulesWithoutLocalSources = settings.InstrumentModulesWithoutLocalSources
+                ExcludeAssembliesWithoutSources = settings.ExcludeAssembliesWithoutSources
             };
 
             return new Coverage(

src/coverlet.collector/DataCollection/CoverletSettings.cs

@@ -82,9 +82,9 @@ internal class CoverletSettings
         public bool DeterministicReport { get; set; }
 
         /// <summary>
-        /// Instruments modules even if the sources from the PDBs can't be resolved.
+        /// Switch for heuristic to automatically exclude assemblies without source.
         /// </summary>
-        public bool InstrumentModulesWithoutLocalSources { get;  set; }
+        public string ExcludeAssembliesWithoutSources { get;  set; }
 
         public override string ToString()
         {
@@ -103,7 +103,7 @@ public override string ToString()
             builder.AppendFormat("SkipAutoProps: '{0}'", SkipAutoProps);
             builder.AppendFormat("DoesNotReturnAttributes: '{0}'", string.Join(",", DoesNotReturnAttributes ?? Enumerable.Empty<string>()));
             builder.AppendFormat("DeterministicReport: '{0}'", DeterministicReport);
-            builder.AppendFormat("InstrumentModulesWithoutLocalSources: '{0}'", InstrumentModulesWithoutLocalSources);
+            builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}'", ExcludeAssembliesWithoutSources);
 
             return builder.ToString();
         }

src/coverlet.collector/DataCollection/CoverletSettingsParser.cs

@@ -48,7 +48,7 @@ public CoverletSettings Parse(XmlElement configurationElement, IEnumerable<strin
                 coverletSettings.SkipAutoProps = ParseSkipAutoProps(configurationElement);
                 coverletSettings.DoesNotReturnAttributes = ParseDoesNotReturnAttributes(configurationElement);
                 coverletSettings.DeterministicReport = ParseDeterministicReport(configurationElement);
-                coverletSettings.InstrumentModulesWithoutLocalSources = ParseInstrumentModulesWithoutLocalSources(configurationElement);
+                coverletSettings.ExcludeAssembliesWithoutSources = ParseExcludeAssembliesWithoutSources(configurationElement);
             }
 
             coverletSettings.ReportFormats = ParseReportFormats(configurationElement);
@@ -213,15 +213,14 @@ private static bool ParseDeterministicReport(XmlElement configurationElement)
         }
 
         /// <summary>
-        /// Parse InstrumentModulesWithoutLocalSources flag
+        /// Parse ExcludeAssembliesWithoutSources flag
         /// </summary>
         /// <param name="configurationElement">Configuration element</param>
-        /// <returns>InstrumentModulesWithoutLocalSources flag</returns>
-        private static bool ParseInstrumentModulesWithoutLocalSources(XmlElement configurationElement)
+        /// <returns>ExcludeAssembliesWithoutSources flag</returns>
+        private static string ParseExcludeAssembliesWithoutSources(XmlElement configurationElement)
         {
-            XmlElement instrumentModulesWithoutLocalSourcesElement = configurationElement[CoverletConstants.InstrumentModulesWithoutLocalSources];
-            bool.TryParse(instrumentModulesWithoutLocalSourcesElement?.InnerText, out bool instrumentModulesWithoutLocalSources);
-            return instrumentModulesWithoutLocalSources;
+            XmlElement instrumentModulesWithoutLocalSourcesElement = configurationElement[CoverletConstants.ExcludeAssembliesWithoutSources];
+            return instrumentModulesWithoutLocalSourcesElement?.InnerText;
         }
 
         /// <summary>

src/coverlet.collector/Utilities/CoverletConstants.cs

@@ -26,6 +26,6 @@ internal static class CoverletConstants
         public const string SkipAutoProps = "SkipAutoProps";
         public const string DoesNotReturnAttributesElementName = "DoesNotReturnAttribute";
         public const string DeterministicReport = "DeterministicReport";
-        public const string InstrumentModulesWithoutLocalSources = "InstrumentModulesWithoutLocalSources";
+        public const string ExcludeAssembliesWithoutSources = "ExcludeAssembliesWithoutSources";
     }
 }

src/coverlet.console/Program.cs

@@ -70,7 +70,7 @@ static int Main(string[] args)
             CommandOption mergeWith = app.Option("--merge-with", "Path to existing coverage result to merge.", CommandOptionType.SingleValue);
             CommandOption useSourceLink = app.Option("--use-source-link", "Specifies whether to use SourceLink URIs in place of file system paths.", CommandOptionType.NoValue);
             CommandOption doesNotReturnAttributes = app.Option("--does-not-return-attribute", "Attributes that mark methods that do not return.", CommandOptionType.MultipleValue);
-            CommandOption instrumentModulesWithoutLocalSources = app.Option("--instrument-modules-without-local-sources", "Specifies whether modules should be instrumented even if the sources from the PDBs can't be found locally.", CommandOptionType.NoValue);
+            CommandOption excludeAssembliesWithoutSources = app.Option("--exclude-assemblies-without-sources", "Specifies behaviour of heuristic to ignore assemblies with missing source documents.", CommandOptionType.SingleValue);
 
             app.OnExecute(() =>
             {
@@ -99,7 +99,7 @@ static int Main(string[] args)
                     UseSourceLink = useSourceLink.HasValue(),
                     SkipAutoProps = skipAutoProp.HasValue(),
                     DoesNotReturnAttributes = doesNotReturnAttributes.Values.ToArray(),
-                    InstrumentModulesWithoutLocalSources = instrumentModulesWithoutLocalSources.HasValue(),
+                    ExcludeAssembliesWithoutSources = excludeAssembliesWithoutSources.Value()
                 };
 
                 ISourceRootTranslator sourceRootTranslator = serviceProvider.GetRequiredService<ISourceRootTranslator>();

src/coverlet.core/Abstractions/IInstrumentationHelper.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Toni Solarin-Sodara
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Coverlet.Core.Enums;
+
 namespace Coverlet.Core.Abstractions
 {
     internal interface IInstrumentationHelper
@@ -15,8 +17,8 @@ internal interface IInstrumentationHelper
         bool IsTypeExcluded(string module, string type, string[] excludeFilters);
         bool IsTypeIncluded(string module, string type, string[] includeFilters);
         void RestoreOriginalModule(string module, string identifier);
-        bool EmbeddedPortablePdbHasLocalSource(string module, out string firstNotFoundDocument);
-        bool PortablePdbHasLocalSource(string module, out string firstNotFoundDocument);
+        bool EmbeddedPortablePdbHasLocalSource(string module, AssemblySearchType excludeAssembliesWithoutSources);
+        bool PortablePdbHasLocalSource(string module, AssemblySearchType excludeAssembliesWithoutSources);
         bool IsLocalMethod(string method);
         void SetLogger(ILogger logger);
     }

src/coverlet.core/Coverage.cs

@@ -44,7 +44,7 @@ internal class CoverageParameters
         [DataMember]
         public bool DeterministicReport { get; set; }
         [DataMember]
-        public bool InstrumentModulesWithoutLocalSources { get; set; }
+        public string ExcludeAssembliesWithoutSources { get; set; }
     }
 
     internal class Coverage

src/coverlet.core/Enums/AssemblySearchType.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Coverlet.Core.Enums
+{
+    internal enum AssemblySearchType
+    {
+        MissingAny,
+        MissingAll,
+        None
+    }
+}