Merge pull request #1337 from dotnet/jnm2/tools2

Enable running example tests massively in parallel via `dotnet test` or UI

Author: jnm2

.github/workflows/tools-tests.yaml

@@ -33,4 +33,4 @@ jobs:
     - name: Run all tests
       run: |
         cd tools
-        dotnet test
+        dotnet test --filter Name!~ExampleTests

tools/ExampleExtractor/ExampleMetadata.cs

@@ -84,4 +84,6 @@ public class ExampleMetadata
 
     [JsonIgnore]
     public string Source => $"{MarkdownFile}:{StartLine}-{EndLine}";
+
+    public override string ToString() => Name;
 }

tools/ExampleTester/ExampleTester.csproj

@@ -1,16 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Basic.Reference.Assemblies.Net60" Version="1.8.2" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.14.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="4.14.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
+    <PackageReference Include="NUnit" Version="4.3.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="5.0.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
 

tools/ExampleTester/ExampleTests.cs

@@ -0,0 +1,35 @@
+using NUnit.Framework;
+using Utilities;
+
+[assembly: Parallelizable(ParallelScope.Children)]
+
+namespace ExampleTester;
+
+public static class ExampleTests
+{
+    private static TesterConfiguration TesterConfiguration { get; } = new(Path.Join(FindSlnDirectory(), "tmp"));
+
+    public static IEnumerable<object[]> LoadExamples() =>
+        from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory)
+        select new object[] { example };
+
+    [TestCaseSource(nameof(LoadExamples))]
+    public static async Task ExamplePasses(GeneratedExample example)
+    {
+        var logger = new StatusCheckLogger(TestContext.Out, "..", "Example tester");
+
+        if (!await example.Test(TesterConfiguration, logger))
+            Assert.Fail("There were one or more failures. See the logged output for details.");
+    }
+
+    private static string FindSlnDirectory()
+    {
+        for (string? current = AppDomain.CurrentDomain.BaseDirectory; current != null; current = Path.GetDirectoryName(current))
+        {
+            if (Directory.EnumerateFiles(current, "*.sln").Any())
+                return current;
+        }
+
+        throw new InvalidOperationException($"Can't find a directory containing a .sln file in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories.");
+    }
+}

tools/ExampleTester/GeneratedExample.cs

@@ -8,8 +8,10 @@
 
 namespace ExampleTester;
 
-internal class GeneratedExample
+public class GeneratedExample
 {
+    private static readonly object CodeExecutionLock = new();
+
     private readonly string directory;
     internal ExampleMetadata Metadata { get; }
 
@@ -20,6 +22,8 @@ private GeneratedExample(string directory)
         Metadata = JsonConvert.DeserializeObject<ExampleMetadata>(metadataJson) ?? throw new ArgumentException($"Invalid (null) metadata in {directory}");
     }
 
+    public override string? ToString() => Metadata.ToString();
+
     internal static List<GeneratedExample> LoadAllExamples(string parentDirectory) =>
         Directory.GetDirectories(parentDirectory).Select(Load).ToList();
 
@@ -134,46 +138,52 @@ bool ValidateOutput()
                 ? new object[] { Metadata.ExecutionArgs ?? new string[0] }
                 : new object[0];
 
-            var oldOut = Console.Out;
             List<string> actualLines;
             Exception? actualException = null;
-            try
+            lock (CodeExecutionLock)
             {
-                var builder = new StringBuilder();
-                Console.SetOut(new StringWriter(builder));
+                var oldOut = Console.Out;
                 try
                 {
-                    var result = method.Invoke(null, arguments);
-                    // For async Main methods, the compilation's entry point is still the Main
-                    // method, so we explicitly wait for the returned task just like the synthesized
-                    // entry point would.
-                    if (result is Task task)
+                    var builder = new StringBuilder();
+                    Console.SetOut(new StringWriter(builder));
+                    try
+                    {
+                        var result = method.Invoke(null, arguments);
+                        // For async Main methods, the compilation's entry point is still the Main
+                        // method, so we explicitly wait for the returned task just like the synthesized
+                        // entry point would.
+                        if (result is Task task)
+                        {
+                            task.GetAwaiter().GetResult();
+                        }
+
+                        // For some reason, we don't *actually* get the result of all finalizers
+                        // without this. We shouldn't need it (as relevant examples already have it) but
+                        // code that works outside the test harness doesn't work inside it.
+                        GC.Collect();
+                        GC.WaitForPendingFinalizers();
+                    }
+                    catch (TargetInvocationException outer)
                     {
-                        task.GetAwaiter().GetResult();
+                        actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception");
                     }
-                    // For some reason, we don't *actually* get the result of all finalizers
-                    // without this. We shouldn't need it (as relevant examples already have it) but
-                    // code that works outside the test harness doesn't work inside it.
-                    GC.Collect();
-                    GC.WaitForPendingFinalizers();
+
+                    // Skip blank lines, to avoid unnecessary trailing empties.
+                    // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata
+                    // or listed console output.
+                    actualLines = builder.ToString()
+                        .Replace("\r\n", "\n")
+                        .Split('\n')
+                        .Select(line => line.TrimEnd())
+                        .Where(line => line != "").ToList();
                 }
-                catch (TargetInvocationException outer)
+                finally
                 {
-                    actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception");
+                    Console.SetOut(oldOut);
                 }
-                // Skip blank lines, to avoid unnecessary trailing empties.
-                // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata
-                // or listed console output.
-                actualLines = builder.ToString()
-                    .Replace("\r\n", "\n")
-                    .Split('\n')
-                    .Select(line => line.TrimEnd())
-                    .Where(line => line != "").ToList();
-            }
-            finally
-            {
-                Console.SetOut(oldOut);
             }
+
             var expectedLines = Metadata.ExpectedOutput ?? new List<string>();
             return ValidateException(actualException, Metadata.ExpectedException) &&
                 (Metadata.IgnoreOutput || ValidateExpectedAgainstActual("output", expectedLines, actualLines));

tools/ExampleTester/Program.cs

@@ -2,7 +2,7 @@
 using System.CommandLine;
 using Utilities;
 
-var logger = new StatusCheckLogger("..", "Example tester");
+var logger = new StatusCheckLogger(Console.Out, "..", "Example tester");
 var headSha = Environment.GetEnvironmentVariable("HEAD_SHA");
 var token = Environment.GetEnvironmentVariable("GH_TOKEN");
 

tools/ExampleTester/TesterConfiguration.cs

@@ -5,12 +5,9 @@ namespace ExampleTester;
 
 public record TesterConfiguration(
     string ExtractedOutputDirectory,
-    bool Quiet,
-    string? SourceFile,
-    string? ExampleName)
-{
-    
-}
+    bool Quiet = false,
+    string? SourceFile = null,
+    string? ExampleName = null);
 
 public class TesterConfigurationBinder : BinderBase<TesterConfiguration>
 {

tools/MarkdownConverter/Spec/Reporter.cs

@@ -28,7 +28,7 @@ public Reporter() : this(null, null) { }
     public Reporter(Reporter? parent, string? filename)
     {
         // This is needed so that all Reporters share the same GitHub logger.
-        this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger("..", "Markdown to Word Converter");
+        this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger(Console.Out, "..", "Markdown to Word Converter");
         this.parent = parent;
         Location = new SourceLocation(filename, null, null, null);
     }

tools/StandardAnchorTags/Program.cs

@@ -25,7 +25,7 @@ public class Program
     /// <returns>0 on success, non-zero on failure</returns>
     static async Task<int> Main(string owner, string repo, bool dryrun =false)
     {
-        var logger = new StatusCheckLogger("..", "TOC and Anchor updater");
+        var logger = new StatusCheckLogger(Console.Out, "..", "TOC and Anchor updater");
         var headSha = Environment.GetEnvironmentVariable("HEAD_SHA");
         var token = Environment.GetEnvironmentVariable("GH_TOKEN");
         using FileStream openStream = File.OpenRead(FilesPath);

tools/Utilities/StatusCheckLogger.cs

@@ -22,15 +22,15 @@ public record StatusCheckMessage(string file, int StartLine, int EndLine, string
 /// </remarks>
 /// <param name="pathToRoot">The path to the root of the repository</param>
 /// <param name="toolName">The name of the tool that is running the check</param>
-public class StatusCheckLogger(string pathToRoot, string toolName)
+public class StatusCheckLogger(TextWriter writer, string pathToRoot, string toolName)
 {
     private List<NewCheckRunAnnotation> annotations = [];
     public bool Success { get; private set; } = true;
 
     // Utility method to format the path to unix style, from the root of the repository.
     private string FormatPath(string path) => Path.GetRelativePath(pathToRoot, path).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
 
-    private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => Console.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}");
+    private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => writer.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}");
 
     /// <summary>
     /// Log a notice from the status check to the console only
@@ -178,9 +178,9 @@ public async Task BuildCheckRunResult(string token, string owner, string repo, s
         // Once running on a branch on the dotnet org, this should work correctly.
         catch (ForbiddenException e)
         {
-            Console.WriteLine("===== WARNING: Could not create a check run.=====");
-            Console.WriteLine("Exception details:");
-            Console.WriteLine(e);
+            writer.WriteLine("===== WARNING: Could not create a check run.=====");
+            writer.WriteLine("Exception details:");
+            writer.WriteLine(e);
         }
     }
 }