Improve error logging to diagnose unstable tests

src/BenchmarkDotNet/Running/BenchmarkRunner.cs

@@ -198,8 +198,8 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
 
                     if (buildResult.GenerateException != null)
                         logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}");
-                    if (buildResult.BuildException != null)
-                        logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}");
+                    if (buildResult.ErrorMessage != null)
+                        logger.WriteLineError($"// Build Error: {buildResult.ErrorMessage}");
 
                     success = true;
                 }
@@ -333,7 +333,7 @@ private static BuildResult Build(BuildPartition buildPartition, string rootArtif
             try
             {
                 if (!generateResult.IsGenerateSuccess)
-                    return BuildResult.Failure(generateResult);
+                    return BuildResult.Failure(generateResult, generateResult.GenerateException);
 
                 return toolchain.Builder.Build(generateResult, buildPartition, buildLogger);
             }

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs

@@ -50,12 +50,12 @@ public BuildResult RestoreThenBuild()
             var packagesResult = AddPackages();
 
             if (!packagesResult.IsSuccess)
-                return BuildResult.Failure(GenerateResult, new Exception(packagesResult.AllInformation));
+                return BuildResult.Failure(GenerateResult, packagesResult.AllInformation);
 
             var restoreResult = Restore();
 
             if (!restoreResult.IsSuccess)
-                return BuildResult.Failure(GenerateResult, new Exception(restoreResult.AllInformation));
+                return BuildResult.Failure(GenerateResult, restoreResult.AllInformation);
 
             var buildResult = Build().ToBuildResult(GenerateResult);
 
@@ -71,20 +71,20 @@ public BuildResult RestoreThenBuildThenPublish()
             var packagesResult = AddPackages();
 
             if (!packagesResult.IsSuccess)
-                return BuildResult.Failure(GenerateResult, new Exception(packagesResult.AllInformation));
+                return BuildResult.Failure(GenerateResult, packagesResult.AllInformation);
 
             var restoreResult = Restore();
 
             if (!restoreResult.IsSuccess)
-                return BuildResult.Failure(GenerateResult, new Exception(restoreResult.AllInformation));
+                return BuildResult.Failure(GenerateResult, restoreResult.AllInformation);
 
             var buildResult = Build();
 
             if (!buildResult.IsSuccess) // if we failed to do the full build, let's try with --no-dependencies
                 buildResult = BuildNoDependencies();
 
             if (!buildResult.IsSuccess)
-                return BuildResult.Failure(GenerateResult, new Exception(buildResult.AllInformation));
+                return BuildResult.Failure(GenerateResult, buildResult.AllInformation);
 
             var publishResult = Publish();
 

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs

@@ -37,6 +37,6 @@ public static DotNetCliCommandResult Failure(TimeSpan time, string standardError
         public BuildResult ToBuildResult(GenerateResult generateResult)
             => IsSuccess || File.Exists(generateResult.ArtifactsPaths.ExecutablePath) // dotnet cli could have successfully built the program, but returned 1 as exit code because it had some warnings
                 ? BuildResult.Success(generateResult)
-                : BuildResult.Failure(generateResult, new Exception(AllInformation));
+                : BuildResult.Failure(generateResult, AllInformation);
     }
 }

src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs

@@ -10,7 +10,7 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli
     [PublicAPI]
     public class NetCoreAppSettings
     {
-        public static readonly TimeSpan DefaultBuildTimeout = TimeSpan.FromMinutes(1); // it should always be a few seconds, but some users might have a bad internet connection
+        public static readonly TimeSpan DefaultBuildTimeout = TimeSpan.FromMinutes(2); // it should always be a few seconds, but some users might have a bad internet connection
 
         [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new NetCoreAppSettings("netcoreapp2.0", null, ".NET Core 2.0");
         [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new NetCoreAppSettings("netcoreapp2.1", null, ".NET Core 2.1");

src/BenchmarkDotNet/Toolchains/Results/BuildResult.cs

@@ -7,18 +7,26 @@ namespace BenchmarkDotNet.Toolchains.Results
     public class BuildResult : GenerateResult
     {
         public bool IsBuildSuccess { get; }
-        public Exception BuildException { get; }
+        public string ErrorMessage { get; }
 
-        private BuildResult(GenerateResult generateResult, bool isBuildSuccess, Exception buildException)
+        private BuildResult(GenerateResult generateResult, bool isBuildSuccess, string errorMessage)
             : base(generateResult.ArtifactsPaths, generateResult.IsGenerateSuccess, generateResult.GenerateException, generateResult.ArtifactsToCleanup)
         {
             IsBuildSuccess = isBuildSuccess;
-            BuildException = buildException;
+            ErrorMessage = errorMessage;
         }
 
-        [PublicAPI] public static BuildResult Success(GenerateResult generateResult) => new BuildResult(generateResult, true, null);
+        [PublicAPI]
+        public static BuildResult Success(GenerateResult generateResult)
+            => new BuildResult(generateResult, true, null);
 
-        [PublicAPI] public static BuildResult Failure(GenerateResult generateResult, Exception exception = null) => new BuildResult(generateResult, false, exception);
+        [PublicAPI]
+        public static BuildResult Failure(GenerateResult generateResult, string errorMessage) 
+            => new BuildResult(generateResult, false, errorMessage);
+
+        [PublicAPI]
+        public static BuildResult Failure(GenerateResult generateResult, Exception exception)
+            => new BuildResult(generateResult, false, $"Exception! {Environment.NewLine}Message: {exception.Message},{Environment.NewLine}Stack trace:{Environment.NewLine}{exception.StackTrace}");
 
         public override string ToString() => "BuildResult: " + (IsBuildSuccess ? "Success" : "Failure");
     }

src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs

@@ -81,7 +81,7 @@ private static (BuildResult result, AssemblyMetadata[] missingReference) Build(G
 
                 var missingReferences = GetMissingReferences(compilationErrors);
 
-                return (BuildResult.Failure(generateResult, new Exception(errors.ToString())), missingReferences);
+                return (BuildResult.Failure(generateResult, errors.ToString()), missingReferences);
             }
         }
 

tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs

@@ -35,7 +35,7 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled()
             var summary = CanExecute<CoreRtBenchmark>(config, fullValidation: false);
 
             Assert.All(summary.Reports, report => Assert.False(report.BuildResult.IsBuildSuccess));
-            Assert.All(summary.Reports, report => Assert.Contains("The configured timeout", report.BuildResult.BuildException.Message));
+            Assert.All(summary.Reports, report => Assert.Contains("The configured timeout", report.BuildResult.ErrorMessage));
         }
     }
 

tests/BenchmarkDotNet.IntegrationTests/ProcessorArchitectureTest.cs

@@ -12,14 +12,6 @@ namespace BenchmarkDotNet.IntegrationTests
 {
     public class ProcessorArchitectureTest : BenchmarkTestExecutor
     {
-        private class PlatformConfig : ManualConfig
-        {
-            public PlatformConfig(Platform platform)
-            {
-                Add(new Job(Job.Dry) { Environment = { Platform = platform } });
-            }
-        }
-
         const string X86FailedCaption = "// x86FAILED";
         const string X64FailedCaption = "// x64FAILED";
         const string AnyCpuOkCaption = "// AnyCpuOkCaption";
@@ -43,12 +35,14 @@ public void SpecifiedProccesorArchitectureMustBeRespected()
         private void Verify(Platform platform, Type benchmark, string failureText)
         {
             var logger = new OutputLogger(Output);
-            // make sure we get an output in the TestRunner log
-            var config = new PlatformConfig(platform).With(logger).With(DefaultColumnProviders.Instance);
+
+            var config = ManualConfig.CreateEmpty()
+                    .With(Job.Dry.With(platform))
+                    .With(logger); // make sure we get an output in the TestRunner log
 
             CanExecute(benchmark, config);
-            var testLog = logger.GetLog();
 
+            var testLog = logger.GetLog();
             Assert.DoesNotContain(failureText, testLog);
             Assert.DoesNotContain(BenchmarkNotFound, testLog);
         }

tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs

@@ -58,7 +58,7 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmark)
         private static BenchmarkReport CreateFailureReport(BenchmarkCase benchmark)
         {
             GenerateResult generateResult = GenerateResult.Failure(ArtifactsPaths.Empty, Array.Empty<string>());
-            BuildResult buildResult = BuildResult.Failure(generateResult);
+            BuildResult buildResult = BuildResult.Failure(generateResult, string.Empty);
             // Null may be legitimately passed as metrics to BenchmarkReport ctor here:
             // https://github.com/dotnet/BenchmarkDotNet/blob/89255c9fceb1b27c475a93d08c152349be4199e9/src/BenchmarkDotNet/Running/BenchmarkRunner.cs#L197
             return new BenchmarkReport(false, benchmark, generateResult, buildResult, default, default, default, default);