PerfCollect diagnoser

samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs

@@ -0,0 +1,18 @@
+using System.IO;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Samples
+{
+    [PerfCollectProfiler(performExtraBenchmarksRun: false)]
+    public class IntroPerfCollectProfiler
+    {
+        private readonly string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        private readonly string content = new string('a', 100_000);
+
+        [Benchmark]
+        public void WriteAllText() => File.WriteAllText(path, content);
+
+        [GlobalCleanup]
+        public void Delete() => File.Delete(path);
+    }
+}

src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs

@@ -0,0 +1,19 @@
+using System;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
+    public class PerfCollectProfilerAttribute : Attribute, IConfigSource
+    {
+        /// <param name="performExtraBenchmarksRun">When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default.</param>
+        /// <param name="timeoutInSeconds">How long should we wait for the perfcollect script to finish processing the trace. 120s by default.</param>
+        public PerfCollectProfilerAttribute(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120)
+        {
+            Config = ManualConfig.CreateEmpty().AddDiagnoser(new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun, timeoutInSeconds)));
+        }
+
+        public IConfig Config { get; }
+    }
+}

src/BenchmarkDotNet/BenchmarkDotNet.csproj

@@ -20,6 +20,7 @@
     <PackageReference Include="Iced" Version="1.17.0" />
     <PackageReference Include="Microsoft.Diagnostics.Runtime" Version="2.2.332302" />
     <PackageReference Include="Perfolizer" Version="0.2.1" />
+    <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageReference Include="System.Management" Version="6.0.0" />
     <PackageReference Include="System.Reflection.Emit" Version="4.7.0" />
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" />

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -106,6 +106,8 @@ internal ImmutableConfig(
 
         public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default);
 
+        internal bool HasPerfCollectProfiler() => diagnosers.OfType<PerfCollectProfiler>().Any();
+
         public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser();
 
         public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode)

src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs

@@ -37,8 +37,13 @@ private static IEnumerable<IDiagnoser> LoadDiagnosers()
             yield return new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig());
 
             if (RuntimeInformation.IsNetCore)
+            {
                 yield return EventPipeProfiler.Default;
 
+                if (RuntimeInformation.IsLinux())
+                    yield return PerfCollectProfiler.Default;
+            }
+
             if (!RuntimeInformation.IsWindows())
                 yield break;
 

src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs

@@ -0,0 +1,252 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Helpers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Portability;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CoreRun;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Toolchains.NativeAot;
+using BenchmarkDotNet.Validators;
+using JetBrains.Annotations;
+using Mono.Unix.Native;
+
+namespace BenchmarkDotNet.Diagnosers
+{
+    public class PerfCollectProfiler : IProfiler
+    {
+        public static readonly IDiagnoser Default = new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun: false));
+
+        private readonly PerfCollectProfilerConfig config;
+        private readonly DateTime creationTime = DateTime.Now;
+        private readonly Dictionary<BenchmarkCase, FileInfo> benchmarkToTraceFile = new ();
+        private readonly HashSet<string> cliPathWithSymbolsInstalled = new ();
+        private FileInfo perfCollectFile;
+        private Process perfCollectProcess;
+
+        [PublicAPI]
+        public PerfCollectProfiler(PerfCollectProfilerConfig config) => this.config = config;
+
+        public string ShortName => "perf";
+
+        public IEnumerable<string> Ids => new[] { nameof(PerfCollectProfiler) };
+
+        public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
+
+        public IEnumerable<IAnalyser> Analysers => Array.Empty<IAnalyser>();
+
+        public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => Array.Empty<Metric>();
+
+        public RunMode GetRunMode(BenchmarkCase benchmarkCase) => config.RunMode;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        {
+            if (!RuntimeInformation.IsLinux())
+            {
+                yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!");
+                yield break;
+            }
+
+            if (Syscall.getuid() != 0)
+            {
+                yield return new ValidationError(true, "You must run as root to use PerfCollectProfiler.");
+                yield break;
+            }
+
+            if (validationParameters.Benchmarks.Any() && !TryInstallPerfCollect(validationParameters))
+            {
+                yield return new ValidationError(true, "Failed to install perfcollect script. Please follow the instructions from https://github.com/dotnet/runtime/blob/main/docs/project/linux-performance-tracing.md");
+            }
+        }
+
+        public void DisplayResults(ILogger logger)
+        {
+            if (!benchmarkToTraceFile.Any())
+                return;
+
+            logger.WriteLineInfo($"Exported {benchmarkToTraceFile.Count} trace file(s). Example:");
+            logger.WriteLineInfo(benchmarkToTraceFile.Values.First().FullName);
+        }
+
+        public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
+        {
+            if (signal == HostSignal.BeforeProcessStart)
+                perfCollectProcess = StartCollection(parameters);
+            else if (signal == HostSignal.AfterProcessExit)
+                StopCollection(parameters);
+        }
+
+        private bool TryInstallPerfCollect(ValidationParameters validationParameters)
+        {
+            var scriptInstallationDirectory = new DirectoryInfo(validationParameters.Config.ArtifactsPath).CreateIfNotExists();
+
+            perfCollectFile = new FileInfo(Path.Combine(scriptInstallationDirectory.FullName, "perfcollect"));
+            if (perfCollectFile.Exists)
+            {
+                return true;
+            }
+
+            var logger = validationParameters.Config.GetCompositeLogger();
+
+            string script = ResourceHelper.LoadTemplate(perfCollectFile.Name);
+            File.WriteAllText(perfCollectFile.FullName, script);
+
+            if (Syscall.chmod(perfCollectFile.FullName, FilePermissions.S_IXUSR) != 0)
+            {
+                logger.WriteError($"Unable to make perfcollect script an executable, the last error was: {Syscall.GetLastError()}");
+            }
+            else
+            {
+                (int exitCode, var output) = ProcessHelper.RunAndReadOutputLineByLine(perfCollectFile.FullName, "install -force", perfCollectFile.Directory.FullName, null, includeErrors: true, logger);
+
+                if (exitCode == 0)
+                {
+                    logger.WriteLine("Successfully installed perfcollect");
+                    return true;
+                }
+
+                logger.WriteLineError("Failed to install perfcollect");
+                foreach (var outputLine in output)
+                {
+                    logger.WriteLine(outputLine);
+                }
+            }
+
+            if (perfCollectFile.Exists)
+            {
+                perfCollectFile.Delete(); // if the file exists it means that perfcollect is installed
+            }
+
+            return false;
+        }
+
+        private Process StartCollection(DiagnoserActionParameters parameters)
+        {
+            EnsureSymbolsForNativeRuntime(parameters);
+
+            var traceName = GetTraceFile(parameters, extension: null).Name;
+
+            var start = new ProcessStartInfo
+            {
+                FileName = perfCollectFile.FullName,
+                Arguments = $"collect \"{traceName}\"",
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                CreateNoWindow = true,
+                WorkingDirectory = perfCollectFile.Directory.FullName
+            };
+
+            return Process.Start(start);
+        }
+
+        private void StopCollection(DiagnoserActionParameters parameters)
+        {
+            var logger = parameters.Config.GetCompositeLogger();
+
+            try
+            {
+                if (!perfCollectProcess.HasExited)
+                {
+                    if (Syscall.kill(perfCollectProcess.Id, Signum.SIGINT) != 0)
+                    {
+                        var lastError = Stdlib.GetLastError();
+                        logger.WriteLineError($"kill(perfcollect, SIGINT) failed with {lastError}");
+                    }
+
+                    if (!perfCollectProcess.WaitForExit((int)config.Timeout.TotalMilliseconds))
+                    {
+                        logger.WriteLineError($"The perfcollect script did not stop in {config.Timeout.TotalSeconds}s. It's going to be force killed now.");
+                        logger.WriteLineInfo("You can create PerfCollectProfiler providing PerfCollectProfilerConfig with custom timeout value.");
+
+                        perfCollectProcess.KillTree(); // kill the entire process tree
+                    }
+
+                    FileInfo traceFile = GetTraceFile(parameters, "trace.zip");
+                    if (traceFile.Exists)
+                    {
+                        benchmarkToTraceFile[parameters.BenchmarkCase] = traceFile;
+                    }
+                }
+                else
+                {
+                    logger.WriteLineError("For some reason the perfcollect script has finished sooner than expected.");
+                    logger.WriteLineInfo($"Please run '{perfCollectFile.FullName} install' as root and re-try.");
+                }
+            }
+            finally
+            {
+                perfCollectProcess.Dispose();
+            }
+        }
+
+        private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters)
+        {
+            string cliPath = parameters.BenchmarkCase.GetToolchain() switch
+            {
+                CsProjCoreToolchain core => core.CustomDotNetCliPath,
+                CoreRunToolchain coreRun => coreRun.CustomDotNetCliPath.FullName,
+                NativeAotToolchain nativeAot => nativeAot.CustomDotNetCliPath,
+                _ => DotNetCliCommandExecutor.DefaultDotNetCliPath.Value
+            };
+
+            if (!cliPathWithSymbolsInstalled.Add(cliPath))
+            {
+                return;
+            }
+
+            string sdkPath = DotNetCliCommandExecutor.GetSdkPath(cliPath); // /usr/share/dotnet/sdk/
+            string dotnetPath = Path.GetDirectoryName(sdkPath); // /usr/share/dotnet/
+            string[] missingSymbols = Directory.GetFiles(dotnetPath, "lib*.so", SearchOption.AllDirectories)
+                .Where(nativeLibPath => !nativeLibPath.Contains("FallbackFolder") && !File.Exists(Path.ChangeExtension(nativeLibPath, "so.dbg")))
+                .Select(Path.GetDirectoryName)
+                .Distinct()
+                .ToArray();
+
+            if (!missingSymbols.Any())
+            {
+                return; // the symbol files are already where we need them!
+            }
+
+            ILogger logger = parameters.Config.GetCompositeLogger();
+            // We install the tool in a dedicated directory in order to always use latest version and avoid issues with broken existing configs.
+            string toolPath = Path.Combine(Path.GetTempPath(), "BenchmarkDotNet", "symbols");
+            DotNetCliCommand cliCommand = new (
+                cliPath: cliPath,
+                arguments: $"tool install dotnet-symbol --tool-path \"{toolPath}\"",
+                generateResult: null,
+                logger: logger,
+                buildPartition: null,
+                environmentVariables: Array.Empty<EnvironmentVariable>(),
+                timeout: TimeSpan.FromMinutes(3),
+                logOutput: true); // the following commands might take a while and fail, let's log them
+
+            var installResult = DotNetCliCommandExecutor.Execute(cliCommand);
+            if (!installResult.IsSuccess)
+            {
+                logger.WriteError("Unable to install dotnet symbol.");
+                return;
+            }
+
+            DotNetCliCommandExecutor.Execute(cliCommand
+                .WithCliPath(Path.Combine(toolPath, "dotnet-symbol"))
+                .WithArguments($"--recurse-subdirectories --symbols \"{dotnetPath}/dotnet\" \"{dotnetPath}/lib*.so\""));
+
+            DotNetCliCommandExecutor.Execute(cliCommand.WithArguments($"tool uninstall dotnet-symbol --tool-path \"{toolPath}\""));
+        }
+
+        private FileInfo GetTraceFile(DiagnoserActionParameters parameters, string extension)
+            => new (ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension)
+                    .Replace(" ", "_")); // perfcollect does not allow for spaces in the trace file name
+    }
+}

src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs

@@ -0,0 +1,19 @@
+using System;
+
+namespace BenchmarkDotNet.Diagnosers
+{
+    public class PerfCollectProfilerConfig
+    {
+        /// <param name="performExtraBenchmarksRun">When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default.</param>
+        /// <param name="timeoutInSeconds">How long should we wait for the perfcollect script to finish processing the trace. 120s by default.</param>
+        public PerfCollectProfilerConfig(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120)
+        {
+            RunMode = performExtraBenchmarksRun ? RunMode.ExtraRun : RunMode.NoOverhead;
+            Timeout = TimeSpan.FromSeconds(timeoutInSeconds);
+        }
+
+        public TimeSpan Timeout { get; }
+
+        public RunMode RunMode { get; }
+    }
+}

src/BenchmarkDotNet/Extensions/CommonExtensions.cs

@@ -79,6 +79,14 @@ internal static string CreateIfNotExists(this string directoryPath)
             return directoryPath;
         }
 
+        internal static DirectoryInfo CreateIfNotExists(this DirectoryInfo directory)
+        {
+            if (!directory.Exists)
+                directory.Create();
+
+            return directory;
+        }
+
         internal static string DeleteFileIfExists(this string filePath)
         {
             if (File.Exists(filePath))

src/BenchmarkDotNet/Extensions/ProcessExtensions.cs

@@ -4,6 +4,7 @@
 using System.Diagnostics;
 using System.IO;
 using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
@@ -128,6 +129,17 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm
             if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath))
                 start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath;
 
+            if (benchmarkCase.Config.HasPerfCollectProfiler())
+            {
+                // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace)
+                start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1";
+                start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1";
+                // enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering)
+                start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName;
+                // workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version
+                start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0";
+            }
+
             // corerun does not understand runtimeconfig.json files;
             // we have to set "COMPlus_GC*" environment variables as documented in
             // https://docs.microsoft.com/en-us/dotnet/core/run-time-config/garbage-collector

src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs

@@ -71,7 +71,10 @@ private static string GetFilePath(string fileName, DiagnoserActionParameters det
 
             fileName = FolderNameHelper.ToFolderName(fileName);
 
-            return Path.Combine(details.Config.ArtifactsPath, $"{fileName}.{fileExtension}");
+            if (!string.IsNullOrEmpty(fileExtension))
+                fileName = $"{fileName}.{fileExtension}";
+
+            return Path.Combine(details.Config.ArtifactsPath, fileName);
         }
     }
 }