Skip to content

Commit d03287b

Browse files
adamsitnikadamsitnik
authored
PerfCollect diagnoser (#2117)
1 parent dbccef2 commit d03287b

File tree

14 files changed

+2573
-4
lines changed

14 files changed

+2573
-4
lines changed

‎samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using System.IO;
2+
using BenchmarkDotNet.Attributes;
3+
4+
namespace BenchmarkDotNet.Samples
5+
{
6+
[PerfCollectProfiler(performExtraBenchmarksRun: false)]
7+
public class IntroPerfCollectProfiler
8+
{
9+
private readonly string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
10+
private readonly string content = new string('a', 100_000);
11+
12+
[Benchmark]
13+
public void WriteAllText() => File.WriteAllText(path, content);
14+
15+
[GlobalCleanup]
16+
public void Delete() => File.Delete(path);
17+
}
18+
}

‎src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using System;
2+
using BenchmarkDotNet.Configs;
3+
using BenchmarkDotNet.Diagnosers;
4+
5+
namespace BenchmarkDotNet.Attributes
6+
{
7+
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
8+
public class PerfCollectProfilerAttribute : Attribute, IConfigSource
9+
{
10+
/// <param name="performExtraBenchmarksRun">When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default.</param>
11+
/// <param name="timeoutInSeconds">How long should we wait for the perfcollect script to finish processing the trace. 120s by default.</param>
12+
public PerfCollectProfilerAttribute(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120)
13+
{
14+
Config = ManualConfig.CreateEmpty().AddDiagnoser(new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun, timeoutInSeconds)));
15+
}
16+
17+
public IConfig Config { get; }
18+
}
19+
}

‎src/BenchmarkDotNet/BenchmarkDotNet.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
<PackageReference Include="Iced" Version="1.17.0" />
2121
<PackageReference Include="Microsoft.Diagnostics.Runtime" Version="2.2.332302" />
2222
<PackageReference Include="Perfolizer" Version="0.2.1" />
23+
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
2324
<PackageReference Include="System.Management" Version="6.0.0" />
2425
<PackageReference Include="System.Reflection.Emit" Version="4.7.0" />
2526
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" />

‎src/BenchmarkDotNet/Configs/ImmutableConfig.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ internal ImmutableConfig(
106106

107107
public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default);
108108

109+
internal bool HasPerfCollectProfiler() => diagnosers.OfType<PerfCollectProfiler>().Any();
110+
109111
public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser();
110112

111113
public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode)

‎src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,13 @@ private static IEnumerable<IDiagnoser> LoadDiagnosers()
3737
yield return new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig());
3838

3939
if (RuntimeInformation.IsNetCore)
40+
{
4041
yield return EventPipeProfiler.Default;
4142

43+
if (RuntimeInformation.IsLinux())
44+
yield return PerfCollectProfiler.Default;
45+
}
46+
4247
if (!RuntimeInformation.IsWindows())
4348
yield break;
4449

‎src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
using System.IO;
5+
using System.Linq;
6+
using BenchmarkDotNet.Analysers;
7+
using BenchmarkDotNet.Engines;
8+
using BenchmarkDotNet.Exporters;
9+
using BenchmarkDotNet.Extensions;
10+
using BenchmarkDotNet.Helpers;
11+
using BenchmarkDotNet.Jobs;
12+
using BenchmarkDotNet.Loggers;
13+
using BenchmarkDotNet.Portability;
14+
using BenchmarkDotNet.Reports;
15+
using BenchmarkDotNet.Running;
16+
using BenchmarkDotNet.Toolchains;
17+
using BenchmarkDotNet.Toolchains.CoreRun;
18+
using BenchmarkDotNet.Toolchains.CsProj;
19+
using BenchmarkDotNet.Toolchains.DotNetCli;
20+
using BenchmarkDotNet.Toolchains.NativeAot;
21+
using BenchmarkDotNet.Validators;
22+
using JetBrains.Annotations;
23+
using Mono.Unix.Native;
24+
25+
namespace BenchmarkDotNet.Diagnosers
26+
{
27+
public class PerfCollectProfiler : IProfiler
28+
{
29+
public static readonly IDiagnoser Default = new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun: false));
30+
31+
private readonly PerfCollectProfilerConfig config;
32+
private readonly DateTime creationTime = DateTime.Now;
33+
private readonly Dictionary<BenchmarkCase, FileInfo> benchmarkToTraceFile = new ();
34+
private readonly HashSet<string> cliPathWithSymbolsInstalled = new ();
35+
private FileInfo perfCollectFile;
36+
private Process perfCollectProcess;
37+
38+
[PublicAPI]
39+
public PerfCollectProfiler(PerfCollectProfilerConfig config) => this.config = config;
40+
41+
public string ShortName => "perf";
42+
43+
public IEnumerable<string> Ids => new[] { nameof(PerfCollectProfiler) };
44+
45+
public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
46+
47+
public IEnumerable<IAnalyser> Analysers => Array.Empty<IAnalyser>();
48+
49+
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => Array.Empty<Metric>();
50+
51+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => config.RunMode;
52+
53+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
54+
{
55+
if (!RuntimeInformation.IsLinux())
56+
{
57+
yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!");
58+
yield break;
59+
}
60+
61+
if (Syscall.getuid() != 0)
62+
{
63+
yield return new ValidationError(true, "You must run as root to use PerfCollectProfiler.");
64+
yield break;
65+
}
66+
67+
if (validationParameters.Benchmarks.Any() && !TryInstallPerfCollect(validationParameters))
68+
{
69+
yield return new ValidationError(true, "Failed to install perfcollect script. Please follow the instructions from https://github.com/dotnet/runtime/blob/main/docs/project/linux-performance-tracing.md");
70+
}
71+
}
72+
73+
public void DisplayResults(ILogger logger)
74+
{
75+
if (!benchmarkToTraceFile.Any())
76+
return;
77+
78+
logger.WriteLineInfo($"Exported {benchmarkToTraceFile.Count} trace file(s). Example:");
79+
logger.WriteLineInfo(benchmarkToTraceFile.Values.First().FullName);
80+
}
81+
82+
public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
83+
{
84+
if (signal == HostSignal.BeforeProcessStart)
85+
perfCollectProcess = StartCollection(parameters);
86+
else if (signal == HostSignal.AfterProcessExit)
87+
StopCollection(parameters);
88+
}
89+
90+
private bool TryInstallPerfCollect(ValidationParameters validationParameters)
91+
{
92+
var scriptInstallationDirectory = new DirectoryInfo(validationParameters.Config.ArtifactsPath).CreateIfNotExists();
93+
94+
perfCollectFile = new FileInfo(Path.Combine(scriptInstallationDirectory.FullName, "perfcollect"));
95+
if (perfCollectFile.Exists)
96+
{
97+
return true;
98+
}
99+
100+
var logger = validationParameters.Config.GetCompositeLogger();
101+
102+
string script = ResourceHelper.LoadTemplate(perfCollectFile.Name);
103+
File.WriteAllText(perfCollectFile.FullName, script);
104+
105+
if (Syscall.chmod(perfCollectFile.FullName, FilePermissions.S_IXUSR) != 0)
106+
{
107+
logger.WriteError($"Unable to make perfcollect script an executable, the last error was: {Syscall.GetLastError()}");
108+
}
109+
else
110+
{
111+
(int exitCode, var output) = ProcessHelper.RunAndReadOutputLineByLine(perfCollectFile.FullName, "install -force", perfCollectFile.Directory.FullName, null, includeErrors: true, logger);
112+
113+
if (exitCode == 0)
114+
{
115+
logger.WriteLine("Successfully installed perfcollect");
116+
return true;
117+
}
118+
119+
logger.WriteLineError("Failed to install perfcollect");
120+
foreach (var outputLine in output)
121+
{
122+
logger.WriteLine(outputLine);
123+
}
124+
}
125+
126+
if (perfCollectFile.Exists)
127+
{
128+
perfCollectFile.Delete(); // if the file exists it means that perfcollect is installed
129+
}
130+
131+
return false;
132+
}
133+
134+
private Process StartCollection(DiagnoserActionParameters parameters)
135+
{
136+
EnsureSymbolsForNativeRuntime(parameters);
137+
138+
var traceName = GetTraceFile(parameters, extension: null).Name;
139+
140+
var start = new ProcessStartInfo
141+
{
142+
FileName = perfCollectFile.FullName,
143+
Arguments = $"collect \"{traceName}\"",
144+
UseShellExecute = false,
145+
RedirectStandardOutput = true,
146+
CreateNoWindow = true,
147+
WorkingDirectory = perfCollectFile.Directory.FullName
148+
};
149+
150+
return Process.Start(start);
151+
}
152+
153+
private void StopCollection(DiagnoserActionParameters parameters)
154+
{
155+
var logger = parameters.Config.GetCompositeLogger();
156+
157+
try
158+
{
159+
if (!perfCollectProcess.HasExited)
160+
{
161+
if (Syscall.kill(perfCollectProcess.Id, Signum.SIGINT) != 0)
162+
{
163+
var lastError = Stdlib.GetLastError();
164+
logger.WriteLineError($"kill(perfcollect, SIGINT) failed with {lastError}");
165+
}
166+
167+
if (!perfCollectProcess.WaitForExit((int)config.Timeout.TotalMilliseconds))
168+
{
169+
logger.WriteLineError($"The perfcollect script did not stop in {config.Timeout.TotalSeconds}s. It's going to be force killed now.");
170+
logger.WriteLineInfo("You can create PerfCollectProfiler providing PerfCollectProfilerConfig with custom timeout value.");
171+
172+
perfCollectProcess.KillTree(); // kill the entire process tree
173+
}
174+
175+
FileInfo traceFile = GetTraceFile(parameters, "trace.zip");
176+
if (traceFile.Exists)
177+
{
178+
benchmarkToTraceFile[parameters.BenchmarkCase] = traceFile;
179+
}
180+
}
181+
else
182+
{
183+
logger.WriteLineError("For some reason the perfcollect script has finished sooner than expected.");
184+
logger.WriteLineInfo($"Please run '{perfCollectFile.FullName} install' as root and re-try.");
185+
}
186+
}
187+
finally
188+
{
189+
perfCollectProcess.Dispose();
190+
}
191+
}
192+
193+
private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters)
194+
{
195+
string cliPath = parameters.BenchmarkCase.GetToolchain() switch
196+
{
197+
CsProjCoreToolchain core => core.CustomDotNetCliPath,
198+
CoreRunToolchain coreRun => coreRun.CustomDotNetCliPath.FullName,
199+
NativeAotToolchain nativeAot => nativeAot.CustomDotNetCliPath,
200+
_ => DotNetCliCommandExecutor.DefaultDotNetCliPath.Value
201+
};
202+
203+
if (!cliPathWithSymbolsInstalled.Add(cliPath))
204+
{
205+
return;
206+
}
207+
208+
string sdkPath = DotNetCliCommandExecutor.GetSdkPath(cliPath); // /usr/share/dotnet/sdk/
209+
string dotnetPath = Path.GetDirectoryName(sdkPath); // /usr/share/dotnet/
210+
string[] missingSymbols = Directory.GetFiles(dotnetPath, "lib*.so", SearchOption.AllDirectories)
211+
.Where(nativeLibPath => !nativeLibPath.Contains("FallbackFolder") && !File.Exists(Path.ChangeExtension(nativeLibPath, "so.dbg")))
212+
.Select(Path.GetDirectoryName)
213+
.Distinct()
214+
.ToArray();
215+
216+
if (!missingSymbols.Any())
217+
{
218+
return; // the symbol files are already where we need them!
219+
}
220+
221+
ILogger logger = parameters.Config.GetCompositeLogger();
222+
// We install the tool in a dedicated directory in order to always use latest version and avoid issues with broken existing configs.
223+
string toolPath = Path.Combine(Path.GetTempPath(), "BenchmarkDotNet", "symbols");
224+
DotNetCliCommand cliCommand = new (
225+
cliPath: cliPath,
226+
arguments: $"tool install dotnet-symbol --tool-path \"{toolPath}\"",
227+
generateResult: null,
228+
logger: logger,
229+
buildPartition: null,
230+
environmentVariables: Array.Empty<EnvironmentVariable>(),
231+
timeout: TimeSpan.FromMinutes(3),
232+
logOutput: true); // the following commands might take a while and fail, let's log them
233+
234+
var installResult = DotNetCliCommandExecutor.Execute(cliCommand);
235+
if (!installResult.IsSuccess)
236+
{
237+
logger.WriteError("Unable to install dotnet symbol.");
238+
return;
239+
}
240+
241+
DotNetCliCommandExecutor.Execute(cliCommand
242+
.WithCliPath(Path.Combine(toolPath, "dotnet-symbol"))
243+
.WithArguments($"--recurse-subdirectories --symbols \"{dotnetPath}/dotnet\" \"{dotnetPath}/lib*.so\""));
244+
245+
DotNetCliCommandExecutor.Execute(cliCommand.WithArguments($"tool uninstall dotnet-symbol --tool-path \"{toolPath}\""));
246+
}
247+
248+
private FileInfo GetTraceFile(DiagnoserActionParameters parameters, string extension)
249+
=> new (ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension)
250+
.Replace(" ", "_")); // perfcollect does not allow for spaces in the trace file name
251+
}
252+
}

‎src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using System;
2+
3+
namespace BenchmarkDotNet.Diagnosers
4+
{
5+
public class PerfCollectProfilerConfig
6+
{
7+
/// <param name="performExtraBenchmarksRun">When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default.</param>
8+
/// <param name="timeoutInSeconds">How long should we wait for the perfcollect script to finish processing the trace. 120s by default.</param>
9+
public PerfCollectProfilerConfig(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120)
10+
{
11+
RunMode = performExtraBenchmarksRun ? RunMode.ExtraRun : RunMode.NoOverhead;
12+
Timeout = TimeSpan.FromSeconds(timeoutInSeconds);
13+
}
14+
15+
public TimeSpan Timeout { get; }
16+
17+
public RunMode RunMode { get; }
18+
}
19+
}

‎src/BenchmarkDotNet/Extensions/CommonExtensions.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,14 @@ internal static string CreateIfNotExists(this string directoryPath)
7979
return directoryPath;
8080
}
8181

82+
internal static DirectoryInfo CreateIfNotExists(this DirectoryInfo directory)
83+
{
84+
if (!directory.Exists)
85+
directory.Create();
86+
87+
return directory;
88+
}
89+
8290
internal static string DeleteFileIfExists(this string filePath)
8391
{
8492
if (File.Exists(filePath))

‎src/BenchmarkDotNet/Extensions/ProcessExtensions.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Diagnostics;
55
using System.IO;
66
using BenchmarkDotNet.Characteristics;
7+
using BenchmarkDotNet.Engines;
78
using BenchmarkDotNet.Environments;
89
using BenchmarkDotNet.Jobs;
910
using BenchmarkDotNet.Loggers;
@@ -128,6 +129,17 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm
128129
if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath))
129130
start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath;
130131

132+
if (benchmarkCase.Config.HasPerfCollectProfiler())
133+
{
134+
// enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace)
135+
start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1";
136+
start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1";
137+
// enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering)
138+
start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName;
139+
// workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version
140+
start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0";
141+
}
142+
131143
// corerun does not understand runtimeconfig.json files;
132144
// we have to set "COMPlus_GC*" environment variables as documented in
133145
// https://docs.microsoft.com/en-us/dotnet/core/run-time-config/garbage-collector

‎src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,10 @@ private static string GetFilePath(string fileName, DiagnoserActionParameters det
7171

7272
fileName = FolderNameHelper.ToFolderName(fileName);
7373

74-
return Path.Combine(details.Config.ArtifactsPath, $"{fileName}.{fileExtension}");
74+
if (!string.IsNullOrEmpty(fileExtension))
75+
fileName = $"{fileName}.{fileExtension}";
76+
77+
return Path.Combine(details.Config.ArtifactsPath, fileName);
7578
}
7679
}
7780
}

0 commit comments

Comments
 (0)