Add collect-linux mode to dotnet-trace integration

docker/agent/run.sh

@@ -43,10 +43,16 @@ fi
 # cgroupfs is mapped to allow docker to create cgroups without permissions issues (cgroup v2)
 # set cgroupns to host to allow the container to share the host's cgroup namespace (matches v1 and v2 namespace modes: https://docs.docker.com/engine/containers/runmetrics/#running-docker-on-cgroup-v2)
 # docker.sock is mapped to be able to manage other docker instances from this one
+# /sys/kernel/tracing is mapped so `dotnet-trace collect-linux` (DotNetTraceCollectMode=collect-linux)
+# can read tracefs to resolve perf_event_open ring-buffer records. EventPipe-based collection
+# (DotNetTraceCollectMode=collect, or perfcollect/Collect=true) does not need this mount; only
+# collect-linux does. Requires tracefs to be mounted on the host (Ubuntu 24.04 + systemd auto-mounts
+# it at boot; check with `mount | grep tracefs`).
 docker run -it --name $name -d --network host --restart always \
     --log-opt max-size=1G --privileged \
     --cgroupns=host \
     -v /sys/fs/cgroup/:/sys/fs/cgroup/ \
+    -v /sys/kernel/tracing:/sys/kernel/tracing \
     -v /var/run/docker.sock:/var/run/docker.sock $dockerargs \
     crank-agent \
     --url $url $others

src/Microsoft.Crank.Agent/TraceExtensions.cs

@@ -96,6 +96,35 @@ public static IEnumerable<EventPipeProvider> ToCLREventPipeProviders(string clre
             return new [] { new EventPipeProvider(CLREventProviderName, defaultEventLevel, clrEventsKeywordsMask, null) };
         }
 
+        // Returns true when `expression` is a non-empty '+'-joined list of recognized
+        // CLR keyword names (case-insensitive). Used to classify provider tokens for
+        // the `dotnet-trace` CLI, where unknown keywords passed via `--clrevents`
+        // are rejected. Differs from `ToCLREventPipeProviders`, which silently drops
+        // unknown parts and only requires *one* recognized keyword.
+        internal static bool IsRecognizedClrKeywordExpression(string expression)
+        {
+            if (String.IsNullOrEmpty(expression))
+            {
+                return false;
+            }
+
+            var parts = expression.Split('+', StringSplitOptions.RemoveEmptyEntries);
+            if (parts.Length == 0)
+            {
+                return false;
+            }
+
+            foreach (var part in parts)
+            {
+                if (!CLREventKeywords.ContainsKey(part))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
         private static EventLevel GetEventLevel(string token)
         {
             if (Int32.TryParse(token, out int level) && level >= 0)

src/Microsoft.Crank.Controller/Documentation.cs

@@ -59,6 +59,14 @@ be deployed as stand-alone.
                                                                 events are being dropped (e.g., high-rate allocation sampling).
   --[JOB].dotnetTraceRequestRundown <true|false>                Whether to request a rundown phase at the end of the EventPipe session. Default is true. Set to false to 
                                                                 shorten stop latency on long benchmarks where rundown dominates.
+  --[JOB].dotnetTraceCollectMode <mode>                         Selects how dotnet-trace captures the trace. ""default"" (in-process EventPipe via DiagnosticsClient, today's 
+                                                                behavior), ""collect"" (shells out to the dotnet-trace CLI's collect verb), or ""collect-linux"" (shells out 
+                                                                to collect-linux for perf_event-based whole-machine sampling with kernel + native frames). collect-linux is 
+                                                                Linux-only and requires root (effective UID 0) and kernel >= 6.4; the job hard-fails (no silent fallback) if 
+                                                                any prerequisite is not met. Default is ""default"".
+  --[JOB].dotnetTraceStopTimeoutSec <seconds>                   Grace period (seconds) to wait for the dotnet-trace CLI to finalize a trace after SIGINT, only used by the 
+                                                                ""collect"" and ""collect-linux"" modes. A value of 0 selects the per-mode default (60s for collect, 180s for 
+                                                                collect-linux). A timeout logs [WRN] and keeps the partial trace but does not fail the job.
   --[JOB].additionalProcesses <process-name>                    Name of the processes for which the CPU usage should be recorded. Can be used multiple times to define multiple values.
   --[JOB].options.traceOutput <filename>                        The name of the trace file. Can be a file prefix (app will add *.DATE*.zip) , or a specific name and no DATE* 
                                                                 will be added e.g., c:\traces\mytrace

src/Microsoft.Crank.Controller/README.md

@@ -113,6 +113,14 @@ Run 'crank [command] -?|-h|--help' for more information about a command.
                                                                 events are being dropped (e.g., high-rate allocation sampling).
   --[JOB].dotnetTraceRequestRundown <true|false>                Whether to request a rundown phase at the end of the EventPipe session. Default is true. Set to false to
                                                                 shorten stop latency on long benchmarks where rundown dominates.
+  --[JOB].dotnetTraceCollectMode <mode>                         Selects how dotnet-trace captures the trace. "default" (in-process EventPipe via DiagnosticsClient, today's
+                                                                behavior), "collect" (shells out to the dotnet-trace CLI's collect verb), or "collect-linux" (shells out
+                                                                to collect-linux for perf_event-based whole-machine sampling with kernel + native frames). collect-linux is
+                                                                Linux-only and requires root (effective UID 0) and kernel >= 6.4; the job hard-fails (no silent fallback) if
+                                                                any prerequisite is not met. Default is "default".
+  --[JOB].dotnetTraceStopTimeoutSec <seconds>                   Grace period (seconds) to wait for the dotnet-trace CLI to finalize a trace after SIGINT, only used by the
+                                                                "collect" and "collect-linux" modes. A value of 0 selects the per-mode default (60s for collect, 180s for
+                                                                collect-linux). A timeout logs [WRN] and keeps the partial trace but does not fail the job.
   --[JOB].additionalProcesses <process-name>                    Name of the processes for which the CPU usage should be recorded. Can be used multiple times to define multiple values.
   --[JOB].options.traceOutput <filename>                        The name of the trace file. Can be a file prefix (app will add *.DATE*.zip) , or a specific name and no DATE*
                                                                 will be added e.g., c:\traces\mytrace

src/Microsoft.Crank.Models/Job.cs

@@ -194,6 +194,23 @@ public Source Source
         // Default of true preserves the legacy implicit overload behavior.
         public bool DotNetTraceRequestRundown { get; set; } = true;
 
+        // Selects how dotnet-trace captures the trace:
+        //   "default"      - in-process EventPipe via DiagnosticsClient (today's behavior).
+        //   "collect"      - shells out to the dotnet-trace CLI's `collect` verb.
+        //   "collect-linux"- shells out to the dotnet-trace CLI's `collect-linux` verb,
+        //                    which uses perf_event_open to capture kernel + native frames
+        //                    machine-wide. Linux-only, requires root and kernel >= 6.4.
+        // Default of "default" preserves the legacy in-process behavior so that
+        // mixed-version controller/agent combinations stay byte-for-byte equivalent.
+        public string DotNetTraceCollectMode { get; set; } = "default";
+
+        // Grace period (seconds) to wait for `dotnet-trace` to finalize a trace after
+        // SIGINT before considering the trace possibly incomplete. Only used by the
+        // CLI collect modes (`collect`, `collect-linux`). A value of 0 selects the
+        // per-mode default (60s for collect, 180s for collect-linux). Timeout logs
+        // [WRN] and keeps the partial trace but does not fail the job.
+        public int DotNetTraceStopTimeoutSec { get; set; } = 0;
+
         // Dump
         public bool DumpProcess { get; set; }
         public DumpTypeOption DumpType { get; set; } = DumpTypeOption.Mini;