Refactor py-spy & pyperf to separate ProfilerInterface.

gprofiler/main.py

@@ -51,7 +51,12 @@
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.factory import get_profilers
 from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface
-from gprofiler.profilers.registry import get_profilers_registry
+from gprofiler.profilers.registry import (
+    ProfilerArgument,
+    get_runtime_possible_modes,
+    get_runtimes_registry,
+    get_sorted_profilers,
+)
 from gprofiler.spark.sampler import SparkSampler
 from gprofiler.state import State, init_state
 from gprofiler.system_metrics import Metrics, NoopSystemMetricsMonitor, SystemMetricsMonitor, SystemMetricsMonitorBase
@@ -824,7 +829,7 @@ def parse_cmd_args() -> configargparse.Namespace:
     if args.extract_resources and args.resources_dest is None:
         parser.error("Must provide --resources-dest when extract-resources")
 
-    if args.perf_dwarf_stack_size > 65528:
+    if args.perf_mode not in ("disabled", "none") and args.perf_dwarf_stack_size > 65528:
         parser.error("--perf-dwarf-stack-size maximum size is 65528")
 
     if args.profiling_mode == CPU_PROFILING_MODE and args.perf_mode in ("dwarf", "smart") and args.frequency > 100:
@@ -840,29 +845,52 @@ def parse_cmd_args() -> configargparse.Namespace:
 
 
 def _add_profilers_arguments(parser: configargparse.ArgumentParser) -> None:
-    registry = get_profilers_registry()
-    for name, config in registry.items():
-        arg_group = parser.add_argument_group(name)
-        mode_var = f"{name.lower()}_mode"
+    # add command-line arguments for each profiling runtime, but only for profilers that are working
+    # with current architecture.
+    runtimes_registry = get_runtimes_registry()
+    for runtime_class, runtime_config in runtimes_registry.items():
+        runtime = runtime_config.runtime_name
+        runtime_possible_modes = get_runtime_possible_modes(runtime_class)
+        arg_group = parser.add_argument_group(runtime)
+        mode_var = f"{runtime.lower()}_mode"
+        if not runtime_possible_modes:
+            # if no mode is possible for this runtime, skip this runtime, and register it as disabled
+            # to overcome issue with Perf showing up on Windows
+            parser.add_argument(
+                f"--{runtime.lower()}-mode",
+                dest=mode_var,
+                default="disabled",
+                help=configargparse.SUPPRESS,
+            )
+            continue
+
         arg_group.add_argument(
-            f"--{name.lower()}-mode",
+            f"--{runtime.lower()}-mode",
             dest=mode_var,
-            default=config.default_mode,
-            help=config.profiler_mode_help,
-            choices=config.possible_modes,
+            default=runtime_config.default_mode,
+            help=runtime_config.mode_help,
+            choices=runtime_possible_modes,
         )
         arg_group.add_argument(
-            f"--no-{name.lower()}",
+            f"--no-{runtime.lower()}",
             action="store_const",
             const="disabled",
             dest=mode_var,
             default=True,
-            help=config.disablement_help,
+            help=runtime_config.disablement_help,
         )
-        for arg in config.profiler_args:
+        # add each available profiler's arguments and runtime common arguments
+        profiling_args: List[ProfilerArgument] = []
+        profiling_args.extend(runtime_config.common_arguments)
+        for config in get_sorted_profilers(runtime_class):
+            profiling_args.extend(config.profiler_args)
+
+        for arg in profiling_args:
             profiler_arg_kwargs = arg.get_dict()
-            name = profiler_arg_kwargs.pop("name")
-            arg_group.add_argument(name, **profiler_arg_kwargs)
+            # do not add parser entries for profiler internal arguments
+            if "internal" not in profiler_arg_kwargs:
+                name = profiler_arg_kwargs.pop("name")
+                arg_group.add_argument(name, **profiler_arg_kwargs)
 
 
 def verify_preconditions(args: configargparse.Namespace, processes_to_profile: Optional[List[Process]]) -> None:

gprofiler/profilers/__init__.py

@@ -1,17 +1,18 @@
 # NOTE: Make sure to import any new process profilers to load it
 from gprofiler.platform import is_linux
 from gprofiler.profilers.dotnet import DotnetProfiler
-from gprofiler.profilers.python import PythonProfiler
+from gprofiler.profilers.python import PySpyProfiler
 
 if is_linux():
     from gprofiler.profilers.java import JavaProfiler
     from gprofiler.profilers.perf import SystemProfiler
     from gprofiler.profilers.php import PHPSpyProfiler
+    from gprofiler.profilers.python_ebpf import PythonEbpfProfiler
     from gprofiler.profilers.ruby import RbSpyProfiler
 
-__all__ = ["PythonProfiler", "DotnetProfiler"]
+__all__ = ["PySpyProfiler", "DotnetProfiler"]
 
 if is_linux():
-    __all__ += ["JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler"]
+    __all__ += ["JavaProfiler", "PHPSpyProfiler", "RbSpyProfiler", "SystemProfiler", "PythonEbpfProfiler"]
 
 del is_linux

gprofiler/profilers/dotnet.py

@@ -19,7 +19,7 @@
 from gprofiler.platform import is_windows
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.profiler_base import ProcessProfilerBase
-from gprofiler.profilers.registry import register_profiler
+from gprofiler.profilers.registry import ProfilingRuntime, register_profiler, register_runtime
 from gprofiler.utils import pgrep_exe, pgrep_maps, random_prefix, removed_path, resource_path, run_process
 from gprofiler.utils.process import process_comm
 from gprofiler.utils.speedscope import load_speedscope_as_collapsed
@@ -49,12 +49,17 @@ def make_application_metadata(self, process: Process) -> Dict[str, Any]:
         return metadata
 
 
+@register_runtime("dotnet", default_mode="disabled")
+class DotnetRuntime(ProfilingRuntime):
+    pass
+
+
 @register_profiler(
     "dotnet",
+    runtime_class=DotnetRuntime,
     possible_modes=["dotnet-trace", "disabled"],
     supported_archs=["x86_64", "aarch64"],
     supported_windows_archs=["AMD64"],
-    default_mode="disabled",
     supported_profiling_modes=["cpu"],
 )
 class DotnetProfiler(ProcessProfilerBase):

gprofiler/profilers/factory.py

@@ -2,11 +2,9 @@
 from typing import TYPE_CHECKING, Any, List, Tuple, Union
 
 from gprofiler.log import get_logger_adapter
-from gprofiler.metadata.system_metadata import get_arch
-from gprofiler.platform import is_windows
 from gprofiler.profilers.perf import SystemProfiler
 from gprofiler.profilers.profiler_base import NoopProfiler
-from gprofiler.profilers.registry import get_profilers_registry
+from gprofiler.profilers.registry import ProfilerConfig, get_runtimes_registry, get_sorted_profilers
 
 if TYPE_CHECKING:
     from gprofiler.gprofiler_types import UserArgs
@@ -24,44 +22,62 @@ def get_profilers(
     process_profilers_instances: List["ProcessProfilerBase"] = []
     system_profiler: Union["SystemProfiler", "NoopProfiler"] = NoopProfiler()
 
-    if profiling_mode != "none":
-        arch = get_arch()
-        for profiler_name, profiler_config in get_profilers_registry().items():
-            lower_profiler_name = profiler_name.lower()
-            profiler_mode = user_args.get(f"{lower_profiler_name}_mode")
-            if profiler_mode in ("none", "disabled"):
-                continue
-
-            supported_archs = (
-                profiler_config.supported_windows_archs if is_windows() else profiler_config.supported_archs
-            )
-            if arch not in supported_archs:
-                logger.warning(f"Disabling {profiler_name} because it doesn't support this architecture ({arch})")
-                continue
+    if profiling_mode == "none":
+        return system_profiler, process_profilers_instances
 
-            if profiling_mode not in profiler_config.supported_profiling_modes:
+    for runtime_class, runtime_config in get_runtimes_registry().items():
+        runtime = runtime_config.runtime_name
+        runtime_args_prefix = runtime.lower()
+        runtime_mode = user_args.get(f"{runtime_args_prefix}_mode")
+        if runtime_mode in ProfilerConfig.DISABLED_MODES:
+            continue
+        # select configs supporting requested runtime_mode or all configs in order of preference
+        requested_configs: List[ProfilerConfig] = get_sorted_profilers(runtime_class)
+        if runtime_mode != ProfilerConfig.ENABLED_MODE:
+            requested_configs = [c for c in requested_configs if runtime_mode in c.get_active_modes()]
+        # select profilers that support this architecture and profiling mode
+        selected_configs: List[ProfilerConfig] = []
+        for config in requested_configs:
+            profiler_name = config.profiler_name
+            if profiling_mode not in config.supported_profiling_modes:
                 logger.warning(
                     f"Disabling {profiler_name} because it doesn't support profiling mode {profiling_mode!r}"
                 )
                 continue
+            selected_configs.append(config)
 
+        if not selected_configs:
+            logger.warning(f"Disabling {runtime} profiling because no profilers were selected")
+            continue
+        # create instances of selected profilers one by one, select first that is ready
+        ready_profiler = None
+        mode_var = f"{runtime.lower()}_mode"
+        runtime_arg_names: List[str] = [arg.dest for arg in runtime_config.common_arguments] + [mode_var]
+        for profiler_config in selected_configs:
+            profiler_name = profiler_config.profiler_name
             profiler_kwargs = profiler_init_kwargs.copy()
+            profiler_arg_names = [arg.dest for arg in profiler_config.profiler_args]
             for key, value in user_args.items():
-                if key.startswith(lower_profiler_name) or key in COMMON_PROFILER_ARGUMENT_NAMES:
+                if key in profiler_arg_names or key in runtime_arg_names or key in COMMON_PROFILER_ARGUMENT_NAMES:
                     profiler_kwargs[key] = value
             try:
                 profiler_instance = profiler_config.profiler_class(**profiler_kwargs)
+                if profiler_instance.check_readiness():
+                    ready_profiler = profiler_instance
+                    break
             except Exception:
-                logger.critical(
-                    f"Couldn't create the {profiler_name} profiler, not continuing."
-                    f" Run with --no-{profiler_name.lower()} to disable this profiler",
-                    exc_info=True,
-                )
-                sys.exit(1)
-            else:
-                if isinstance(profiler_instance, SystemProfiler):
-                    system_profiler = profiler_instance
-                else:
-                    process_profilers_instances.append(profiler_instance)
-
+                if len(requested_configs) == 1:
+                    logger.critical(
+                        f"Couldn't create the {profiler_name} profiler for runtime {runtime}, not continuing."
+                        f" Request different profiler for runtime with --{runtime_args_prefix}-mode, or disable"
+                        f" {runtime} profiling with --{runtime_args_prefix}-mode=disabled to disable this profiler",
+                        exc_info=True,
+                    )
+                    sys.exit(1)
+        if isinstance(ready_profiler, SystemProfiler):
+            system_profiler = ready_profiler
+        elif ready_profiler is not None:
+            process_profilers_instances.append(ready_profiler)
+        else:
+            logger.warning(f"Disabling {runtime} profiling because no profilers were ready")
     return system_profiler, process_profilers_instances

gprofiler/profilers/java.py

@@ -67,7 +67,7 @@
 from gprofiler.metadata.application_metadata import ApplicationMetadata
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.profiler_base import SpawningProcessProfilerBase
-from gprofiler.profilers.registry import ProfilerArgument, register_profiler
+from gprofiler.profilers.registry import ProfilerArgument, ProfilingRuntime, register_profiler, register_runtime
 from gprofiler.utils import (
     GPROFILER_DIRECTORY_NAME,
     TEMPORARY_STORAGE_PATH,
@@ -785,10 +785,18 @@ def read_output(self) -> Optional[str]:
             raise
 
 
+@register_runtime(
+    "Java",
+    default_mode="ap",
+)
+class JavaRuntime(ProfilingRuntime):
+    pass
+
+
 @register_profiler(
     "Java",
+    runtime_class=JavaRuntime,
     possible_modes=["ap", "disabled"],
-    default_mode="ap",
     supported_archs=["x86_64", "aarch64"],
     profiler_arguments=[
         ProfilerArgument(

gprofiler/profilers/perf.py

@@ -35,7 +35,13 @@
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.node import clean_up_node_maps, generate_map_for_node_processes, get_node_processes
 from gprofiler.profilers.profiler_base import ProfilerBase
-from gprofiler.profilers.registry import ProfilerArgument, register_profiler
+from gprofiler.profilers.registry import (
+    InternalArgument,
+    ProfilerArgument,
+    ProfilingRuntime,
+    register_profiler,
+    register_runtime,
+)
 from gprofiler.utils import (
     reap_process,
     remove_files_by_prefix,
@@ -374,15 +380,33 @@ def wait_and_script(self) -> str:
                 remove_path(inject_data, missing_ok=True)
 
 
-@register_profiler(
+@register_runtime(
     "Perf",
-    possible_modes=["fp", "dwarf", "smart", "disabled"],
     default_mode="fp",
-    supported_archs=["x86_64", "aarch64"],
-    profiler_mode_argument_help="Run perf with either FP (Frame Pointers), DWARF, or run both and intelligently merge"
+    mode_help="Run perf with either FP (Frame Pointers), DWARF, or run both and intelligently merge"
     " them by choosing the best result per process. If 'disabled' is chosen, do not invoke"
     " 'perf' at all. The output, in that case, is the concatenation of the results from all"
-    " of the runtime profilers. Defaults to 'smart'.",
+    " of the runtime profilers. Defaults to 'fp'.",
+    common_arguments=[
+        ProfilerArgument(
+            "--perf-no-memory-restart",
+            help="Disable checking if perf used memory exceeds threshold and restarting perf",
+            action="store_false",
+            dest="perf_memory_restart",
+        ),
+    ],
+    disablement_help="Disable the global perf of processes,"
+    " and instead only concatenate runtime-specific profilers results",
+)
+class PerfRuntime(ProfilingRuntime):
+    pass
+
+
+@register_profiler(
+    "Perf",
+    runtime_class=PerfRuntime,
+    possible_modes=["fp", "dwarf", "smart", "disabled"],
+    supported_archs=["x86_64", "aarch64"],
     profiler_arguments=[
         ProfilerArgument(
             "--perf-dwarf-stack-size",
@@ -392,15 +416,9 @@ def wait_and_script(self) -> str:
             default=DEFAULT_PERF_DWARF_STACK_SIZE,
             dest="perf_dwarf_stack_size",
         ),
-        ProfilerArgument(
-            "--perf-no-memory-restart",
-            help="Disable checking if perf used memory exceeds threshold and restarting perf",
-            action="store_false",
-            dest="perf_memory_restart",
-        ),
+        InternalArgument(dest="perf_inject"),
+        InternalArgument(dest="perf_node_attach"),
     ],
-    disablement_help="Disable the global perf of processes,"
-    " and instead only concatenate runtime-specific profilers results",
     supported_profiling_modes=["cpu"],
 )
 class SystemProfiler(ProfilerBase):

gprofiler/profilers/php.py

@@ -18,19 +18,24 @@
 from gprofiler.log import get_logger_adapter
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.profiler_base import ProfilerBase
-from gprofiler.profilers.registry import ProfilerArgument, register_profiler
+from gprofiler.profilers.registry import ProfilerArgument, ProfilingRuntime, register_profiler, register_runtime
 from gprofiler.utils import random_prefix, reap_process, resource_path, start_process, wait_event
 
 logger = get_logger_adapter(__name__)
 # Currently tracing only php-fpm, TODO: support mod_php in apache.
 DEFAULT_PROCESS_FILTER = "php-fpm"
 
 
+@register_runtime("PHP", default_mode="disabled")
+class PHPRuntime(ProfilingRuntime):
+    pass
+
+
 @register_profiler(
     "PHP",
+    runtime_class=PHPRuntime,
     possible_modes=["phpspy", "disabled"],
     supported_archs=["x86_64", "aarch64"],
-    default_mode="disabled",
     profiler_arguments=[
         ProfilerArgument(
             "--php-proc-filter",