Feature/required framework packages

.github/workflows/conda-package.yml

@@ -196,11 +196,11 @@ jobs:
       # we want to make sure that configuration files are geting populated
       - name: Run npbench benchmark
         run: |
-          dpbench -i numpy -b azimint_hist run --npbench --precision=${{matrix.precision}}
+          dpbench -i numpy -b azimint_hist run --experimental-npbench --precision=${{matrix.precision}}
 
       - name: Run rodinia benchmark
         run: |
-          dpbench run --rodinia --no-dpbench --no-validate -r 1 --precision=${{matrix.precision}}
+          dpbench run --experimental-rodinia --no-dpbench --no-validate -r 1 --precision=${{matrix.precision}}
 
   upload_anaconda:
     name: Upload dppy/label/dev ['${{ matrix.os }}', python='${{ matrix.python }}']

README.md

@@ -117,9 +117,9 @@ SPDX-License-Identifier: Apache-2.0
 4. All available options are available using `dpbench --help` and `dpbench <command> --help`:
 
     ```
-    usage: dpbench [-h] [-b [BENCHMARKS]] [-i [IMPLEMENTATIONS]] [-a | --all-implementations | --no-all-implementations] [--version] [-r [RUN_ID]] [--last-run | --no-last-run]
-                [-d [RESULTS_DB]] [--log-level [{critical,fatal,error,warning,info,debug}]]
-                {run,report,config} ...
+    usage: dpbench [-h] [-b [BENCHMARKS]] [-i [IMPLEMENTATIONS]] [-a | --all-implementations | --no-all-implementations] [--version] [-r [RUN_ID]] [--last-run | --no-last-run] [-d [RESULTS_DB]]
+               [--log-level [{critical,fatal,error,warning,info,debug}]]
+               {run,report,config} ...
 
     positional arguments:
     {run,report,config}
@@ -131,50 +131,52 @@ SPDX-License-Identifier: Apache-2.0
     -i [IMPLEMENTATIONS], --implementations [IMPLEMENTATIONS]
                             Comma separated list of implementations. Use --all-implementations to load all available implementations.
     -a, --all-implementations, --no-all-implementations
-                            If set, all available implementations will be loaded. (default: False)
+                            If set, all available implementations will be loaded.
     --version             show program's version number and exit
     -r [RUN_ID], --run-id [RUN_ID]
                             run_id to perform actions on. Use --last-run to use latest available run, or leave empty to create new one.
     --last-run, --no-last-run
-                            Sets run_id to the latest run_id from the database. (default: False)
+                            Sets run_id to the latest run_id from the database.
     -d [RESULTS_DB], --results-db [RESULTS_DB]
                             Path to a database to store results.
     --log-level [{critical,fatal,error,warning,info,debug}]
                             Log level.
     ```
 
     ```
-    usage: dpbench run [-h] [-p [{S,M,L}]] [-s | --validate | --no-validate] [--dpbench | --no-dpbench] [--npbench | --no-npbench] [--polybench | --no-polybench] [-r [REPEAT]] [-t [TIMEOUT]]
-                    [--precision [{single,double}]] [--print-results | --no-print-results] [--save | --no-save] [--sycl-device [SYCL_DEVICE]]
-                    [--skip-expected-failures | --no-skip-expected-failures]
+    usage: dpbench run [-h] [-p [{S,M16Gb,M,L}]] [-s | --validate | --no-validate] [--dpbench | --no-dpbench] [--experimental-npbench | --no-experimental-npbench] [--experimental-polybench | --no-experimental-polybench]
+                   [--experimental-rodinia | --no-experimental-rodinia] [-r [REPEAT]] [-t [TIMEOUT]] [--precision [{single,double}]] [--print-results | --no-print-results] [--save | --no-save] [--sycl-device [SYCL_DEVICE]]
+                   [--skip-expected-failures | --no-skip-expected-failures]
 
     Subcommand to run benchmark executions.
 
     options:
     -h, --help            show this help message and exit
-    -p [{S,M,L}], --preset [{S,M,L}]
+    -p [{S,M16Gb,M,L}], --preset [{S,M16Gb,M,L}]
                             Preset to use for benchmark execution.
     -s, --validate, --no-validate
-                            Set if the validation will be run for each benchmark. (default: True)
+                            Set if the validation will be run for each benchmark.
     --dpbench, --no-dpbench
-                            Set if run dpbench benchmarks. (default: True)
-    --npbench, --no-npbench
-                            Set if run npbench benchmarks. (default: False)
-    --polybench, --no-polybench
-                            Set if run polybench benchmarks. (default: False)
+                            Set if run dpbench benchmarks.
+    --experimental-npbench, --no-experimental-npbench
+                            Set if run npbench benchmarks.
+    --experimental-polybench, --no-experimental-polybench
+                            Set if run polybench benchmarks.
+    --experimental-rodinia, --no-experimental-rodinia
+                            Set if run rodinia benchmarks.
     -r [REPEAT], --repeat [REPEAT]
                             Number of repeats for each benchmark.
     -t [TIMEOUT], --timeout [TIMEOUT]
                             Timeout time in seconds for each benchmark execution.
     --precision [{single,double}]
                             Data precision to use for array initialization.
     --print-results, --no-print-results
-                            Show the result summary or not (default: True)
-    --save, --no-save     Either to save execution into database. (default: True)
+                            Show the result summary or not
+    --save, --no-save     Either to save execution into database.
     --sycl-device [SYCL_DEVICE]
                             Sycl device to overwrite for framework configurations.
     --skip-expected-failures, --no-skip-expected-failures
-                            Either to save execution into database. (default: True)
+                            Either to save execution into database.
     ```
 
     ```

dpbench/config/reader.py

@@ -13,6 +13,8 @@
 
 import tomli
 
+from dpbench.infrastructure.frameworks.fabric import get_framework_class
+
 from .benchmark import Benchmark, BenchmarkImplementation, Presets
 from .config import Config
 from .framework import Framework
@@ -117,7 +119,7 @@ def read_configs(  # noqa: C901: TODO: move modules into config
     for framework in config.frameworks:
         config.implementations += framework.postfixes
 
-    if implementations is None:
+    if implementations is None or len(implementations) == 0:
         implementations = {impl.postfix for impl in config.implementations}
 
     if load_implementations:
@@ -228,6 +230,15 @@ def read_frameworks(
         if len(framework.postfixes) == 0:
             continue
 
+        cls = get_framework_class(framework)
+        unavailable_pkgs = cls.get_missing_required_packages()
+        if len(unavailable_pkgs) > 0:
+            logging.warning(
+                f"Framework {framework.simple_name} unavailable "
+                + f"due to missing packages {unavailable_pkgs}"
+            )
+            continue
+
         config.frameworks.append(framework)
 
 

dpbench/console/run.py

@@ -47,19 +47,22 @@ def add_run_arguments(parser: argparse.ArgumentParser):
         help="Set if run dpbench benchmarks.",
     )
     parser.add_argument(
-        "--npbench",
+        "--experimental-npbench",
+        dest="npbench",
         action=argparse.BooleanOptionalAction,
         default=False,
         help="Set if run npbench benchmarks.",
     )
     parser.add_argument(
-        "--polybench",
+        "--experimental-polybench",
+        dest="polybench",
         action=argparse.BooleanOptionalAction,
         default=False,
         help="Set if run polybench benchmarks.",
     )
     parser.add_argument(
-        "--rodinia",
+        "--experimental-rodinia",
+        dest="rodinia",
         action=argparse.BooleanOptionalAction,
         default=False,
         help="Set if run rodinia benchmarks.",

dpbench/infrastructure/frameworks/cupy_framework.py

@@ -19,6 +19,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
         super().__init__(fname, config)
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["cupy"]
+
     def copy_to_func(self) -> Callable:
         """Returns the copy-method that should be used
         for copying the benchmark arguments."""

dpbench/infrastructure/frameworks/dpnp_framework.py

@@ -35,6 +35,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
             )
             raise sdce
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["dpnp"]
+
     def device_filter_string(self) -> str:
         """Returns the sycl device's filter string if the framework has an
         associated sycl device."""

dpbench/infrastructure/frameworks/fabric.py

@@ -39,7 +39,7 @@ def build_framework_map() -> dict[str, Framework]:
     return result
 
 
-def build_framework(framework_config: cfg.Framework) -> Framework:
+def get_framework_class(framework_config: cfg.Framework) -> Framework:
     available_classes = [
         Framework,
         DpcppFramework,
@@ -61,4 +61,9 @@ def build_framework(framework_config: cfg.Framework) -> Framework:
         )
         constructor = Framework
 
+    return constructor
+
+
+def build_framework(framework_config: cfg.Framework) -> Framework:
+    constructor = get_framework_class(framework_config)
     return constructor(config=framework_config)

dpbench/infrastructure/frameworks/framework.py

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import logging
+from importlib.util import find_spec
 from typing import Any, Callable, Dict, final
 
 import pkg_resources
@@ -45,6 +46,20 @@ def __init__(
 
         self.device_info = cpuinfo.get_cpu_info().get("brand_raw")
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return []
+
+    @classmethod
+    def get_missing_required_packages(cls) -> None:
+        unavailable_packages = []
+        for pkg in cls.required_packages():
+            spec = find_spec(pkg)
+            if spec is None:
+                unavailable_packages.append(pkg)
+
+        return unavailable_packages
+
     def device_filter_string(self) -> str:
         """Returns the sycl device's filter string if the framework has an
         associated sycl device."""

dpbench/infrastructure/frameworks/numba_cuda_framework.py

@@ -19,6 +19,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
         super().__init__(fname, config)
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["cupy"]
+
     def copy_to_func(self) -> Callable:
         """Returns the copy-method that should be used
         for copying the benchmark arguments."""

dpbench/infrastructure/frameworks/numba_dpex_framework.py

@@ -33,6 +33,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
             )
             raise sdce
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["numba_dpex"]
+
     def device_filter_string(self) -> str:
         """Returns the sycl device's filter string if the framework has an
         associated sycl device."""

dpbench/infrastructure/frameworks/numba_framework.py

@@ -27,3 +27,7 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
         """
 
         super().__init__(fname, config)
+
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["numba"]

dpbench/infrastructure/frameworks/numba_mlir_framework.py

@@ -28,6 +28,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
             self.device_info = dpctl.SyclDevice(self.sycl_device).name
 
+    @staticmethod
+    def required_packages() -> list[str]:
+        return ["numba_mlir"]
+
     def copy_to_func(self) -> Callable:
         """Returns the copy-method that should be used
         for copying the benchmark arguments to device."""