+
diff --git a/asv_bench/asv.conf.dask.json b/asv_bench/asv.conf.dask.json
index cc12302b149..30e44e14821 100644
--- a/asv_bench/asv.conf.dask.json
+++ b/asv_bench/asv.conf.dask.json
@@ -48,7 +48,7 @@
// timeout in seconds for installing any dependencies in environment
// defaults to 10 min
- //"install_timeout": 600,
+ "install_timeout": 6000,
// the base URL to show a commit for the project.
"show_commit_url": "https://github.com/modin-project/modin/commit/",
diff --git a/asv_bench/asv.conf.hdk.json b/asv_bench/asv.conf.hdk.json
index 7d8b947ced1..317becb9fc8 100644
--- a/asv_bench/asv.conf.hdk.json
+++ b/asv_bench/asv.conf.hdk.json
@@ -25,6 +25,10 @@
// variable.
"environment_type": "conda",
+ // timeout in seconds for installing any dependencies in environment
+ // defaults to 10 min
+ "install_timeout": 6000,
+
// the base URL to show a commit for the project.
"show_commit_url": "https://github.com/modin-project/modin/commit/",
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
index fc7a3d99525..234004dbbb2 100644
--- a/asv_bench/asv.conf.json
+++ b/asv_bench/asv.conf.json
@@ -48,7 +48,7 @@
// timeout in seconds for installing any dependencies in environment
// defaults to 10 min
- //"install_timeout": 600,
+ "install_timeout": 6000,
// the base URL to show a commit for the project.
"show_commit_url": "https://github.com/modin-project/modin/commit/",
diff --git a/asv_bench/asv.conf.unidist.json b/asv_bench/asv.conf.unidist.json
index df011617ea3..b8e04c5bb1e 100644
--- a/asv_bench/asv.conf.unidist.json
+++ b/asv_bench/asv.conf.unidist.json
@@ -48,7 +48,7 @@
// timeout in seconds for installing any dependencies in environment
// defaults to 10 min
- //"install_timeout": 600,
+ "install_timeout": 6000,
// the base URL to show a commit for the project.
"show_commit_url": "https://github.com/modin-project/modin/commit/",
diff --git a/asv_bench/benchmarks/scalability/scalability_benchmarks.py b/asv_bench/benchmarks/scalability/scalability_benchmarks.py
index 05a465b2ab8..f9850ff1999 100644
--- a/asv_bench/benchmarks/scalability/scalability_benchmarks.py
+++ b/asv_bench/benchmarks/scalability/scalability_benchmarks.py
@@ -17,7 +17,7 @@
from modin.pandas.utils import from_pandas
try:
- from modin.utils import to_pandas
+ from modin.utils import to_pandas, to_numpy
except ImportError:
# This provides compatibility with older versions of the Modin, allowing us to test old commits.
from modin.pandas.utils import to_pandas
@@ -70,4 +70,22 @@ def time_to_pandas(self, shape, cpus):
to_pandas(self.data)
+class TimeToNumPy:
+ param_names = ["shape", "cpus"]
+ params = [
+ get_benchmark_shapes("TimeToNumPy"),
+ [4, 16, 32],
+ ]
+
+ def setup(self, shape, cpus):
+ from modin.config import NPartitions
+
+ NPartitions.get = lambda: cpus
+ self.data = generate_dataframe("int", *shape, RAND_LOW, RAND_HIGH, impl="modin")
+
+ def time_to_numpy(self, shape, cpus):
+ # to_numpy is already synchronous
+ to_numpy(self.data)
+
+
from ..utils import setup # noqa: E402, F401
diff --git a/asv_bench/benchmarks/utils/common.py b/asv_bench/benchmarks/utils/common.py
index e67ca677de9..9a0a1dab276 100644
--- a/asv_bench/benchmarks/utils/common.py
+++ b/asv_bench/benchmarks/utils/common.py
@@ -459,17 +459,8 @@ def trigger_import(*dfs):
if ASV_USE_STORAGE_FORMAT != "hdk" or ASV_USE_IMPL == "pandas":
return
- from modin.experimental.core.execution.native.implementations.hdk_on_native.db_worker import (
- DbWorker,
- )
-
for df in dfs:
- df.shape # to trigger real execution
- df._query_compiler._modin_frame._partitions[0][
- 0
- ].frame_id = DbWorker().import_arrow_table(
- df._query_compiler._modin_frame._partitions[0][0].get()
- ) # to trigger real execution
+ df._query_compiler._modin_frame.force_import()
def execute(
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
index 98d58aa4291..af3ce71014f 100644
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -116,6 +116,7 @@
# Scalability benchmarks
"TimeFromPandas",
"TimeToPandas",
+ "TimeToNumPy",
],
),
(
diff --git a/codecov.yml b/codecov.yml
index 69cb76019a4..95adf7b6b26 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1 +1,11 @@
comment: false
+coverage:
+ status:
+ project:
+ default:
+ branches:
+ - master
+ target: 85%
+ patch:
+ default:
+ target: 30%
diff --git a/docs/conf.py b/docs/conf.py
index 1e6ac43a891..9cf86535675 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -13,6 +13,7 @@
import ray
+
# stub ray.remote to be a no-op so it doesn't shadow docstrings
def noop_decorator(*args, **kwargs):
if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
@@ -24,7 +25,7 @@ def noop_decorator(*args, **kwargs):
ray.remote = noop_decorator
# fake modules if they're missing
-for mod_name in ("cudf", "cupy", "pyarrow.gandiva", "pyhdk"):
+for mod_name in ("cudf", "cupy", "pyarrow.gandiva", "pyhdk", "pyhdk.hdk"):
try:
__import__(mod_name)
except ImportError:
@@ -37,6 +38,17 @@ def noop_decorator(*args, **kwargs):
sys.modules["cupy"].ndarray = type("ndarray", (object,), {})
if not hasattr(sys.modules["pyhdk"], "PyDbEngine"):
sys.modules["pyhdk"].PyDbEngine = type("PyDbEngine", (object,), {})
+if not hasattr(sys.modules["pyhdk.hdk"], "HDK"):
+ sys.modules["pyhdk.hdk"].HDK = type("HDK", (object,), {})
+if not hasattr(sys.modules["pyhdk.hdk"], "QueryNode"):
+ sys.modules["pyhdk.hdk"].QueryNode = type("QueryNode", (object,), {})
+if not hasattr(sys.modules["pyhdk.hdk"], "ExecutionResult"):
+ sys.modules["pyhdk.hdk"].ExecutionResult = type("ExecutionResult", (object,), {})
+if not hasattr(sys.modules["pyhdk.hdk"], "RelAlgExecutor"):
+ sys.modules["pyhdk.hdk"].RelAlgExecutor = type("RelAlgExecutor", (object,), {})
+if not hasattr(sys.modules["pyhdk"], "__version__"):
+ # Show all known pyhdk config options in documentation
+ sys.modules["pyhdk"].__version__ = "999"
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import modin
diff --git a/docs/development/using_hdk.rst b/docs/development/using_hdk.rst
index 63455178462..86a6b39ffc5 100644
--- a/docs/development/using_hdk.rst
+++ b/docs/development/using_hdk.rst
@@ -34,4 +34,22 @@ If for some reasons ``Native`` engine is explicitly set using ``modin.config`` o
If you encounter ``LLVM ERROR: inconsistency in registered CommandLine options`` error when using HDK,
please refer to the respective section in :doc:`Troubleshooting ` page to avoid the issue.
-.. _HDK: https://github.com/intel-ai/hdk
\ No newline at end of file
+
+Running on a GPU
+----------------
+
+Prerequisites:
+
+* HDK's GPU mode is currently supported on Linux and Intel GPU only.
+* HDK supports Gen9 architecture and higher (including Xe & Arc).
+* HDK's GPU mode requires proper driver installation. Follow this guide_ to set up your system. Make sure to install the compute runtime packages: ``intel-opencl-icd``, ``intel-level-zero-gpu``, ``level-zero``.
+* Make sure your GPU is visible and accessible.
+
+.. note::
+ You can use ``hwinfo`` and ``clinfo`` utilities to verify the driver installation and device accessibility.
+
+HDK supports a heterogeneous execution mode (experimental) that is disabled by default in Modin. Starting with pyHDK version 0.7 Modin can run the workload on Intel GPU.
+Run on a GPU via ``MODIN_HDK_LAUNCH_PARAMETERS="cpu_only=0" python