Merge branch 'master' of https://github.com/ray-project/ray into eoakes/gcs-client

Author: edoakes

.pre-commit-config.yaml

@@ -45,10 +45,38 @@ repos:
       - id: ruff
         args: [ --select, "I", --fix, --exit-non-zero-on-fix ]
 
+  # pydoclint-local is for local commits only due to pre-commit-hook only passing
+  # updated files to the hook and overwriting the baseline text file
   - repo: https://github.com/jsh9/pydoclint
     rev: "0.6.6"
     hooks:
       - id: pydoclint
+        name: pydoclint-local
+        stages: [pre-commit, pre-push]
+        args: [
+          --style=google,
+          --baseline=ci/lint/pydoclint-baseline.txt,
+          --exclude=thirdparty|^python/ray/serve/tests/test_config_files/syntax_error\.py$|^python/ray/_private/parameter\.py$,
+          --auto-regenerate-baseline=False,
+          # Current settings (not because we think they're right, but because we
+          # don't want a baseline the size of the codebase)
+          --arg-type-hints-in-docstring=False,
+          --skip-checking-raises=True,
+          --check-return-types=False,
+          --allow-init-docstring=True,
+          --check-class-attributes=False,
+          # --check-style-mismatch=True, # Bring this back once things are a bit cleaner
+        ]
+        types: [python]
+        files: '^python/ray/'
+
+  # pydoclint-ci is for CI, overwrites the baseline text file, and is run with the manual stage flag
+  - repo: https://github.com/jsh9/pydoclint
+    rev: "0.6.6"
+    hooks:
+      - id: pydoclint
+        name: pydoclint-ci
+        stages: [manual]
         args: [
           --style=google,
           --baseline=ci/lint/pydoclint-baseline.txt,

WORKSPACE

@@ -125,8 +125,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 """,
-    sha256 = "2c4392591fe9469d006452ef22f32712f35087d87fb1764ec03e23544eb8770d",
-    urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-x86_64-unknown-linux-gnu.tar.gz"],
+    sha256 = "920cbcaad514cc185634f6f0dcd71df5e8f4ee4456d440a22e0f8c0f142a8203",
+    urls = ["https://github.com/astral-sh/uv/releases/download/0.8.17/uv-x86_64-unknown-linux-gnu.tar.gz"],
 )
 
 http_archive(
@@ -138,8 +138,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 """,
-    sha256 = "5200278ae00b5c0822a7db7a99376b2167e8e9391b29c3de22f9e4fdebc9c0e8",
-    urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-aarch64-apple-darwin.tar.gz"],
+    sha256 = "e4d4859d7726298daa4c12e114f269ff282b2cfc2b415dc0b2ca44ae2dbd358e",
+    urls = ["https://github.com/astral-sh/uv/releases/download/0.8.17/uv-aarch64-apple-darwin.tar.gz"],
 )
 
 http_archive(

ci/lint/lint.sh

@@ -48,7 +48,11 @@ pre_commit() {
 pre_commit_pydoclint() {
   # Run pre-commit pydoclint on all files
   pip install -c python/requirements_compiled.txt pre-commit clang-format
-  pre-commit run pydoclint --all-files --show-diff-on-failure
+  pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure
+  git diff --quiet -- ci/lint/pydoclint-baseline.txt || {
+  echo "Baseline needs update. Run the CI-style hook: \"pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure\" locally and commit the baseline."
+  exit 1
+  }
 }
 
 code_format() {

ci/raydepsets/pre_hooks/remove-compiled-headers.sh

@@ -2,13 +2,6 @@
 
 set -euo pipefail
 
-PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")"
-if [[ "${PYTHON_CODE}" != "py311" ]]; then
-	echo "--- Python version is not 3.11"
-	echo "--- Current Python version: ${PYTHON_CODE}"
-	exit 1
-fi
-
 mkdir -p /tmp/ray-deps
 
 # Remove the GPU constraints

ci/raydepsets/tests/test_cli.py

@@ -111,7 +111,7 @@ def test_uv_version(self):
             stderr=subprocess.PIPE,
         )
         assert result.returncode == 0
-        assert "uv 0.8.10" in result.stdout.decode("utf-8")
+        assert "uv 0.8.17" in result.stdout.decode("utf-8")
         assert result.stderr.decode("utf-8") == ""
 
     def test_compile(self):

doc/BUILD.bazel

@@ -202,6 +202,41 @@ py_test(
     ],
 )
 
+py_test(
+    name = "doc_code_direct_transport_gloo",
+    size = "small",
+    srcs = ["source/ray-core/doc_code/direct_transport_gloo.py"],
+    main = "source/ray-core/doc_code/direct_transport_gloo.py",
+    tags = [
+        "exclusive",
+        "team:core",
+    ],
+)
+
+py_test(
+    name = "doc_code_direct_transport_nccl",
+    size = "small",
+    srcs = ["source/ray-core/doc_code/direct_transport_nccl.py"],
+    main = "source/ray-core/doc_code/direct_transport_nccl.py",
+    tags = [
+        "exclusive",
+        "multi_gpu",
+        "team:core",
+    ],
+)
+
+py_test(
+    name = "doc_code_direct_transport_nixl",
+    size = "small",
+    srcs = ["source/ray-core/doc_code/direct_transport_nixl.py"],
+    main = "source/ray-core/doc_code/direct_transport_nixl.py",
+    tags = [
+        "exclusive",
+        "multi_gpu",
+        "team:core",
+    ],
+)
+
 py_test_run_all_subdirectory(
     size = "medium",
     include = ["source/ray-core/doc_code/*.py"],
@@ -214,6 +249,8 @@ py_test_run_all_subdirectory(
         "source/ray-core/doc_code/cgraph_overlap.py",
         # not testing this as it purposefully segfaults
         "source/ray-core/doc_code/cgraph_troubleshooting.py",
+        "source/ray-core/doc_code/direct_transport_nccl.py",
+        "source/ray-core/doc_code/direct_transport_nixl.py",
     ],
     extra_srcs = [],
     tags = [

doc/source/ray-core/api/direct-transport.rst

@@ -0,0 +1,28 @@
+Ray Direct Transport (RDT) API
+==============================
+
+Usage with Core APIs
+--------------------
+Enable RDT for actor tasks with the :func:`@ray.method <ray.method>` decorator, or pass `_tensor_transport` to :func:`ray.put`. You can then pass the resulting `ray.ObjectRef` to other actor tasks, or use :func:`ray.get` to retrieve the result. See :ref:`Ray Direct Transport (RDT) <direct-transport>` for more details on usage.
+
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ray.method
+    ray.put
+    ray.get
+
+Collective tensor transports
+----------------------------
+Collective tensor transports require a collective group to be created before RDT objects can be used. Use these methods to create and manage collective groups for the `gloo` and `nccl` tensor transports.
+
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ray.experimental.collective.create_collective_group
+    ray.experimental.collective.get_collective_groups
+    ray.experimental.collective.destroy_collective_group

doc/source/ray-core/api/index.rst

@@ -12,3 +12,4 @@ Ray Core API
     cli.rst
     ../../ray-observability/reference/cli.rst
     ../../ray-observability/reference/api.rst
+    direct-transport.rst