[CI] Run doc notebooks in CI

.buildkite/pipeline.gpu.large.yml

@@ -39,3 +39,16 @@
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
     - pip install -Ur ./python/requirements_ml_docker.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/ml/...
+
+
+- label: ":tv: :book: Doc GPU tests and examples"
+  conditions:
+    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
+    - pip install -Ur ./python/requirements_ml_docker.txt
+    - ./ci/env/env_info.sh
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-tf,-pytorch,-py37,-post_wheel_build doc/...
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,tf,-pytorch,-py37,-post_wheel_build doc/...
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-tf,pytorch,-py37,-post_wheel_build doc/...

.buildkite/pipeline.ml.yml

@@ -377,6 +377,6 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37,-post_wheel_build doc/...
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-post_wheel_build doc/...
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-post_wheel_build doc/...
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-tf,-pytorch,-py37,-post_wheel_build doc/...
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,tf,-pytorch,-py37,-post_wheel_build doc/...
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-tf,pytorch,-py37,-post_wheel_build doc/...

bazel/python.bzl

@@ -1,10 +1,13 @@
+load("@bazel_skylib//lib:paths.bzl", "paths")
+
 # py_test_module_list creates a py_test target for each
 # Python file in `files`
 def py_test_module_list(files, size, deps, extra_srcs, name_suffix="", **kwargs):
     for file in files:
         # remove .py
-        name = file[:-3] + name_suffix
-        main = file
+        name = paths.split_extension(file)[0] + name_suffix
+        if name == file:
+            basename = basename + "_test"
         native.py_test(
             name = name,
             size = size,
@@ -14,24 +17,32 @@ def py_test_module_list(files, size, deps, extra_srcs, name_suffix="", **kwargs)
         )
 
 def py_test_run_all_subdirectory(include, exclude, extra_srcs, **kwargs):
-    for file in native.glob(include = include, exclude = exclude):
+    for file in native.glob(include = include, exclude = exclude, allow_empty=False):
         print(file)
-        basename = file.rpartition("/")[-1]
+        basename = paths.split_extension(file)[0]
+        if basename == file:
+            basename = basename + "_test"
         native.py_test(
-            name = basename[:-3],
+            name = basename,
             srcs = extra_srcs + [file],
             **kwargs
         )
 
 # Runs all included notebooks as py_test targets, by first converting them to .py files with "test_myst_doc.py".
 def py_test_run_all_notebooks(include, exclude, **kwargs):
-    for file in native.glob(include = include, exclude = exclude):
+    for file in native.glob(include = include, exclude = exclude, allow_empty=False):
         print(file)
-        basename = file.rpartition("/")[-1]
+        basename = paths.split_extension(file)[0]
+        if basename == file:
+            basename = basename + "_test"
         native.py_test(
-            name = basename[:-3],
+            name = basename,
             main = "test_myst_doc.py",
-            srcs = ["test_myst_doc.py"],
-            args = ["--path", file],
+            srcs = ["//doc:test_myst_doc.py"],
+            # --find-recursively will look for file in all
+            # directories inside cwd recursively if it cannot
+            # find it right away. This allows to deal with
+            # mismatches between `name` and `data` args.
+            args = ["--find-recursively", "--path", file],
             **kwargs
         )

ci/pipeline/determine_tests_to_run.py

@@ -192,7 +192,11 @@ def get_commit_range():
             elif changed_file.startswith("docker/"):
                 RAY_CI_DOCKER_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
-            elif changed_file.startswith("doc/") and changed_file.endswith(".py"):
+            elif changed_file.startswith("doc/") and (
+                changed_file.endswith(".py")
+                or changed_file.endswith(".ipynb")
+                or changed_file.endswith("BUILD")
+            ):
                 RAY_CI_DOC_AFFECTED = 1
             elif any(changed_file.startswith(prefix) for prefix in skip_prefix_list):
                 # nothing is run but linting in these cases

doc/BUILD

@@ -1,6 +1,8 @@
 load("//bazel:python.bzl", "py_test_run_all_subdirectory")
 load("//bazel:python.bzl", "py_test_run_all_notebooks")
 
+exports_files(["test_myst_doc.py"])
+
 # --------------------------------------------------------------------
 # Tests from the doc directory.
 # Please keep these sorted alphabetically, but start with the
@@ -138,17 +140,7 @@ py_test_run_all_subdirectory(
     tags = ["exclusive", "team:serve"],
 )
 
-# --------------------------------------------------------------------
-# Test all doc/source/tune/examples notebooks.
-# --------------------------------------------------------------------
 
-py_test_run_all_notebooks(
-    size = "medium",
-    include = ["doc/source/tune/examples/*.ipynb"],
-    exclude = [],
-    data = ["//doc/source/tune/examples:tune_examples"],
-    tags = ["exclusive", "team:ml"],
-)
 
 # --------------------------------------------------------------------
 # Test all doc/source/tune/doc_code code included in rst/md files.
@@ -162,18 +154,6 @@ py_test_run_all_subdirectory(
     tags = ["exclusive", "team:ml"],
 )
 
-# --------------------------------------------------------------------
-# Test all doc/source/ray-air/examples notebooks.
-# --------------------------------------------------------------------
-
-py_test_run_all_notebooks(
-    size = "medium",
-    include = ["doc/source/ray-air/examples/*.ipynb"],
-    exclude = [],
-    data = ["//doc/source/ray-air/examples:air_examples"],
-    tags = ["exclusive", "team:ml"],
-)
-
 # --------------------------------------------------------------------
 # Test all doc/source/ray-air/doc_code code included in rst/md files.
 # --------------------------------------------------------------------
@@ -188,13 +168,13 @@ py_test_run_all_subdirectory(
 
 
 # --------------------------------------------------------------------
-# Test all doc/source/ray-overview/doc_code snippets, used on ray.io
+# Test all doc/source/ray-overview/doc_test snippets, used on ray.io
 # --------------------------------------------------------------------
 
 py_test_run_all_subdirectory(
     size = "small",
-    include = ["source/ray-overview/doc_code/*.py"],
-    exclude = ["source/ray-overview/doc_code/ray_data.py"],
+    include = ["source/ray-overview/doc_test/*.py"],
+    exclude = ["source/ray-overview/doc_test/ray_data.py"],
     extra_srcs = [],
     tags = ["exclusive", "team:ml"],
 )

doc/source/ray-air/examples/BUILD

@@ -1,5 +1,19 @@
+load("//bazel:python.bzl", "py_test_run_all_notebooks")
+
 filegroup(
     name = "air_examples",
     srcs = glob(["*.ipynb"]),
     visibility = ["//doc:__subpackages__"]
-)
+)
+
+# --------------------------------------------------------------------
+# Test all doc/source/ray-air/examples notebooks.
+# --------------------------------------------------------------------
+
+py_test_run_all_notebooks(
+    size = "large",
+    include = ["*.ipynb"],
+    exclude = [],
+    data = ["//doc/source/ray-air/examples:air_examples"],
+    tags = ["exclusive", "team:ml"],
+)

doc/source/tune/examples/BUILD

@@ -1,5 +1,31 @@
+load("//bazel:python.bzl", "py_test_run_all_notebooks")
+
 filegroup(
     name = "tune_examples",
     srcs = glob(["*.ipynb"]),
     visibility = ["//doc:__subpackages__"]
+)
+
+# --------------------------------------------------------------------
+# Test all doc/source/tune/examples notebooks.
+# --------------------------------------------------------------------
+
+# pbt_ppo_example.ipynb is not tested right now due to large resource
+# requirements
+
+py_test_run_all_notebooks(
+    size = "medium",
+    include = ["*.ipynb"],
+    exclude = ["pbt_ppo_example.ipynb", "tune-xgboost.ipynb"],
+    data = ["//doc/source/tune/examples:tune_examples"],
+    tags = ["exclusive", "team:ml"],
+)
+
+# GPU tests
+py_test_run_all_notebooks(
+    size = "large",
+    include = ["tune-xgboost.ipynb"],
+    exclude = [],
+    data = ["//doc/source/tune/examples:tune_examples"],
+    tags = ["exclusive", "team:ml", "gpu"],
 )

doc/test_myst_doc.py

@@ -5,6 +5,7 @@
 import argparse
 import tempfile
 import sys
+from pathlib import Path
 
 import jupytext
 
@@ -13,12 +14,23 @@
     "--path",
     help="path to the jupytext-compatible file",
 )
+parser.add_argument(
+    "--find-recursively",
+    action="store_true",
+    help="if true, will attempt to find path recursively in cwd",
+)
 
 if __name__ == "__main__":
 
     args, remainder = parser.parse_known_args()
 
-    with open(args.path, "r") as f:
+    path = Path(args.path)
+    cwd = Path.cwd()
+    if args.find_recursively and not path.exists():
+        path = next((p for p in cwd.rglob("*") if str(p).endswith(args.path)), None)
+    assert path and path.exists()
+
+    with open(path, "r") as f:
         notebook = jupytext.read(f)
 
     name = ""