Generate pyproject dependencies with dfg

.pre-commit-config.yaml

@@ -61,7 +61,7 @@ repos:
             pass_filenames: false
             language: python
     - repo: https://github.com/rapidsai/dependency-file-generator
-      rev: v1.4.0
+      rev: v1.5.1
       hooks:
           - id: rapids-dependency-file-generator
             args: ["--clean"]

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -11,12 +11,12 @@ dependencies:
 - cmake>=3.23.1,!=3.25.0
 - cuda-python>=11.7.1,<12.0
 - cudatoolkit=11.8
-- cudf=23.04.*
-- cupy>=7.8.0,<12.0.0a0
+- cudf==23.4.*
+- cupy>=9.5.0,<12.0.0a0
 - cxx-compiler
 - cython>=0.29,<0.30
-- dask-cuda=23.04.*
-- dask-cudf=23.04.*
+- dask-cuda==23.4.*
+- dask-cudf==23.4.*
 - dask-ml
 - dask>=2023.1.1
 - distributed>=2023.1.1
@@ -33,41 +33,44 @@ dependencies:
 - libcublas=11.11.3.6
 - libcufft-dev=10.9.0.58
 - libcufft=10.9.0.58
-- libcumlprims=23.04.*
+- libcumlprims==23.4.*
 - libcurand-dev=10.3.0.86
 - libcurand=10.3.0.86
 - libcusolver-dev=11.4.1.48
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
 - libfaiss>=1.7.1
-- libraft-headers=23.04.*
-- libraft=23.04.*
+- libraft-headers==23.4.*
+- libraft==23.4.*
+- librmm==23.4.*
 - nbsphinx
 - ninja
 - nltk
+- numba
 - numpydoc
 - pip
 - pydata-sphinx-theme
-- pylibraft=23.04.*
+- pylibraft==23.4.*
 - pytest
+- pytest-benchmark
 - pytest-cases
 - pytest-cov
 - pytest-xdist
 - python>=3.8,<3.11
-- raft-dask=23.04.*
+- raft-dask==23.4.*
 - recommonmark
-- rmm=23.04.*
+- rmm==23.4.*
 - scikit-build>=0.13.1
-- scikit-learn=1.2
+- scikit-learn==1.2
+- scipy
 - seaborn
-- sparse
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - statsmodels
 - sysroot_linux-64==2.17
-- treelite=3.1.0
+- treelite==3.1.0
 - umap-learn
 - pip:
   - git+https://github.com/dask/dask-glm@main

conda/environments/cpp_all_cuda-118_arch-x86_64.yaml

@@ -17,17 +17,17 @@ dependencies:
 - libcublas=11.11.3.6
 - libcufft-dev=10.9.0.58
 - libcufft=10.9.0.58
-- libcumlprims=23.04.*
+- libcumlprims==23.4.*
 - libcurand-dev=10.3.0.86
 - libcurand=10.3.0.86
 - libcusolver-dev=11.4.1.48
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
 - libfaiss>=1.7.1
-- libraft-headers=23.04.*
-- libraft=23.04.*
+- libraft-headers==23.4.*
+- libraft==23.4.*
+- librmm==23.4.*
 - ninja
-- rmm=23.04.*
 - sysroot_linux-64==2.17
 name: cpp_all_cuda-118_arch-x86_64

conda/recipes/cuml/meta.yaml

@@ -79,6 +79,7 @@ requirements:
     - python x.x
     - raft-dask ={{ minor_version }}
     - treelite {{ treelite_version }}
+    - seaborn
 
 tests:
   requirements:

dependencies.yaml

@@ -51,6 +51,26 @@ files:
       - py_run
       - py_version
       - test_notebooks
+  py_build:
+    output: pyproject
+    extras:
+      table: build-system
+    includes:
+      - common_build
+      - py_build
+  py_run:
+    output: pyproject
+    extras:
+      table: project
+    includes:
+      - py_run
+  py_test:
+    output: pyproject
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -65,18 +85,18 @@ dependencies:
           - pre-commit
   common_build:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.23.1,!=3.25.0
           - ninja
       - output_types: conda
         packages:
           - c-compiler
           - cxx-compiler
-          - libcumlprims=23.04.*
-          - libraft-headers=23.04.*
-          - libraft=23.04.*
-          - rmm=23.04.*
+          - libcumlprims==23.4.*
+          - libraft==23.4.*
+          - libraft-headers==23.4.*
+          - librmm==23.4.*
     specific:
       - output_types: conda
         matrices:
@@ -92,29 +112,41 @@ dependencies:
               - sysroot_linux-aarch64==2.17
   py_build:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
           - cuda-python>=11.7.1,<12.0
           - scikit-build>=0.13.1
           - cython>=0.29,<0.30
-          - treelite=3.1.0
-      - output_types: conda
+          - &treelite treelite==3.1.0
+          - pylibraft==23.4.*
+          - rmm==23.4.*
+      - output_types: pyproject
         packages:
-          - cudf=23.04.*
-          - raft-dask=23.04.*
-          - pylibraft=23.04.*
+          - wheel
+          - setuptools
+          - &treelite_runtime treelite_runtime==3.1.0
   py_run:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
+          - cudf==23.4.*
+          - cupy>=9.5.0,<12.0.0a0
           - dask>=2023.1.1
+          - dask-cuda==23.4.*
+          - dask-cudf==23.4.*
           - distributed>=2023.1.1
           - joblib>=0.11
-      - output_types: conda
+          - numba
+            # TODO: Are seaborn and scipy really hard dependencies, or should
+            # we make them optional (i.e. an extra for pip
+            # installation/run_constrained for conda)?
+          - seaborn
+          - scipy
+          - raft-dask==23.4.*
+          - *treelite
+      - output_types: pyproject
         packages:
-          - cupy>=7.8.0,<12.0.0a0
-          - dask-cuda=23.04.*
-          - dask-cudf=23.04.*
+          - *treelite_runtime
     specific:
       - output_types: requirements
         matrices:
@@ -205,7 +237,7 @@ dependencies:
           - numpydoc
           - pydata-sphinx-theme
           - recommonmark
-          - &scikit_learn scikit-learn=1.2
+          - &scikit_learn scikit-learn==1.2
           - sphinx<6
           - sphinx-copybutton
           - sphinx-markdown-tables
@@ -233,21 +265,18 @@ dependencies:
               - python>=3.8,<3.11
   test_python:
     common:
-      - output_types: [conda, requirements]
+      - output_types: [conda, requirements, pyproject]
         packages:
-          - cython>=0.29,<0.30
           - dask-ml
-          - hdbscan
           - hypothesis>=6.0,<7
           - nltk
           - numpydoc
           - pytest
+          - pytest-benchmark
           - pytest-cases
           - pytest-cov
           - pytest-xdist
           - *scikit_learn
-          - seaborn
-          - sparse
           - statsmodels
           - umap-learn
       - output_types: conda
@@ -257,6 +286,15 @@ dependencies:
               # TODO: Figure out what to do with this dependency
               # since the repo is now archived.
               - git+https://github.com/dask/dask-glm@main
+          # TODO: The conda-forge package is being generated from Leland's fork
+          # (https://github.com/lmcinnes/hdbscan) which is far behind now. Do
+          # we need to have that updated?
+          - hdbscan
+      - output_types: pyproject
+        packages:
+          - dask-glm @ git+https://github.com/dask/dask-glm@main
+            # TODO: Can we stop pulling from the master branch now that there was a release in October?
+          - hdbscan @ git+https://github.com/scikit-learn-contrib/hdbscan.git@master
   test_notebooks:
     common:
       - output_types: [conda, requirements]
@@ -265,4 +303,4 @@ dependencies:
           - matplotlib
           - numpy
           - pandas
-          - *scikit_learn
+          - *scikit_learn

python/pyproject.toml

@@ -14,17 +14,18 @@
 
 [build-system]
 requires = [
-    "wheel",
-    "setuptools",
-    "cython>=0.29,<0.30",
-    "scikit-build>=0.13.1",
     "cmake>=3.23.1,!=3.25.0",
+    "cuda-python>=11.7.1,<12.0",
+    "cython>=0.29,<0.30",
     "ninja",
+    "pylibraft==23.4.*",
+    "rmm==23.4.*",
+    "scikit-build>=0.13.1",
+    "setuptools",
     "treelite==3.1.0",
     "treelite_runtime==3.1.0",
-    "rmm==23.4.*",
-    "pylibraft==23.4.*"
-]
+    "wheel",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
@@ -56,16 +57,20 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.8"
 dependencies = [
+    "cudf==23.4.*",
+    "cupy>=9.5.0,<12.0.0a0",
+    "dask-cuda==23.4.*",
+    "dask-cudf==23.4.*",
+    "dask>=2023.1.1",
+    "distributed>=2023.1.1",
+    "joblib>=0.11",
     "numba",
+    "raft-dask==23.4.*",
     "scipy",
     "seaborn",
     "treelite==3.1.0",
     "treelite_runtime==3.1.0",
-    "cudf==23.4.*",
-    "dask-cudf==23.4.*",
-    "pylibraft==23.4.*",
-    "raft-dask==23.4.*",
-]
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
     "Programming Language :: Python",
@@ -76,21 +81,21 @@ classifiers = [
 
 [project.optional-dependencies]
 test = [
+    "dask-glm @ git+https://github.com/dask/dask-glm@main",
+    "dask-ml",
+    "hdbscan @ git+https://github.com/scikit-learn-contrib/hdbscan.git@master",
+    "hypothesis>=6.0,<7",
+    "nltk",
+    "numpydoc",
     "pytest",
-    "hypothesis",
-    "pytest-xdist",
     "pytest-benchmark",
     "pytest-cases",
-    "nltk",
-    "dask-ml",
-    "numpydoc",
-    "umap-learn",
-    "statsmodels",
+    "pytest-cov",
+    "pytest-xdist",
     "scikit-learn==1.2",
-    "hdbscan @ git+https://github.com/scikit-learn-contrib/hdbscan.git@master",  # noqa:E501
-    "dask-glm @ git+https://github.com/dask/dask-glm@main",
-    "dask-cuda",
-]
+    "statsmodels",
+    "umap-learn",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/cuml"