Merge branch 'branch-23.02' into enh-ivf-pq-batched-building

.github/workflows/dependency-files.yml

@@ -0,0 +1,12 @@
+name: dependency-files
+
+on:
+  pull_request:
+
+jobs:
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main
+    with:
+      enable_check_size: false
+      enable_check_style: false

.pre-commit-config.yaml

@@ -97,6 +97,8 @@ repos:
         rev: v2.1.0
         hooks:
               - id: codespell
+                exclude: (?x)^(^CHANGELOG.md$)
+
 
 default_language_version:
       python: python3

README.md

@@ -176,7 +176,7 @@ pairwise_distance(in1, in2, out=output, metric="euclidean")
 
 ## Installing
 
-RAFT itself can be installed through conda, [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake), pip, or by building the repository from source. Please refer to the [build instructions](docs/source/build.md) for more a comprehensive guide on building RAFT and using it in downstream projects.
+RAFT itself can be installed through conda, [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake), pip, or by building the repository from source. Please refer to the [build instructions](docs/source/build.md) for more a comprehensive guide on installing and building RAFT and using it in downstream projects.
 
 ### Conda
 
@@ -262,9 +262,9 @@ Several CMake targets can be made available by adding components in the table be
 ### Source
 
 The easiest way to build RAFT from source is to use the `build.sh` script at the root of the repository:
-1. Create an environment with the needed dependencies: 
+1. Create an environment with the needed dependencies:
 ```
-mamba env create --name raft_dev_env -f conda/environments/raft_dev_cuda11.5.yml
+mamba env create --name raft_dev_env -f conda/environments/all_cuda-115_arch-x86_64.yaml
 mamba activate raft_dev_env
 ```
 ```

build.sh

@@ -18,7 +18,7 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps"
+VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean --uninstall  -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps"
 HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>] [--limit-bench=<targets>]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
@@ -34,6 +34,7 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
    -v                          - verbose build mode
    -g                          - build for debug
    -n                          - no install step
+   --uninstall                 - uninstall files for specified targets which were built and installed prior
    --compile-libs              - compile shared libraries for all components
    --compile-nn                - compile shared library for nn component
    --compile-dist              - compile shared library for distance and current random components
@@ -56,9 +57,9 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
 "
 LIBRAFT_BUILD_DIR=${LIBRAFT_BUILD_DIR:=${REPODIR}/cpp/build}
 SPHINX_BUILD_DIR=${REPODIR}/docs
-PY_RAFT_BUILD_DIR=${REPODIR}/python/raft/build
-PY_LIBRAFT_BUILD_DIR=${REPODIR}/python/pylibraft/_skbuild
-BUILD_DIRS="${LIBRAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PY_LIBRAFT_BUILD_DIR}"
+RAFT_DASK_BUILD_DIR=${REPODIR}/python/raft-dask/_skbuild
+PYLIBRAFT_BUILD_DIR=${REPODIR}/python/pylibraft/_skbuild
+BUILD_DIRS="${LIBRAFT_BUILD_DIR} ${PYLIBRAFT_BUILD_DIR} ${RAFT_DASK_BUILD_DIR}"
 
 # Set defaults for vars modified by flags to this script
 CMAKE_LOG_LEVEL=""
@@ -190,6 +191,65 @@ if (( ${NUMARGS} != 0 )); then
     done
 fi
 
+# This should run before build/install
+if hasArg --uninstall; then
+    UNINSTALL=1
+
+    if hasArg pylibraft || hasArg libraft || (( ${NUMARGS} == 1 )); then
+
+      echo "Removing libraft files..."
+      if [ -e ${LIBRAFT_BUILD_DIR}/install_manifest.txt ]; then
+          xargs rm -fv < ${LIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1
+      fi
+    fi
+
+    if hasArg pylibraft || (( ${NUMARGS} == 1 )); then
+      echo "Uninstalling pylibraft package..."
+      if [ -e ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt ]; then
+          xargs rm -fv < ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1
+      fi
+
+      # Try to uninstall via pip if it is installed
+      if [ -x "$(command -v pip)" ]; then
+        echo "Using pip to uninstall pylibraft"
+        pip uninstall -y pylibraft
+
+      # Otherwise, try to uninstall through conda if that's where things are installed
+      elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" == "$CONDA_PREFIX" ]; then
+        echo "Using conda to uninstall pylibraft"
+        conda uninstall -y pylibraft
+
+      # Otherwise, fail
+      else
+        echo "Could not uninstall pylibraft from pip or conda. pylibraft package will need to be manually uninstalled"
+      fi
+    fi
+
+    if hasArg raft-dask || (( ${NUMARGS} == 1 )); then
+      echo "Uninstalling raft-dask package..."
+      if [ -e ${RAFT_DASK_BUILD_DIR}/install_manifest.txt ]; then
+          xargs rm -fv < ${RAFT_DASK_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1
+      fi
+
+      # Try to uninstall via pip if it is installed
+      if [ -x "$(command -v pip)" ]; then
+        echo "Using pip to uninstall raft-dask"
+        pip uninstall -y raft-dask
+
+      # Otherwise, try to uninstall through conda if that's where things are installed
+      elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" == "$CONDA_PREFIX" ]; then
+        echo "Using conda to uninstall raft-dask"
+        conda uninstall -y raft-dask
+
+      # Otherwise, fail
+      else
+        echo "Could not uninstall raft-dask from pip or conda. raft-dask package will need to be manually uninstalled."
+      fi
+    fi
+    exit 0
+fi
+
+
 # Process flags
 if hasArg -n; then
     INSTALL_TARGET=""
@@ -286,9 +346,8 @@ fi
 if hasArg clean; then
     CLEAN=1
 fi
-if hasArg uninstall; then
-    UNINSTALL=1
-fi
+
+
 
 if [[ ${CMAKE_TARGET} == "" ]]; then
     CMAKE_TARGET="all"
@@ -370,7 +429,7 @@ if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then
     fi
 
     cd ${REPODIR}/python/raft-dask
-    python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${LIBRAFT_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
+    python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${RAFT_DASK_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
     if [[ ${INSTALL_TARGET} != "" ]]; then
         python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS}
     fi
@@ -384,7 +443,7 @@ if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then
     fi
 
     cd ${REPODIR}/python/pylibraft
-    python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${LIBRAFT_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
+    python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${RAFT_DASK_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
     if [[ ${INSTALL_TARGET} != "" ]]; then
         python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS}
     fi

ci/gpu/build.sh

@@ -34,10 +34,10 @@ unset GIT_DESCRIBE_TAG
 export UCX_PY_VERSION='0.30.*'
 
 # Whether to install dask nightly or stable packages.
-export INSTALL_DASK_MAIN=1
+export INSTALL_DASK_MAIN=0
 
 # Dask version to install when `INSTALL_DASK_MAIN=0`
-export DASK_STABLE_VERSION="2022.9.2"
+export DASK_STABLE_VERSION="2022.11.1"
 
 ################################################################################
 # SETUP - Check environment

ci/release/update-version.sh

@@ -41,7 +41,7 @@ sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cma
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py
 
-for FILE in conda/environments/*.yml; do
+for FILE in conda/environments/*.yaml dependencies.yaml; do
   sed_runner "s/dask-cuda=${CURRENT_SHORT_TAG}/dask-cuda=${NEXT_SHORT_TAG}/g" ${FILE};
   sed_runner "s/rapids-build-env=${CURRENT_SHORT_TAG}/rapids-build-env=${NEXT_SHORT_TAG}/g" ${FILE};
   sed_runner "s/rapids-doc-env=${CURRENT_SHORT_TAG}/rapids-doc-env=${NEXT_SHORT_TAG}/g" ${FILE};

conda/environments/all_cuda-115_arch-x86_64.yaml

@@ -0,0 +1,38 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- rapidsai
+- rapidsai-nightly
+- dask/label/dev
+- conda-forge
+- nvidia
+dependencies:
+- breathe
+- c-compiler
+- clang-tools=11.1.0
+- clang=11.1.0
+- cmake>=3.23.1,!=3.25.0
+- cuda-python >=11.7.1,<12.0
+- cudatoolkit=11.5
+- cxx-compiler
+- cython>=0.29,<0.30
+- dask-cuda=23.02.*
+- dask==2022.11.1
+- distributed==2022.11.1
+- doxygen>=1.8.20
+- faiss-proc=*=cuda
+- gcc_linux-64=9.*
+- libfaiss>=1.7.0
+- ninja
+- pytest
+- rapids-build-env=23.02.*
+- rapids-doc-env=23.02.*
+- rapids-notebook-env=23.02.*
+- rmm=23.02.*
+- scikit-build>=0.13.1
+- sphinx-markdown-tables
+- sysroot_linux-64==2.17
+- ucx-proc=*=gpu
+- ucx-py=0.30.*
+- ucx>=1.13.0
+name: all_cuda-115_arch-x86_64

conda/recipes/libraft/meta.yaml

@@ -57,9 +57,6 @@ outputs:
         - libcusolver {{ libcusolver_version }}
         - libcusparse {{ libcusparse_version }}
         - librmm {{ minor_version }}
-        - nccl {{ nccl_version }}
-        - ucx-proc=*=gpu
-        - ucx-py {{ ucx_py_version }}
     about:
       home: http://rapids.ai/
       license: Apache-2.0
@@ -83,16 +80,10 @@ outputs:
       host:
         - cudatoolkit {{ cuda_version }}.*
         - librmm {{ minor_version }}
-        - nccl {{ nccl_version }}
-        - ucx-proc=*=gpu
-        - ucx-py {{ ucx_py_version }}
         - {{ pin_subpackage('libraft-headers', exact=True) }}
       run:
         - cudatoolkit {{ cuda_spec }}
         - librmm {{ minor_version }}
-        - nccl {{ nccl_version }}
-        - ucx-proc=*=gpu
-        - ucx-py {{ ucx_py_version }}
         - libcusolver {{ libcusolver_version }}
         - libcusparse {{ libcusparse_version }}
         - {{ pin_subpackage('libraft-headers', exact=True) }}