Merge branch 'branch-23.06' of github.com:rapidsai/cudf into clarify-…

…result-ordering-notes
rapidsai · May 24, 2023 · 51262d4 · 51262d4
2 parents 8f31baf + 19554a1
commit 51262d4
Show file tree

Hide file tree

Showing 363 changed files with 14,377 additions and 5,524 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -54,16 +54,19 @@ jobs:
       sha: ${{ inputs.sha }}
       skip_upload_pkgs: libcudf-example
   docs-build:
-    if: github.ref_type == 'branch' && github.event_name == 'push'
+    if: github.ref_type == 'branch'
     needs: python-build
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
     with:
-      build_type: branch
-      node_type: "gpu-v100-latest-1"
       arch: "amd64"
+      branch: ${{ inputs.branch }}
+      build_type: ${{ inputs.build_type || 'branch' }}
       container_image: "rapidsai/ci:latest"
+      date: ${{ inputs.date }}
+      node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
+      sha: ${{ inputs.sha }}
   wheel-build-cudf:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -111,8 +111,6 @@ jobs:
     with:
       build_type: pull-request
       package-name: cudf
-      # Install cupy-cuda11x for arm from a special index url
-      test-before-arm64: "python -m pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64"
       test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests"
       test-smoketest: "python ./ci/wheel_smoke_test_cudf.py"
   wheel-build-dask-cudf:
@@ -123,7 +121,7 @@ jobs:
       build_type: pull-request
       package-name: dask_cudf
       package-dir: python/dask_cudf
-      before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf && python -m pip install --no-deps ./local-cudf/cudf*.whl"
+      before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf && python -m pip install --no-deps ./local-cudf/cudf*.whl"
   wheel-tests-dask-cudf:
     needs: wheel-build-dask-cudf
     secrets: inherit
@@ -132,5 +130,5 @@ jobs:
       build_type: pull-request
       package-name: dask_cudf
       # Install the cudf we just built, and also test against latest dask/distributed/dask-cuda.
-      test-before: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && python -m pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       test-unittest: "python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests"
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -86,7 +86,6 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       package-name: cudf
-      test-before-arm64: "python -m pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64"
       test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests"
   wheel-tests-dask-cudf:
     secrets: inherit
@@ -98,5 +97,5 @@ jobs:
       sha: ${{ inputs.sha }}
       package-name: dask_cudf
       # Test against latest dask/distributed/dask-cuda.
-      test-before: "pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before: "python -m pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       test-unittest: "python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
         hooks:
               - id: cython-lint
       - repo: https://github.com/pre-commit/mirrors-mypy
-        rev: 'v0.971'
+        rev: 'v1.3.0'
         hooks:
               - id: mypy
                 additional_dependencies: [types-cachetools]

diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/README.md b/README.md
@@ -71,7 +71,7 @@ conda install -c rapidsai -c conda-forge -c nvidia \
 We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD
 of our latest development branch.
 
-Note: cuDF is supported only on Linux, and with Python versions 3.8 and later.
+Note: cuDF is supported only on Linux, and with Python versions 3.9 and later.
 
 See the [Get RAPIDS version picker](https://rapids.ai/start.html) for more OS and version info.
 

diff --git a/build.sh b/build.sh
@@ -169,7 +169,7 @@ function buildLibCudfJniInDocker {
                 -DUSE_GDS=ON \
                 -DGPU_ARCHS=${CUDF_CMAKE_CUDA_ARCHITECTURES} \
                 -DCUDF_JNI_LIBCUDF_STATIC=ON \
-                -Dtest=*,!CuFileTest,!CudaFatalTest"
+                -Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest"
 }
 
 if hasArg -h || hasArg --h || hasArg --help; then

diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -47,7 +47,7 @@ sphinx-build -b text source _text
 popd
 
 
-if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then
+if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then
   rapids-logger "Upload Docs to S3"
   aws s3 sync --no-progress --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html"
   aws s3 sync --no-progress --delete docs/cudf/_html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html"

diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh
@@ -22,3 +22,11 @@ sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
 sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/pyproject.toml
 # Need to provide the == to avoid modifying the URL
 sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml
+
+if [[ $CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/cudf/pyproject.toml
+    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml
+    sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/{cudf,dask_cudf}/pyproject.toml
+    sed -i "/ptxcompiler/d" python/cudf/pyproject.toml
+    sed -i "/cubinlinker/d" python/cudf/pyproject.toml
+fi
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -78,10 +78,11 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/dask_cudf/so
 
 # bump rmm & dask-cuda
 for FILE in conda/environments/*.yaml dependencies.yaml; do
-  sed_runner "s/dask-cuda==${CURRENT_SHORT_TAG_PEP440}/dask-cuda==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
-  sed_runner "s/rmm==${CURRENT_SHORT_TAG_PEP440}/rmm==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
   sed_runner "s/cudf==${CURRENT_SHORT_TAG_PEP440}/cudf==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
   sed_runner "s/cudf_kafka==${CURRENT_SHORT_TAG_PEP440}/cudf_kafka==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/dask-cuda==${CURRENT_SHORT_TAG_PEP440}/dask-cuda==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/kvikio==${CURRENT_SHORT_TAG_PEP440}/kvikio==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/rmm==${CURRENT_SHORT_TAG_PEP440}/rmm==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
 done
 
 # Doxyfile update

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -18,7 +18,7 @@ dependencies:
 - cuda-python>=11.7.1,<12.0
 - cuda-sanitizer-api=11.8.86
 - cudatoolkit=11.8
-- cupy>=9.5.0,<12.0.0a0
+- cupy>=12.0.0
 - cxx-compiler
 - cython>=0.29,<0.30
 - dask-core==2023.3.2
@@ -31,13 +31,14 @@ dependencies:
 - fmt>=9.1.0,<10
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
-- gmock==1.10.0.*
-- gtest==1.10.0.*
+- gmock>=1.13.0
+- gtest>=1.13.0
 - hypothesis
 - ipython
 - libarrow==11.0.0.*
 - libcurand-dev=10.3.0.86
 - libcurand=10.3.0.86
+- libkvikio==23.6.*
 - librdkafka>=1.9.0,<1.10.0a0
 - librmm==23.6.*
 - mimesis>=4.1.0
@@ -47,8 +48,8 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba>=0.56.4,<0.57
-- numpy>=1.21
+- numba>=0.57
+- numpy>=1.21,<1.24
 - numpydoc
 - nvcc_linux-64=11.8
 - nvtx>=0.2.1
@@ -69,7 +70,7 @@ dependencies:
 - pytest-xdist
 - python-confluent-kafka>=1.9.0,<1.10.0a0
 - python-snappy>=0.6.0
-- python>=3.8,<3.11
+- python>=3.9,<3.11
 - pytorch<1.12.0
 - rmm==23.6.*
 - s3fs>=2022.3.0

diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml
@@ -32,6 +32,7 @@ build:
     - SCCACHE_S3_KEY_PREFIX=cudf-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=cudf-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
+    - SCCACHE_S3_NO_CREDENTIALS
   ignore_run_exports:
     # libcudf's run_exports pinning is looser than we would like
     - libcudf
@@ -52,7 +53,6 @@ requirements:
     - cython >=0.29,<0.30
     - scikit-build >=0.13.1
     - setuptools
-    - numba >=0.56.4,<0.57
     - dlpack >=0.5,<0.6.0a0
     - pyarrow =11
     - libcudf ={{ version }}
@@ -63,12 +63,11 @@ requirements:
     - python
     - typing_extensions
     - pandas >=1.3,<1.6.0dev0
-    - cupy >=9.5.0,<12.0.0a0
-    - numba >=0.56.4,<0.57
-    - numpy >=1.21
+    - cupy >=12.0.0
+    - numba >=0.57
+    - numpy >=1.21,<1.24  # Temporarily upper bound numpy to avoid overflow deprecations
     - {{ pin_compatible('pyarrow', max_pin='x.x.x') }}
     - libcudf {{ version }}
-    - fastavro >=0.22.0
     - {{ pin_compatible('rmm', max_pin='x.x') }}
     - fsspec >=0.6.0
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}

diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml
@@ -31,6 +31,7 @@ build:
     - SCCACHE_S3_KEY_PREFIX=cudf-kafka-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
+    - SCCACHE_S3_NO_CREDENTIALS
 
 requirements:
   build:

diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml
@@ -31,6 +31,7 @@ build:
     - SCCACHE_S3_KEY_PREFIX=custreamz-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=custreamz-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
+    - SCCACHE_S3_NO_CREDENTIALS
 
 requirements:
   host:

diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml
@@ -32,6 +32,7 @@ build:
     - SCCACHE_S3_KEY_PREFIX=dask-cudf-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=dask-cudf-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
+    - SCCACHE_S3_NO_CREDENTIALS
 
 requirements:
   host:

diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml
@@ -14,7 +14,7 @@ cmake_version:
   - ">=3.23.1,!=3.25.0"
 
 gtest_version:
-  - "=1.10.0"
+  - ">=1.13.0"
 
 libarrow_version:
   - "=11"
@@ -25,6 +25,16 @@ dlpack_version:
 librdkafka_version:
   - ">=1.9.0,<1.10.0a0"
 
+# The CTK libraries below are missing from the conda-forge::cudatoolkit
+# package. The "*_host_*" version specifiers correspond to `11.8` packages and the
+# "*_run_*" version specifiers correspond to `11.x` packages.
+
+libcufile_host_version:
+  - "1.4.0.31"
+
+libcufile_run_version:
+  - ">=1.0.0.82,<=1.4.0.31"
+
 libcurand_host_version:
   - "=10.3.0.86"
 

diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
@@ -30,6 +30,7 @@ build:
     - SCCACHE_S3_KEY_PREFIX=libcudf-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=libcudf-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
+    - SCCACHE_S3_NO_CREDENTIALS
 
 requirements:
   build:
@@ -41,10 +42,13 @@ requirements:
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
     - librmm ={{ minor_version }}
+    - libkvikio ={{ minor_version }}
     - cudatoolkit ={{ cuda_version }}
     - cuda-nvrtc ={{ cuda_version }}
     - cuda-nvrtc-dev ={{ cuda_version }}
     - cuda-nvtx ={{ cuda_version }}
+    - libcufile {{ libcufile_host_version }}      # [linux64]
+    - libcufile-dev {{ libcufile_host_version }}  # [linux64]
     - libcurand {{ libcurand_host_version }}
     - libcurand-dev {{ libcurand_host_version }}
     - libarrow {{ libarrow_version }}
@@ -72,7 +76,10 @@ outputs:
       run:
         - cudatoolkit {{ cuda_spec }}
         - librmm ={{ minor_version }}
+        - libkvikio ={{ minor_version }}
         - libarrow {{ libarrow_version }}
+        - libcufile {{ libcufile_run_version }}      # [linux64]
+        - libcufile-dev {{ libcufile_run_version }}  # [linux64]
         - dlpack {{ dlpack_version }}
         - gtest {{ gtest_version }}
         - gmock {{ gtest_version }}
@@ -91,6 +98,7 @@ outputs:
         - test -f $PREFIX/include/cudf/column/column_factories.hpp
         - test -f $PREFIX/include/cudf/column/column_view.hpp
         - test -f $PREFIX/include/cudf/concatenate.hpp
+        - test -f $PREFIX/include/cudf/contiguous_split.hpp
         - test -f $PREFIX/include/cudf/copying.hpp
         - test -f $PREFIX/include/cudf/datetime.hpp
         - test -f $PREFIX/include/cudf/timezone.hpp
@@ -99,6 +107,8 @@ outputs:
         - test -f $PREFIX/include/cudf/detail/binaryop.hpp
         - test -f $PREFIX/include/cudf/detail/calendrical_month_sequence.cuh
         - test -f $PREFIX/include/cudf/detail/concatenate.hpp
+        - test -f $PREFIX/include/cudf/detail/concatenate_masks.hpp
+        - test -f $PREFIX/include/cudf/detail/contiguous_split.hpp
         - test -f $PREFIX/include/cudf/detail/copy.hpp
         - test -f $PREFIX/include/cudf/detail/datetime.hpp
         - test -f $PREFIX/include/cudf/detail/fill.hpp
@@ -139,7 +149,7 @@ outputs:
         - test -f $PREFIX/include/cudf/detail/utilities/integer_utils.hpp
         - test -f $PREFIX/include/cudf/detail/utilities/linked_column.hpp
         - test -f $PREFIX/include/cudf/detail/utilities/logger.hpp
-        - test -f $PREFIX/include/cudf/detail/utilities/pinned_allocator.hpp
+        - test -f $PREFIX/include/cudf/detail/utilities/pinned_host_vector.hpp
         - test -f $PREFIX/include/cudf/detail/utilities/vector_factories.hpp
         - test -f $PREFIX/include/cudf/detail/utilities/visitor_overload.hpp
         - test -f $PREFIX/include/cudf/dictionary/detail/concatenate.hpp
@@ -387,6 +397,8 @@ outputs:
         - {{ pin_subpackage('libcudf', exact=True) }}
         - {{ pin_subpackage('libcudf_kafka', exact=True) }}
         - cudatoolkit {{ cuda_spec }}
+        - gtest {{ gtest_version }}
+        - gmock {{ gtest_version }}
         - libcurand {{ libcurand_run_version }}
     about:
       home: https://rapids.ai/

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -346,6 +346,7 @@ add_library(
   src/io/comp/nvcomp_adapter.cpp
   src/io/comp/nvcomp_adapter.cu
   src/io/comp/snap.cu
+  src/io/comp/statistics.cu
   src/io/comp/uncomp.cpp
   src/io/comp/unsnap.cu
   src/io/csv/csv_gpu.cu
@@ -397,7 +398,6 @@ add_library(
   src/io/utilities/parsing_utils.cu
   src/io/utilities/row_selection.cpp
   src/io/utilities/trie.cu
-  src/io/utilities/type_conversion.cpp
   src/jit/cache.cpp
   src/jit/parser.cpp
   src/jit/util.cpp
@@ -514,6 +514,7 @@ add_library(
   src/stream_compaction/stable_distinct.cu
   src/stream_compaction/unique.cu
   src/stream_compaction/unique_count.cu
+  src/stream_compaction/unique_count_column.cu
   src/strings/attributes.cu
   src/strings/capitalize.cu
   src/strings/case.cu