Initial Python bindings for cuProj

.github/workflows/build.yaml

@@ -66,17 +66,17 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
-  wheel-build:
+  wheel-build-cuspatial:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel.sh
-  wheel-publish:
-    needs: wheel-build
+      script: ci/build_wheel_cuspatial.sh
+  wheel-publish-cuspatial:
+    needs: wheel-build-cuspatial
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
     with:
@@ -85,3 +85,22 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cuspatial
+  wheel-build-cuproj:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_cuproj.sh
+  wheel-publish-cuproj:
+    needs: wheel-build-cuproj
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cuproj

.github/workflows/pr.yaml

@@ -19,8 +19,10 @@ jobs:
       - conda-python-tests
       - conda-notebook-tests
       - docs-build
-      - wheel-build
-      - wheel-tests
+      - wheel-build-cuspatial
+      - wheel-tests-cuspatial
+      - wheel-build-cuproj
+      - wheel-tests-cuproj
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08
   checks:
@@ -72,17 +74,31 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build:
+  wheel-build-cuspatial:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: pull-request
-      script: ci/build_wheel.sh
-  wheel-tests:
-    needs: wheel-build
+      script: ci/build_wheel_cuspatial.sh
+  wheel-tests-cuspatial:
+    needs: wheel-build-cuspatial
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
     with:
       build_type: pull-request
-      script: ci/test_wheel.sh
+      script: ci/test_wheel_cuspatial.sh
+  wheel-build-cuproj:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cuproj.sh
+  wheel-tests-cuproj:
+    needs: wheel-build-cuproj
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cuproj.sh

build.sh

@@ -18,11 +18,12 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcuspatial cuspatial tests benchmarks -v -g -n -h --allgpuarch --show_depr_warn"
+VALIDARGS="clean libcuspatial cuspatial cuproj tests benchmarks -v -g -n -h --allgpuarch --show_depr_warn"
 HELP="$0 [clean] [libcuspatial] [cuspatial] [tests] [-v] [-g] [-n] [-h] [-l] [--show_depr_warn] [--cmake-args=\"<args>\"]
    clean                       - remove all existing build artifacts and configuration (start over)
    libcuspatial                - build the libcuspatial C++ code only
    cuspatial                   - build the cuspatial Python package
+   cuproj                      - build the cuproj Python package
    tests                       - build tests
    benchmarks                  - build benchmarks
    -v                          - verbose build mode
@@ -36,8 +37,10 @@ HELP="$0 [clean] [libcuspatial] [cuspatial] [tests] [-v] [-g] [-n] [-h] [-l] [--
    'cuspatial' targets
 "
 LIBCUSPATIAL_BUILD_DIR=${REPODIR}/cpp/build
+LIBCUPROJ_BUILD_DIR=${REPODIR}/cpp/build/cuproj
 CUSPATIAL_BUILD_DIR=${REPODIR}/python/cuspatial/_skbuild
-BUILD_DIRS="${LIBCUSPATIAL_BUILD_DIR} ${CUSPATIAL_BUILD_DIR}"
+CUPROJ_BUILD_DIR=${REPODIR}/python/cuproj/_skbuild
+BUILD_DIRS="${LIBCUSPATIAL_BUILD_DIR} ${CUSPATIAL_BUILD_DIR} ${LIBCUPROJ_BUILD_DIR} ${CUPROJ_BUILD_DIR}"
 
 # Set defaults for vars modified by flags to this script
 VERBOSE_FLAG=""
@@ -126,6 +129,11 @@ if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUSPATIAL_CPP"* ]]; then
     EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUSPATIAL_CPP=ON"
 fi
 
+# Append `-DFIND_CUPROJ_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
+if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUPROJ_CPP"* ]]; then
+    EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUPROJ_CPP=ON"
+fi
+
 # If clean given, run it prior to any other steps
 if hasArg clean; then
     # If the dirs to clean are mounted dirs in a container, the
@@ -179,3 +187,13 @@ if (( ${NUMARGS} == 0 )) || hasArg cuspatial; then
         python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUSPATIAL_BUILD_DIR} ${EXTRA_CMAKE_ARGS}
     fi
 fi
+
+# Build and install the cuproj Python package
+if (( ${NUMARGS} == 0 )) || hasArg cuproj; then
+
+    cd ${REPODIR}/python/cuproj
+    python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUPROJ_BUILD_DIR} ${EXTRA_CMAKE_ARGS}
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+        python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUPROJ_BUILD_DIR} ${EXTRA_CMAKE_ARGS}
+    fi
+fi

ci/build_python.sh

@@ -9,15 +9,24 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-rapids-logger "Begin py build"
-
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
+rapids-logger "Begin py build cuSpatial"
+
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
 rapids-mamba-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/cuspatial
 
+rapids-logger "Begin py build cuProj"
+
+# TODO: Remove `--no-test` flag once importing on a CPU
+# node works correctly
+rapids-mamba-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/cuproj
+
 rapids-upload-conda-to-s3 python

ci/build_wheel_cuproj.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+source rapids-configure-sccache
+source rapids-date-string
+
+# Use gha-tools rapids-pip-wheel-version to generate wheel version then
+# update the necessary files
+version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
+echo "The package name and/or version was modified in the package source. The git diff is:"
+git diff
+
+cd python/cuproj
+
+SKBUILD_CONFIGURE_OPTIONS="-DCUPROJ_BUILD_WHEELS=ON" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

ci/release/apply_wheel_modifications.sh

@@ -7,8 +7,15 @@ VERSION=${1}
 CUDA_SUFFIX=${2}
 
 sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cuspatial/pyproject.toml
-
 sed -i "s/^name = \"cuspatial\"/name = \"cuspatial${CUDA_SUFFIX}\"/g" python/cuspatial/pyproject.toml
-
 sed -i "s/rmm==/rmm${CUDA_SUFFIX}==/g" python/cuspatial/pyproject.toml
 sed -i "s/cudf==/cudf${CUDA_SUFFIX}==/g" python/cuspatial/pyproject.toml
+
+sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cuproj/pyproject.toml
+sed -i "s/^name = \"cuproj\"/name = \"cuproj${CUDA_SUFFIX}\"/g" python/cuproj/pyproject.toml
+sed -i "s/rmm==/rmm${CUDA_SUFFIX}==/g" python/cuproj/pyproject.toml
+sed -i "s/cuspatial==/cuspatial${CUDA_SUFFIX}==/g" python/cuproj/pyproject.toml
+
+if [[ $CUDA_SUFFIX == "-cu12" ]]; then
+  sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cuproj/pyproject.toml
+fi

ci/release/update-version.sh

@@ -33,17 +33,18 @@ function sed_runner() {
 sed_runner 's/'"CUSPATIAL VERSION .* LANGUAGES"'/'"CUSPATIAL VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/CMakeLists.txt
 sed_runner 's/'"CUPROJ VERSION .* LANGUAGES"'/'"CUPROJ VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/cuproj/CMakeLists.txt
 sed_runner 's/'"cuspatial_version .*)"'/'"cuspatial_version ${NEXT_FULL_TAG})"'/g' python/cuspatial/CMakeLists.txt
+sed_runner 's/'"cuproj_version .*)"'/'"cuproj_version ${NEXT_FULL_TAG})"'/g' python/cuproj/CMakeLists.txt
 
 # RTD update
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py
 
 # Python __init__.py updates
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cuspatial/cuspatial/__init__.py
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cuproj/cuproj/__init__.py
 
 # rapids-cmake version
 sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
-sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' python/cuspatial/CMakeLists.txt
 
 # Doxyfile update
 sed_runner "/PROJECT_NUMBER[ ]*=/ s|=.*|= ${NEXT_FULL_TAG}|g" cpp/doxygen/Doxyfile
@@ -72,10 +73,12 @@ for DEP in "${DEPENDENCIES[@]}"; do
     sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE}
   done
   sed_runner "s/${DEP}==.*\",/${DEP}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cuspatial/pyproject.toml
+  sed_runner "s/${DEP}==.*\",/${DEP}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cuproj/pyproject.toml
 done
 
 # Version in pyproject.toml
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG_PEP440}\"/g" python/cuspatial/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG_PEP440}\"/g" python/cuproj/pyproject.toml
 
 # Dependency versions in dependencies.yaml
 sed_runner "/-cu[0-9]\{2\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" dependencies.yaml

ci/test_notebooks.sh

@@ -27,7 +27,7 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  cuspatial libcuspatial
+  cuspatial libcuspatial cuproj
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 pushd notebooks

ci/test_python.sh

@@ -34,7 +34,7 @@ rapids-print-env
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  libcuspatial cuspatial
+  libcuspatial cuspatial cuproj
 
 rapids-logger "Check GPU usage"
 nvidia-smi
@@ -58,5 +58,20 @@ pytest \
   tests
 popd
 
+rapids-logger "pytest cuproj"
+pushd python/cuproj/cuproj
+# It is essential to cd into python/cuproj/cuproj as `pytest-xdist` + `coverage` seem to work only at this directory level.
+pytest \
+  --cache-clear \
+  --junitxml="${RAPIDS_TESTS_DIR}/junit-cuproj.xml" \
+  --numprocesses=8 \
+  --dist=loadscope \
+  --cov-config=../.coveragerc \
+  --cov=cuproj \
+  --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuproj-coverage.xml" \
+  --cov-report=term \
+  tests
+popd
+
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

ci/test_wheel_cuproj.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -eou pipefail
+
+mkdir -p ./dist
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+
+# Install additional dependencies
+apt update
+DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
+python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install $(echo ./dist/cuproj*.whl)[test]
+
+if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
+    python ./ci/wheel_smoke_test_cuproj.py
+else
+    python -m pytest -n 8 ./python/cuproj/cuproj/tests
+fi

ci/test_wheel.sh → ci/test_wheel_cuspatial.sh

@@ -16,7 +16,7 @@ python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
 python -m pip install $(echo ./dist/cuspatial*.whl)[test]
 
 if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
-    python ./ci/wheel_smoke_test.py
+    python ./ci/wheel_smoke_test_cuspatial.py
 else
     python -m pytest -n 8 ./python/cuspatial/cuspatial/tests
 fi

ci/wheel_smoke_test_cuproj.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+from cuproj import Transformer as cuTransformer
+
+from cupy.testing import assert_allclose
+
+if __name__ == '__main__':
+    # Sydney opera house latitude and longitude
+    lat = -33.8587
+    lon = 151.2140
+
+    # Transform to UTM using cuproj
+    cu_transformer = cuTransformer.from_crs("epsg:4326", "EPSG:32756")
+    cuproj_x, cuproj_y = cu_transformer.transform(lat, lon)
+
+    assert_allclose(cuproj_x, 334783.9544807102)
+    assert_allclose(cuproj_y, 6252075.961741454)

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -12,6 +12,8 @@ dependencies:
 - cudatoolkit
 - cudf==23.8.*
 - cuml==23.8.*
+- cupy>=12.0.0
+- cuspatial==23.8.*
 - cxx-compiler
 - cython>=0.29,<0.30
 - doxygen

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -15,6 +15,8 @@ dependencies:
 - cuda-version=12.0
 - cudf==23.8.*
 - cuml==23.8.*
+- cupy>=12.0.0
+- cuspatial==23.8.*
 - cxx-compiler
 - cython>=0.29,<0.30
 - doxygen

conda/recipes/cuproj/build.sh

@@ -0,0 +1,4 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+# This assumes the script is executed from the root of the repo directory
+./build.sh cuproj

conda/recipes/cuproj/conda_build_config.yaml

@@ -0,0 +1,17 @@
+c_compiler_version:
+  - 11
+
+cxx_compiler_version:
+  - 11
+
+cuda_compiler:
+  - cuda-nvcc
+
+cuda11_compiler:
+  - nvcc
+
+sysroot_version:
+  - "2.17"
+
+cmake_version:
+  - ">=3.26.4"