cuSpatial pip packages (#1148)

Initial draft PR to build statically linked cuspatial wheels. Closes #869. Authors: - Paul Taylor (https://github.com/trxcllnt) - Michael Wang (https://github.com/isVoid) Approvers: - H. Thomson Comer (https://github.com/thomcom) - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) - Mark Harris (https://github.com/harrism) - Vyas Ramasubramani (https://github.com/vyasr) URL: #1148
rapidsai · Jun 2, 2023 · 4b76c44 · 4b76c44
1 parent f07ac48
commit 4b76c44
Show file tree

Hide file tree

Showing 36 changed files with 575 additions and 2,756 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.5
 
 ARG CUDA=11.8
-ARG LLVM=15
-ARG RAPIDS=23.04
+ARG LLVM=16
+ARG RAPIDS=23.06
 ARG DISTRO=ubuntu22.04
 ARG REPO=rapidsai/devcontainers
 
@@ -19,12 +19,12 @@ ENV PYTHON_PACKAGE_MANAGER="${PYTHON_PACKAGE_MANAGER}"
 
 USER coder
 
-COPY --chown=coder:coder opt/cuspatial /opt/cuspatial
-
-RUN /bin/bash -c 'mkdir -m 0755 -p ~/.{aws,cache,conda,config,local}'
+RUN /bin/bash -c 'mkdir -m 0755 -p ~/.{aws,cache,conda,config/pip,local}'
 
 WORKDIR /home/coder/
 
+ENV PYTHONSAFEPATH="1"
+ENV PYTHONUNBUFFERED="1"
 ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"

diff --git a/.devcontainer/conda/devcontainer.json b/.devcontainer/conda/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -15,16 +15,19 @@
   },
 
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-single-envs}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}/single}"
   ],
+  "updateContentCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-conda-env"],
 
   "containerEnv": {
     "DEFAULT_CONDA_ENV": "rapids"
@@ -37,8 +40,7 @@
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-single-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.devcontainer/opt/${localWorkspaceFolderBasename},target=/opt/${localWorkspaceFolderBasename},type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}/single,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/.devcontainer/conda/isolated/.devcontainer/devcontainer.json b/.devcontainer/conda/isolated/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -15,7 +15,7 @@
   },
 
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
@@ -27,8 +27,10 @@
   ],
 
   "updateContentCommand": [
-    "/bin/bash", "-c", "cp -ar /workspaces/${localWorkspaceFolderBasename} /home/coder/${localWorkspaceFolderBasename}"
+    "/bin/bash", "-c", "cp -ar /workspaces/${localWorkspaceFolderBasename} /home/coder/${localWorkspaceFolderBasename} && rapids-make-vscode-workspace --update"
   ],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-conda-env"],
 
   "containerEnv": {
     "DEFAULT_CONDA_ENV": "rapids"

diff --git a/.devcontainer/conda/unified/.devcontainer/devcontainer.json b/.devcontainer/conda/unified/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -15,16 +15,19 @@
   },
 
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-unified-envs}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}/unified}"
   ],
+  "updateContentCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-conda-env"],
 
   "containerEnv": {
     "DEFAULT_CONDA_ENV": "rapids"
@@ -39,8 +42,7 @@
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-unified-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.devcontainer/opt/${localWorkspaceFolderBasename},target=/opt/${localWorkspaceFolderBasename},type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -15,16 +15,19 @@
   },
 
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-single-envs}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}/single}"
   ],
+  "updateContentCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-conda-env"],
 
   "containerEnv": {
     "DEFAULT_CONDA_ENV": "rapids"
@@ -37,8 +40,7 @@
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-single-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.devcontainer/opt/${localWorkspaceFolderBasename},target=/opt/${localWorkspaceFolderBasename},type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}/single,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/.devcontainer/opt/cuspatial/bin/post-attach-command.sh b/.devcontainer/opt/cuspatial/bin/post-attach-command.sh
diff --git a/.devcontainer/opt/cuspatial/bin/post-create-command.sh b/.devcontainer/opt/cuspatial/bin/post-create-command.sh
diff --git a/.devcontainer/opt/cuspatial/bin/update-content-command.sh b/.devcontainer/opt/cuspatial/bin/update-content-command.sh
diff --git a/.devcontainer/pip/devcontainer.json b/.devcontainer/pip/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -16,7 +16,7 @@
 
   "features": {
     "ghcr.io/devcontainers/features/python:1": {},
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
@@ -25,10 +25,15 @@
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,configlWorkspaceFolderBasename}-single-local}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/${localWorkspaceFolderBasename}/single}"
   ],
+  "updateContentCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-pip-env"],
 
   "containerEnv": {
+    "PYTHONSAFEPATH": "true",
+    "PYTHONUNBUFFERED": "true",
     "DEFAULT_VIRTUAL_ENV": "rapids"
   },
 
@@ -38,8 +43,7 @@
     "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.${localWorkspaceFolderBasename}-single-local,target=/home/coder/.local,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.devcontainer/opt/${localWorkspaceFolderBasename},target=/opt/${localWorkspaceFolderBasename},type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.local/${localWorkspaceFolderBasename}/single,target=/home/coder/.local,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/.devcontainer/pip/isolated/.devcontainer/devcontainer.json b/.devcontainer/pip/isolated/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -16,7 +16,7 @@
 
   "features": {
     "ghcr.io/devcontainers/features/python:1": {},
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
@@ -25,12 +25,14 @@
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,local}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip}"
   ],
 
   "updateContentCommand": [
-    "/bin/bash", "-c", "cp -ar /workspaces/${localWorkspaceFolderBasename} /home/coder/${localWorkspaceFolderBasename}"
+    "/bin/bash", "-c", "cp -ar /workspaces/${localWorkspaceFolderBasename} /home/coder/${localWorkspaceFolderBasename} && rapids-make-vscode-workspace --update"
   ],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-pip-env"],
 
   "containerEnv": {
     "DEFAULT_VIRTUAL_ENV": "rapids"
@@ -41,8 +43,7 @@
   "mounts": [
     "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.local,target=/home/coder/.local,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/.devcontainer/pip/unified/.devcontainer/devcontainer.json b/.devcontainer/pip/unified/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
 {
-  "shutdownAction": "none",
+  "shutdownAction": "stopContainer",
 
   "build": {
     "context": "${localWorkspaceFolder}/.devcontainer",
@@ -16,7 +16,7 @@
 
   "features": {
     "ghcr.io/devcontainers/features/python:1": {},
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:latest": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.6": {}
   },
 
   "overrideFeatureInstallOrder": [
@@ -25,8 +25,11 @@
   ],
 
   "initializeCommand": [
-    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,${localWorkspaceFolderBasename}-unified-local}"
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/${localWorkspaceFolderBasename}/unified}"
   ],
+  "updateContentCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postCreateCommand": ["rapids-make-vscode-workspace", "--update"],
+  "postAttachCommand": ["rapids-make-pip-env"],
 
   "containerEnv": {
     "DEFAULT_VIRTUAL_ENV": "rapids"
@@ -40,8 +43,7 @@
     "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.${localWorkspaceFolderBasename}-unified-local,target=/home/coder/.local,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.devcontainer/opt/${localWorkspaceFolderBasename},target=/opt/${localWorkspaceFolderBasename},type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.local/${localWorkspaceFolderBasename}/unified,target=/home/coder/.local,type=bind,consistency=consistent"
   ],
 
   "customizations": {

diff --git a/setup.cfg → .flake8 b/setup.cfg → .flake8
@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.
+# Copyright (c) 2019-2023, NVIDIA CORPORATION.
 
 [flake8]
 filename = *.py, *.pyx, *.pxd, *.pxi

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -66,3 +66,24 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cuspatial
+      package-dir: python/cuspatial
+      skbuild-configure-options: "-DCUSPATIAL_BUILD_WHEELS=ON"
+  wheel-publish:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cuspatial
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -19,6 +19,8 @@ jobs:
       - conda-python-tests
       - conda-notebook-tests
       - docs-build
+      - wheel-build
+      - wheel-tests
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06
   checks:
@@ -70,3 +72,23 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci:latest"
       run_script: "ci/build_docs.sh"
+  wheel-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
+    with:
+      build_type: pull-request
+      package-dir: python/cuspatial
+      package-name: cuspatial
+      skbuild-configure-options: "-DCUSPATIAL_BUILD_WHEELS=ON"
+  wheel-tests:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
+    with:
+      build_type: pull-request
+      package-name: cuspatial
+      test-smoketest: "python ./ci/wheel_smoke_test.py"
+      test-unittest: "python -m pytest -n 8 ./python/cuspatial/cuspatial/tests"
+      test-before-amd64: "apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev && python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'"
+      test-before-arm64: "apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev && python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'"
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -30,3 +30,15 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-tests:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      package-name: cuspatial
+      test-unittest: "python -m pytest -n 8 ./python/cuspatial/cuspatial/tests"
+      test-before-amd64: "apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev && python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'"
+      test-before-arm64: "apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev && python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'"