Merge branch 'branch-24.10' into fea/get_current_device_resource_ref

.github/workflows/build.yaml

@@ -66,7 +66,30 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-libcuspatial:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcuspatial.sh
+  wheel-publish-libcuspatial:
+    needs: wheel-build-libcuspatial
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcuspatial
+      package-type: cpp
   wheel-build-cuspatial:
+    needs: wheel-publish-libcuspatial
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -85,6 +108,7 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cuspatial
+      package-type: python
   wheel-build-cuproj:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -104,3 +128,4 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cuproj
+      package-type: python

.github/workflows/pr.yaml

@@ -21,6 +21,7 @@ jobs:
       - conda-notebook-tests
       - docs-build
       - wheel-build-cuspatial
+      - wheel-build-libcuspatial
       - wheel-tests-cuspatial
       - wheel-build-cuproj
       - wheel-tests-cuproj
@@ -83,10 +84,19 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build-cuspatial:
+  wheel-build-libcuspatial:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      script: ci/build_wheel_libcuspatial.sh
+  wheel-build-cuspatial:
+    needs: [checks, wheel-build-libcuspatial]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_cuspatial.sh

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
               args: ["--config pyproject.toml"]
               additional_dependencies: ["tomli"]
       - repo: https://github.com/rapidsai/pre-commit-hooks
-        rev: v0.3.1
+        rev: v0.4.0
         hooks:
             - id: verify-copyright
             - id: verify-alpha-spec

README.md

@@ -164,6 +164,13 @@ There are two ways to fix this:
     sudo -y add-apt-repository ppa:ubuntugis/ppa
     sudo apt install libgdal-dev
     ```
+  * Rocky Linux users can install a newer GDAL as follows
+    ```shell
+    yum update -y
+    yum config-manager --set-enabled powertools
+    yum update -y
+    yum install -y gdal-devel
+    ```
 2. Pin fiona's version to a range that's compatible with your version of `libgdal-dev`
   * For Ubuntu20.04 ([GDAL v3.0.4](https://packages.ubuntu.com/focal/libgdal-dev)):
     ```shell

build.sh

@@ -124,11 +124,6 @@ if hasArg benchmarks; then
     BUILD_BENCHMARKS=ON
 fi
 
-# Append `-DFIND_CUSPATIAL_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
-if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUSPATIAL_CPP"* ]]; then
-    EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUSPATIAL_CPP=ON"
-fi
-
 # Append `-DFIND_CUPROJ_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
 if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUPROJ_CPP"* ]]; then
     EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUPROJ_CPP=ON"

ci/build_wheel.sh

@@ -5,6 +5,26 @@ set -euo pipefail
 
 package_name=$1
 package_dir=$2
+package_type=$3
+
+# The 'libcuspatial' wheel should package 'libcuspatial.so', and all others
+# should exclude it (they dynamically load it if they need it).
+#
+# Capturing that here in argument-parsing to allow this build_wheel.sh
+# script to be re-used by all wheel builds in the project.
+case "${package_dir}" in
+  python/libcuspatial)
+    EXCLUDE_ARGS=(
+      --exclude "libcudf.so"
+    )
+  ;;
+  *)
+    EXCLUDE_ARGS=(
+      --exclude "libcudf.so"
+      --exclude "libcuspatial.so"
+    )
+  ;;
+esac
 
 source rapids-configure-sccache
 source rapids-date-string
@@ -18,6 +38,9 @@ cd "${package_dir}"
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
 
 mkdir -p final_dist
-python -m auditwheel repair -w final_dist dist/*
+python -m auditwheel repair \
+    "${EXCLUDE_ARGS[@]}" \
+    -w final_dist \
+    dist/*
 
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist

ci/build_wheel_cuproj.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-ci/build_wheel.sh cuproj python/cuproj
+ci/build_wheel.sh cuproj python/cuproj python

ci/build_wheel_cuspatial.sh

@@ -1,8 +1,17 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON"
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-ci/build_wheel.sh cuspatial python/cuspatial
+# Downloads libcuspatial wheel from this current build,
+# then ensures 'cuspatial' wheel builds always use the 'libcuspatial' just built in the same CI run.
+#
+# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# are used when creating the isolated build environment.
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuspatial_dist
+echo "libcuspatial-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcuspatial_dist/libcuspatial_*.whl)" > /tmp/constraints.txt
+export PIP_CONSTRAINT="/tmp/constraints.txt"
+
+ci/build_wheel.sh cuspatial python/cuspatial python

ci/build_wheel_libcuspatial.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+ci/build_wheel.sh libcuspatial python/libcuspatial cpp

ci/test_wheel_cuproj.sh

@@ -5,19 +5,31 @@ set -eou pipefail
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-# Install additional dependencies
-apt update
-DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
-python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
+# install build dependencies for fiona
+if type -f yum > /dev/null 2>&1; then
+  yum update -y
+  # some of gdal-devel's dependencies, like 'libdap', come from the powertools repo
+  yum config-manager --set-enabled powertools
+  yum update -y
+  yum install -y gdal-devel
+else
+  apt update
+  DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
+fi
 
-# Download the cuspatial built in the previous step
-RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cuspatial-dep
-python -m pip install --no-deps ./local-cuspatial-dep/cuspatial*.whl
+# Download the cuproj and cuspatial built in the previous step
+RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/cuproj*.whl)[test]
+python -m pip install \
+  --no-binary 'fiona' \
+  "$(echo ./dist/cuspatial*.whl)" \
+  "$(echo ./dist/cuproj*.whl)[test]" \
+  "$(echo ./dist/libcuspatial*.whl)" \
+  'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuproj"
 pushd python/cuproj/cuproj

ci/test_wheel_cuspatial.sh

@@ -5,15 +5,29 @@ set -eou pipefail
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-# Install additional dependencies
-apt update
-DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
-python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9'
+# install build dependencies for fiona
+if type -f yum > /dev/null 2>&1; then
+  yum update -y
+  # some of gdal-devel's dependencies, like 'libdap', come from the powertools repo
+  yum config-manager --set-enabled powertools
+  yum update -y
+  yum install -y gdal-devel
+else
+  apt update
+  DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
+fi
+
+# Download the cuspatial and libcuspatial built in the previous step
+RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
-python -m pip install $(echo ./dist/cuspatial*.whl)[test]
+python -m pip install \
+  --no-binary 'fiona' \
+  "$(echo ./dist/cuspatial*.whl)[test]" \
+  "$(echo ./dist/libcuspatial*.whl)" \
+  'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuspatial"
 pushd python/cuspatial/cuspatial

conda/recipes/cuproj/conda_build_config.yaml

@@ -18,7 +18,3 @@ c_stdlib_version:
 
 cmake_version:
   - ">=3.26.4,!=3.30.0"
-
-# Workaround until proj 9.3.1 migration completes
-proj:
-  - "9.3.0"

conda/recipes/cuproj/meta.yaml

@@ -62,7 +62,7 @@ requirements:
     - cython >=3.0.0
     - python
     - rapids-build-backend >=0.3.0,<0.4.0.dev0
-    - rmm ={{ minor_version }}
+    - librmm ={{ minor_version }}
     - scikit-build-core >=0.10.0
     - proj
     - sqlite
@@ -74,17 +74,11 @@ requirements:
     {% endif %}
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
     - python
-    - rmm ={{ minor_version }}
     - cupy>=12.0.0
 
 test:            # [linux64]
   imports:       # [linux64]
     - cuproj     # [linux64]
-  requires:
-    - cupy>=12.0.0
-    - cuspatial ={{ minor_version }}
-    - rmm ={{ minor_version }}
-
 
 about:
   home: https://rapids.ai/

cpp/cmake/thirdparty/get_ranger.cmake

@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,8 +26,6 @@ function(find_and_configure_ranger)
     rapids_cpm_find(
       ranger 00.01.00
       GLOBAL_TARGETS "${global_targets}"
-      BUILD_EXPORT_SET cuspatial-exports
-      INSTALL_EXPORT_SET cuspatial-exports
       CPM_ARGS
       GIT_REPOSITORY https://github.com/harrism/ranger.git
       GIT_TAG main