add wheel output

.github/workflows/build.yaml

@@ -66,3 +66,45 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-cpp:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_cpp.sh
+  wheel-build-python:
+    needs: wheel-build-cpp
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_python.sh
+  wheel-publish-cpp:
+    needs: wheel-build-cpp
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libkvikio
+      package-type: cpp
+  wheel-publish-python:
+    needs: wheel-build-python
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: kvikio
+      package-type: python

.github/workflows/pr.yaml

@@ -19,6 +19,9 @@ jobs:
       - conda-python-tests
       - docs-build
       - devcontainer
+      - wheel-cpp-build
+      - wheel-python-build
+      - wheel-python-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   checks:
@@ -64,7 +67,31 @@ jobs:
     with:
       arch: '["amd64"]'
       cuda: '["12.2"]'
+      # HACK: the yq and rapids-generate-script here is a stopgap until both of these PRs are merged:
+      #    - https://github.com/rapidsai/kvikio/pull/369
+      #    - https://github.com/rapidsai/devcontainers/pull/283
       build_command: |
         sccache -z;
         build-all --verbose;
         sccache -s;
+  wheel-cpp-build:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      script: ci/build_wheel_cpp.sh
+  wheel-python-build:
+    needs: wheel-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_python.sh
+  wheel-python-tests:
+    needs: wheel-python-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/test_wheel.sh

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     hooks:
       - id: black
         files: python/.*
-        args: ["--config", "python/pyproject.toml"]
+        args: ["--config", "python/kvikio/pyproject.toml"]
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
@@ -37,11 +37,11 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies: [types-cachetools]
-        args: ["--config-file=python/pyproject.toml",
-                "python/kvikio",
-                "python/tests",
-                "python/examples",
-                "python/benchmarks"]
+        args: ["--config-file=python/kvikio/pyproject.toml",
+                "python/kvikio/kvikio",
+                "python/kvikio/tests",
+                "python/kvikio/examples",
+                "python/kvikio/benchmarks"]
         pass_filenames: false
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v16.0.6

build.sh

@@ -146,7 +146,7 @@ fi
 # Build and install the kvikio Python package
 if (( NUMARGS == 0 )) || hasArg kvikio; then
     echo "building kvikio..."
-    cd ${REPODIR}/python
+    cd ${REPODIR}/python/kvikio
     SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBKVIKIO_BUILD_DIR};${EXTRA_CMAKE_ARGS}" \
         python -m pip install --no-build-isolation --no-deps .
 fi

ci/build_wheel_cpp.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="libkvikio"
+package_dir="python/libkvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
+fi
+
+cd "${package_dir}"
+
+python -m pip install wheel
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+python -m wheel tags --platform any dist/* --remove
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist

ci/build_wheel_python.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="kvikio"
+package_dir="python/kvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+alpha_spec=''
+if ! rapids-is-release-build; then
+    alpha_spec=',>=0.0.0a0'
+fi
+sed -r -i "s/libkvikio==(.*)\"/libkvikio${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
+
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
+    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
+fi
+
+CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist)
+
+cd "${package_dir}"
+
+python -m pip install wheel
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE}
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

ci/run_pytests.sh

@@ -4,6 +4,6 @@
 set -euo pipefail
 
 # Support invoking run_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/kvikio
 
 pytest --cache-clear --verbose "$@" tests

ci/test_wheel.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -eou pipefail
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+WHEELHOUSE="${PWD}/dist/"
+RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}"
+
+python -m pip install "kvikio-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}"
+
+python -m pytest ./python/kvikio/tests

conda/recipes/kvikio/meta.yaml

@@ -33,7 +33,7 @@ build:
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
   script:
-    - cd python
+    - cd python/kvikio
     - python -m pip install . -vv
   ignore_run_exports_from:
     {% if cuda_major == "11" %}

dependencies.yaml

@@ -6,7 +6,10 @@ files:
       cuda: ["11.8", "12.2"]
       arch: [x86_64]
     includes:
-      - build
+      - build-universal
+      - build-cpp
+      - build-py
+      - build-py-wrapper
       - checks
       - cuda
       - cuda_version
@@ -38,22 +41,44 @@ files:
       - py_version
   py_build:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: build-system
     includes:
-      - build
+      - build-universal
+      - build-cpp
+      - build-py
+      - build-py-wrapper
   py_run:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: project
     includes:
       - depends_on_cupy
       - run
+  py_wheel_cpp:
+    output: pyproject
+    pyproject_dir: python/libkvikio
+    extras:
+      table: build-system
+    includes:
+      - build-universal
+      - build-py
+  py_wheel_python:
+    output: pyproject
+    pyproject_dir: python/kvikio
+    extras:
+      table: build-system
+    includes:
+      - build-universal
+      - build-cpp
+      - build-py
+      - build-py-wrapper
+      - build-use-libkvikio-wheel
   py_optional_test:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: project.optional-dependencies
       key: test
@@ -65,21 +90,18 @@ channels:
   - conda-forge
   - nvidia
 dependencies:
-  build:
+  build-universal:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.26.4
-          - cython>=3.0.0
           - ninja
+  build-cpp:
+    common:
       - output_types: conda
         packages:
           - c-compiler
           - cxx-compiler
-          - scikit-build-core>=0.7.0
-      - output_types: [requirements, pyproject]
-        packages:
-          - scikit-build-core[pyproject]>=0.7.0
     specific:
       - output_types: conda
         matrices:
@@ -109,6 +131,24 @@ dependencies:
               cuda: "12.*"
             packages:
               - cuda-nvcc
+  build-py:
+    common:
+      - output_types: conda
+        packages:
+          - scikit-build-core>=0.7.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
+  build-use-libkvikio-wheel:
+    common:
+      - output_types: [pyproject]
+        packages:
+          - libkvikio==24.6.*
+  build-py-wrapper:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - cython>=3.0.0
   checks:
     common:
       - output_types: [conda, requirements]

docs/source/zarr.rst

@@ -11,5 +11,5 @@ The following is an example of how to use the convenience function :py:meth:`kvi
 to create a new Zarr array and how open an existing Zarr array.
 
 
-.. literalinclude:: ../../python/examples/zarr_cupy_nvcomp.py
+.. literalinclude:: ../../python/kvikio/examples/zarr_cupy_nvcomp.py
     :language: python