add wheel output

.github/workflows/pr.yaml

@@ -19,6 +19,8 @@ jobs:
       - conda-python-tests
       - docs-build
       - devcontainer
+      - wheel-cpp-build
+      - wheel-python-build
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   checks:
@@ -64,7 +66,33 @@ jobs:
     with:
       arch: '["amd64"]'
       cuda: '["12.2"]'
+      # HACK: the yq and rapids-generate-script here is a stopgap until both of these PRs are merged:
+      #    - https://github.com/rapidsai/kvikio/pull/369
+      #    - https://github.com/rapidsai/devcontainers/pull/283
       build_command: |
+        sudo yq -i '.repos[2].python[0].sub_dir = "python/kvikio"' /opt/rapids-build-utils/manifest.yaml;
+        rapids-generate-scripts;
         sccache -z;
         build-all --verbose;
         sccache -s;
+  wheel-cpp-build:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      script: ci/build_wheel_cpp.sh
+  wheel-python-build:
+    needs: wheel-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_python.sh
+  wheel-python-tests:
+    needs: wheel-python-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/run_pytests.sh

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     hooks:
       - id: black
         files: python/.*
-        args: ["--config", "python/pyproject.toml"]
+        args: ["--config", "python/kvikio/pyproject.toml"]
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
@@ -37,11 +37,11 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies: [types-cachetools]
-        args: ["--config-file=python/pyproject.toml",
-                "python/kvikio",
-                "python/tests",
-                "python/examples",
-                "python/benchmarks"]
+        args: ["--config-file=python/kvikio/pyproject.toml",
+                "python/kvikio/kvikio",
+                "python/kvikio/tests",
+                "python/kvikio/examples",
+                "python/kvikio/benchmarks"]
         pass_filenames: false
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v16.0.6

build.sh

@@ -146,7 +146,7 @@ fi
 # Build and install the kvikio Python package
 if (( NUMARGS == 0 )) || hasArg kvikio; then
     echo "building kvikio..."
-    cd ${REPODIR}/python
+    cd ${REPODIR}/python/kvikio
     SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBKVIKIO_BUILD_DIR};${EXTRA_CMAKE_ARGS}" \
         python -m pip install --no-build-isolation --no-deps .
 fi

ci/build_wheel_cpp.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="libkvikio"
+package_dir="python/libkvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
+fi
+
+librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp)
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}"
+python -m wheel tags --platform any dist/* --remove
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist

ci/build_wheel_python.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="kvikio"
+package_dir="python/kvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
+fi
+
+CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist)
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE}
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

conda/recipes/kvikio/meta.yaml

@@ -33,7 +33,7 @@ build:
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
   script:
-    - cd python
+    - cd python/kvikio
     - python -m pip install . -vv
   ignore_run_exports_from:
     {% if cuda_major == "11" %}

dependencies.yaml

@@ -38,22 +38,37 @@ files:
       - py_version
   py_build:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: build-system
     includes:
       - build
   py_run:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: project
     includes:
       - depends_on_cupy
       - run
+  py_wheel_cpp:
+    output: pyproject
+    pyproject_dir: python/libkvikio
+    extras:
+      table: project
+    includes:
+      - build
+  py_wheel_python:
+    output: pyproject
+    pyproject_dir: python/kvikio
+    extras:
+      table: project
+    includes:
+      - build
+      - wheel_lib_dep
   py_optional_test:
     output: pyproject
-    pyproject_dir: python
+    pyproject_dir: python/kvikio
     extras:
       table: project.optional-dependencies
       key: test
@@ -208,6 +223,11 @@ dependencies:
             packages: &cupy_packages_cu11
               - cupy-cuda11x>=12.0.0
           - {matrix: null, packages: *cupy_packages_cu11}
+  wheel_lib_dep:
+    common:
+      - output_types: pyproject
+        packages:
+          - libkvikio==24.6.*
   docs:
     common:
       - output_types: [conda, requirements]

python/CMakeLists.txt → python/kvikio/CMakeLists.txt

@@ -14,7 +14,7 @@
 
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 
-include(../cpp/cmake/rapids_config.cmake)
+include(../../cpp/cmake/rapids_config.cmake)
 include(rapids-cpm)
 rapids_cpm_init()
 include(rapids-cuda)
@@ -26,18 +26,7 @@ project(
   LANGUAGES CXX CUDA
 )
 
-option(FIND_KVIKIO_CPP
-       "Search for existing KVIKIO C++ installations before defaulting to local files" OFF
-)
-
-# TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile
-include(../cpp/cmake/Modules/FindcuFile.cmake)
-
-if(FIND_KVIKIO_CPP)
-  find_package(KvikIO "${RAPIDS_VERSION}")
-else()
-  set(KvikIO_FOUND OFF)
-endif()
+find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED)
 
 find_package(CUDAToolkit REQUIRED)
 

python/kvikio/README.md

@@ -0,0 +1 @@
+../../README.md

python/kvikio/kvikio/VERSION

@@ -0,0 +1 @@
+../../../VERSION

python/pyproject.toml → python/kvikio/pyproject.toml

@@ -8,10 +8,10 @@ requires = [
     "cython>=3.0.0",
     "ninja",
     "scikit-build-core[pyproject]>=0.7.0",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
-name = "kvikio"
+name = "kvikio-cu11"
 dynamic = ["version"]
 description = "KvikIO - GPUDirect Storage"
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -21,12 +21,12 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.9"
 dependencies = [
-    "cupy-cuda11x>=12.0.0",
-    "numcodecs <0.12.0",
-    "numpy>=1.23,<2.0a0",
-    "packaging",
-    "zarr",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+    "cmake>=3.26.4",
+    "cython>=3.0.0",
+    "libkvikio==24.6.*",
+    "ninja",
+    "scikit-build-core[pyproject]>=0.7.0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
     "Topic :: Database",
@@ -44,7 +44,7 @@ test = [
     "dask>=2022.05.2",
     "pytest",
     "pytest-cov",
-] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/kvikio"

python/libkvikio/CMakeLists.txt

@@ -0,0 +1,36 @@
+# =============================================================================
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
+
+include(../../cpp/cmake/rapids_config.cmake)
+include(rapids-cpm)
+rapids_cpm_init()
+include(rapids-cuda)
+rapids_cuda_init_architectures(libkvikio-python)
+
+# TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile
+include(../../cpp/cmake/Modules/FindcuFile.cmake)
+
+project(
+  libkvikio-python
+  VERSION "${RAPIDS_VERSION}"
+  LANGUAGES CXX
+)
+
+set(KvikIO_BUILD_EXAMPLES OFF)
+set(KvikIO_BUILD_TESTS OFF)
+set(CUDA_STATIC_RUNTIME ON)
+
+add_subdirectory(../../cpp kvikio-cpp)

python/libkvikio/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/libkvikio/README.md

@@ -0,0 +1 @@
+../../README.md

python/libkvikio/libkvikio/VERSION

@@ -0,0 +1 @@
+../../../VERSION

python/libkvikio/libkvikio/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from libkvikio._version import __git_commit__, __version__  # noqa: F401

python/libkvikio/libkvikio/_version.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+
+__version__ = (
+    importlib.resources.files("libkvikio").joinpath("VERSION").read_text().strip()
+)
+__git_commit__ = ""