add wheel output (#369)

closes #250 I'm not sure if it makes sense to split the wheel into one dedicated wheel for the shared library, and another for the python wrapper. That's what @vyasr has been doing on other libraries, like rapidsai/rmm#1529. If it makes sense here, I'll add that. Authors: - Mike Sarahan (https://github.com/msarahan) - Philip Hyunsu Cho (https://github.com/hcho3) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) - Mads R. B. Kristensen (https://github.com/madsbk) - James Lamb (https://github.com/jameslamb) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: #369
rapidsai · Jun 19, 2024 · 3cc6678 · 3cc6678
1 parent 88033f4
commit 3cc6678
Show file tree

Hide file tree

Showing 19 changed files with 409 additions and 14 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -66,3 +66,45 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-cpp:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_cpp.sh
+  wheel-build-python:
+    needs: wheel-build-cpp
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_python.sh
+  wheel-publish-cpp:
+    needs: wheel-build-cpp
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libkvikio
+      package-type: cpp
+  wheel-publish-python:
+    needs: wheel-build-python
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: kvikio
+      package-type: python
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -19,6 +19,9 @@ jobs:
       - conda-python-tests
       - docs-build
       - devcontainer
+      - wheel-cpp-build
+      - wheel-python-build
+      - wheel-python-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   checks:
@@ -68,3 +71,24 @@ jobs:
         sccache -z;
         build-all --verbose;
         sccache -s;
+  wheel-cpp-build:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      script: ci/build_wheel_cpp.sh
+  wheel-python-build:
+    needs: wheel-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_python.sh
+  wheel-python-tests:
+    needs: wheel-python-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/test_wheel.sh
diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="libkvikio"
+package_dir="python/libkvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+rapids-generate-version > ./VERSION
+
+cd "${package_dir}"
+
+python -m pip install wheel
+# libkvikio is a header-only C++ library with no Python code, so
+# it is entirely platform-agnostic. We cannot use auditwheel for
+# retagging since it has no extension modules, so we use `wheel`
+# directly instead.
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+python -m wheel tags --platform any dist/* --remove
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist
diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="kvikio"
+package_dir="python/kvikio"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+rapids-generate-version > ./VERSION
+
+CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist)
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE}
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -eou pipefail
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+WHEELHOUSE="${PWD}/dist/"
+RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}"
+
+python -m pip install "$(echo ${WHEELHOUSE}/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+
+python -m pytest ./python/kvikio/tests
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -95,7 +95,11 @@ add_library(kvikio::kvikio ALIAS kvikio)
 
 # Enable CUDA in KvikIO
 if(CUDAToolkit_FOUND)
-  target_link_libraries(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:CUDA::toolkit>)
+  if(CUDA_STATIC_RUNTIME)
+    target_link_libraries(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:CUDA::cudart_static>)
+  else()
+    target_link_libraries(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:CUDA::cudart>)
+  endif()
   target_compile_definitions(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:KVIKIO_CUDA_FOUND>)
 else()
   message(WARNING "Building KvikIO without CUDA")
@@ -167,7 +171,11 @@ if(NOT already_set_kvikio)
 
   find_package(CUDAToolkit QUIET)
   if(CUDAToolkit_FOUND)
-    target_link_libraries(kvikio::kvikio INTERFACE CUDA::toolkit)
+    if(CUDA_STATIC_RUNTIME)
+      target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart_static)
+    else()
+      target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart)
+    endif()
     target_compile_definitions(kvikio::kvikio INTERFACE KVIKIO_CUDA_FOUND)
   else()
     message(WARNING "Building KvikIO without CUDA")

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -6,7 +6,9 @@ files:
       cuda: ["11.8", "12.2"]
       arch: [x86_64]
     includes:
-      - build
+      - build-universal
+      - build-cpp
+      - build-py-wrapper
       - checks
       - cuda
       - cuda_version
@@ -44,14 +46,13 @@ files:
       table: build-system
     includes:
       - rapids_build_skbuild
-  py_rapids_build:
+  py_build_cpp_wheel:
     output: pyproject
-    pyproject_dir: python/kvikio
+    pyproject_dir: python/libkvikio
     extras:
-      table: tool.rapids-build-backend
-      key: requires
+      table: build-system
     includes:
-      - build
+      - rapids_build_skbuild
   py_run:
     output: pyproject
     pyproject_dir: python/kvikio
@@ -60,6 +61,25 @@ files:
     includes:
       - depends_on_cupy
       - run
+  py_wheel_cpp:
+    output: pyproject
+    pyproject_dir: python/libkvikio
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build-universal
+  py_wheel_python:
+    output: pyproject
+    pyproject_dir: python/kvikio
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build-universal
+      - build-cpp
+      - build-py-wrapper
+      - build-use-libkvikio-wheel
   py_optional_test:
     output: pyproject
     pyproject_dir: python/kvikio
@@ -74,13 +94,14 @@ channels:
   - conda-forge
   - nvidia
 dependencies:
-  build:
+  build-universal:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cmake>=3.26.4
-          - cython>=3.0.0
           - ninja
+  build-cpp:
+    common:
       - output_types: conda
         packages:
           - c-compiler
@@ -114,6 +135,26 @@ dependencies:
               cuda: "12.*"
             packages:
               - cuda-nvcc
+  build-use-libkvikio-wheel:
+    common:
+      - output_types: conda
+        packages: &libkvikio_packages
+          - libkvikio==24.8.*,>=0.0.0a0
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - libkvikio-cu12==24.8.*,>=0.0.0a0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - libkvikio-cu11==24.8.*,>=0.0.0a0
+          - {matrix: null, packages: *libkvikio_packages}
+  build-py-wrapper:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - cython>=3.0.0
   checks:
     common:
       - output_types: [conda, requirements]

diff --git a/python/.gitattributes b/python/.gitattributes
diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt
@@ -41,14 +41,23 @@ endif()
 
 find_package(CUDAToolkit REQUIRED)
 
+set(cython_lib_dir kvikio)
+
 if(NOT KvikIO_FOUND)
   add_subdirectory(../../cpp kvikio-cpp)
-  set(cython_lib_dir kvikio)
   install(TARGETS kvikio DESTINATION ${cython_lib_dir})
 endif()
 
 include(rapids-cython-core)
 rapids_cython_init()
 
 add_subdirectory(cmake)
+
+# It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it
+# here.
+install_aliased_imported_targets(
+  TARGETS nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION
+  ${cython_lib_dir}/_lib
+)
+
 add_subdirectory(kvikio/_lib)
diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -13,3 +13,5 @@
 # =============================================================================
 
 include(thirdparty/get_nvcomp.cmake)
+# Needed for install_aliased_imported_targets
+include(thirdparty/WheelHelpers.cmake)
diff --git a/python/kvikio/cmake/thirdparty/WheelHelpers.cmake b/python/kvikio/cmake/thirdparty/WheelHelpers.cmake
@@ -0,0 +1,59 @@
+# =============================================================================
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+include_guard(GLOBAL)
+
+# Making libraries available inside wheels by installing the associated targets.
+function(install_aliased_imported_targets)
+  list(APPEND CMAKE_MESSAGE_CONTEXT "install_aliased_imported_targets")
+
+  set(options "")
+  set(one_value "DESTINATION")
+  set(multi_value "TARGETS")
+  cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN})
+
+  message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__DESTINATION}'")
+
+  foreach(target IN LISTS __TARGETS)
+
+    if(NOT TARGET ${target})
+      message(VERBOSE "No target named ${target}")
+      continue()
+    endif()
+
+    get_target_property(alias_target ${target} ALIASED_TARGET)
+    if(alias_target)
+      set(target ${alias_target})
+    endif()
+
+    get_target_property(is_imported ${target} IMPORTED)
+    if(NOT is_imported)
+      # If the target isn't imported, install it into the wheel
+      install(TARGETS ${target} DESTINATION ${__DESTINATION})
+      message(VERBOSE "install(TARGETS ${target} DESTINATION ${__DESTINATION})")
+    else()
+      # If the target is imported, make sure it's global
+      get_target_property(type ${target} TYPE)
+      if(${type} STREQUAL "UNKNOWN_LIBRARY")
+        install(FILES $<TARGET_FILE:${target}> DESTINATION ${__DESTINATION})
+        message(VERBOSE "install(FILES $<TARGET_FILE:${target}> DESTINATION ${__DESTINATION})")
+      else()
+        install(IMPORTED_RUNTIME_ARTIFACTS ${target} DESTINATION ${__DESTINATION})
+        message(
+          VERBOSE
+          "install(IMPORTED_RUNTIME_ARTIFACTS $<TARGET_FILE:${target}> DESTINATION ${__DESTINATION})"
+        )
+      endif()
+    endif()
+  endforeach()
+endfunction()
diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml
@@ -109,10 +109,11 @@ nvcomp_batch = "kvikio.nvcomp_codec:NvCompBatchCodec"
 
 [tool.rapids-build-backend]
 build-backend = "scikit_build_core.build"
-dependencies-file = "../dependencies.yaml"
+dependencies-file = "../../dependencies.yaml"
 requires = [
     "cmake>=3.26.4",
     "cython>=3.0.0",
+    "libkvikio==24.8.*,>=0.0.0a0",
     "ninja",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.