diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 35c8b8b2cb..7b826f0cb4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,3 +66,45 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-cpp: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_python.sh + wheel-publish-cpp: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libkvikio + package-type: cpp + wheel-publish-python: + needs: wheel-build-python + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: kvikio + package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 907ca4a3d1..3f84139578 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,9 @@ jobs: - conda-python-tests - docs-build - devcontainer + - wheel-cpp-build + - wheel-python-build + - wheel-python-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 checks: @@ -68,3 +71,24 @@ jobs: sccache -z; build-all --verbose; sccache -s; + wheel-cpp-build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: pull-request + script: ci/build_wheel_cpp.sh + wheel-python-build: + needs: wheel-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + build_type: pull-request + script: ci/build_wheel_python.sh + wheel-python-tests: + needs: wheel-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + with: + build_type: pull-request + script: ci/test_wheel.sh diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100755 index 0000000000..9893474da3 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libkvikio" +package_dir="python/libkvikio" + +source rapids-configure-sccache +source rapids-date-string + +rapids-generate-version > ./VERSION + +cd "${package_dir}" + +python -m pip install wheel +# libkvikio is a header-only C++ library with no Python code, so +# it is entirely platform-agnostic. We cannot use auditwheel for +# retagging since it has no extension modules, so we use `wheel` +# directly instead. +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m wheel tags --platform any dist/* --remove + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100755 index 0000000000..1fbe80800b --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="kvikio" +package_dir="python/kvikio" + +source rapids-configure-sccache +source rapids-date-string + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +rapids-generate-version > ./VERSION + +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 0000000000..94a31b04b6 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" + +python -m pip install "$(echo ${WHEELHOUSE}/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" + +python -m pytest ./python/kvikio/tests diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b1ac294d7a..956a5cfcee 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -95,7 +95,11 @@ add_library(kvikio::kvikio ALIAS kvikio) # Enable CUDA in KvikIO if(CUDAToolkit_FOUND) - target_link_libraries(kvikio INTERFACE $) + if(CUDA_STATIC_RUNTIME) + target_link_libraries(kvikio INTERFACE $) + else() + target_link_libraries(kvikio INTERFACE $) + endif() target_compile_definitions(kvikio INTERFACE $) else() message(WARNING "Building KvikIO without CUDA") @@ -167,7 +171,11 @@ if(NOT already_set_kvikio) find_package(CUDAToolkit QUIET) if(CUDAToolkit_FOUND) - target_link_libraries(kvikio::kvikio INTERFACE CUDA::toolkit) + if(CUDA_STATIC_RUNTIME) + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart_static) + else() + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart) + endif() target_compile_definitions(kvikio::kvikio INTERFACE KVIKIO_CUDA_FOUND) else() message(WARNING "Building KvikIO without CUDA") diff --git a/dependencies.yaml b/dependencies.yaml index 9f7a133d87..0ed8cae952 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,7 +6,9 @@ files: cuda: ["11.8", "12.2"] arch: [x86_64] includes: - - build + - build-universal + - build-cpp + - build-py-wrapper - checks - cuda - cuda_version @@ -44,14 +46,13 @@ files: table: build-system includes: - rapids_build_skbuild - py_rapids_build: + py_build_cpp_wheel: output: pyproject - pyproject_dir: python/kvikio + pyproject_dir: python/libkvikio extras: - table: tool.rapids-build-backend - key: requires + table: build-system includes: - - build + - rapids_build_skbuild py_run: output: pyproject pyproject_dir: python/kvikio @@ -60,6 +61,25 @@ files: includes: - depends_on_cupy - run + py_wheel_cpp: + output: pyproject + pyproject_dir: python/libkvikio + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build-universal + py_wheel_python: + output: pyproject + pyproject_dir: python/kvikio + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build-universal + - build-cpp + - build-py-wrapper + - build-use-libkvikio-wheel py_optional_test: output: pyproject pyproject_dir: python/kvikio @@ -74,13 +94,14 @@ channels: - conda-forge - nvidia dependencies: - build: + build-universal: common: - output_types: [conda, requirements, pyproject] packages: - cmake>=3.26.4 - - cython>=3.0.0 - ninja + build-cpp: + common: - output_types: conda packages: - c-compiler @@ -114,6 +135,26 @@ dependencies: cuda: "12.*" packages: - cuda-nvcc + build-use-libkvikio-wheel: + common: + - output_types: conda + packages: &libkvikio_packages + - libkvikio==24.8.*,>=0.0.0a0 + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - libkvikio-cu12==24.8.*,>=0.0.0a0 + - matrix: {cuda: "11.*"} + packages: + - libkvikio-cu11==24.8.*,>=0.0.0a0 + - {matrix: null, packages: *libkvikio_packages} + build-py-wrapper: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0 checks: common: - output_types: [conda, requirements] diff --git a/python/.gitattributes b/python/.gitattributes deleted file mode 100644 index 2656ae48af..0000000000 --- a/python/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -kvikio/_version.py export-subst diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index e93ab99016..9217645637 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -41,9 +41,10 @@ endif() find_package(CUDAToolkit REQUIRED) +set(cython_lib_dir kvikio) + if(NOT KvikIO_FOUND) add_subdirectory(../../cpp kvikio-cpp) - set(cython_lib_dir kvikio) install(TARGETS kvikio DESTINATION ${cython_lib_dir}) endif() @@ -51,4 +52,12 @@ include(rapids-cython-core) rapids_cython_init() add_subdirectory(cmake) + +# It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it +# here. +install_aliased_imported_targets( + TARGETS nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + ${cython_lib_dir}/_lib +) + add_subdirectory(kvikio/_lib) diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt index 4249773480..fa94bc3f8e 100644 --- a/python/kvikio/cmake/CMakeLists.txt +++ b/python/kvikio/cmake/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,3 +13,5 @@ # ============================================================================= include(thirdparty/get_nvcomp.cmake) +# Needed for install_aliased_imported_targets +include(thirdparty/WheelHelpers.cmake) diff --git a/python/kvikio/cmake/thirdparty/WheelHelpers.cmake b/python/kvikio/cmake/thirdparty/WheelHelpers.cmake new file mode 100644 index 0000000000..3abe98a064 --- /dev/null +++ b/python/kvikio/cmake/thirdparty/WheelHelpers.cmake @@ -0,0 +1,59 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +include_guard(GLOBAL) + +# Making libraries available inside wheels by installing the associated targets. +function(install_aliased_imported_targets) + list(APPEND CMAKE_MESSAGE_CONTEXT "install_aliased_imported_targets") + + set(options "") + set(one_value "DESTINATION") + set(multi_value "TARGETS") + cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) + + message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__DESTINATION}'") + + foreach(target IN LISTS __TARGETS) + + if(NOT TARGET ${target}) + message(VERBOSE "No target named ${target}") + continue() + endif() + + get_target_property(alias_target ${target} ALIASED_TARGET) + if(alias_target) + set(target ${alias_target}) + endif() + + get_target_property(is_imported ${target} IMPORTED) + if(NOT is_imported) + # If the target isn't imported, install it into the wheel + install(TARGETS ${target} DESTINATION ${__DESTINATION}) + message(VERBOSE "install(TARGETS ${target} DESTINATION ${__DESTINATION})") + else() + # If the target is imported, make sure it's global + get_target_property(type ${target} TYPE) + if(${type} STREQUAL "UNKNOWN_LIBRARY") + install(FILES $ DESTINATION ${__DESTINATION}) + message(VERBOSE "install(FILES $ DESTINATION ${__DESTINATION})") + else() + install(IMPORTED_RUNTIME_ARTIFACTS ${target} DESTINATION ${__DESTINATION}) + message( + VERBOSE + "install(IMPORTED_RUNTIME_ARTIFACTS $ DESTINATION ${__DESTINATION})" + ) + endif() + endif() + endforeach() +endfunction() diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 671413a275..f93685f2b3 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -109,10 +109,11 @@ nvcomp_batch = "kvikio.nvcomp_codec:NvCompBatchCodec" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" -dependencies-file = "../dependencies.yaml" +dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libkvikio==24.8.*,>=0.0.0a0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt new file mode 100644 index 0000000000..278e09f462 --- /dev/null +++ b/python/libkvikio/CMakeLists.txt @@ -0,0 +1,43 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../cpp/cmake/rapids_config.cmake) +include(rapids-cpm) +include(rapids-cuda) +rapids_cuda_init_architectures(libkvikio-python) +rapids_cpm_init() + +project( + libkvikio-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +# Check if kvikio is already available. If so, it's the user's responsibility to ensure that the +# CMake package is also available at build time of the Python kvikio package. +find_package(kvikio "${RAPIDS_VERSION}") + +if(kvikio_FOUND) + return() +endif() + +unset(kvikio_FOUND) + +set(KvikIO_BUILD_EXAMPLES OFF) +set(KvikIO_BUILD_TESTS OFF) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp kvikio-cpp) diff --git a/python/libkvikio/LICENSE b/python/libkvikio/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libkvikio/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libkvikio/README.md b/python/libkvikio/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libkvikio/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libkvikio/libkvikio/VERSION b/python/libkvikio/libkvikio/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libkvikio/libkvikio/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py new file mode 100644 index 0000000000..ed1426bd9a --- /dev/null +++ b/python/libkvikio/libkvikio/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libkvikio._version import __git_commit__, __version__ + +__all__ = [__git_commit__, __version__] diff --git a/python/libkvikio/libkvikio/_version.py b/python/libkvikio/libkvikio/_version.py new file mode 100644 index 0000000000..8b67326806 --- /dev/null +++ b/python/libkvikio/libkvikio/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__version__", "__git_commit__"] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml new file mode 100644 index 0000000000..628e44f1b8 --- /dev/null +++ b/python/libkvikio/pyproject.toml @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libkvikio" +dynamic = ["version"] +description = "KvikIO - GPUDirect Storage (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/kvikio" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libkvikio"] +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libkvikio/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +requires = [ + "cmake>=3.26.4", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.