diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5f80d8cfda..af6cb910e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,30 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libraft: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libraft.sh + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-publish-libraft: + needs: wheel-build-libraft + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libraft + package-type: cpp wheel-build-pylibraft: + needs: wheel-publish-libraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: @@ -85,6 +108,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibraft + package-type: python wheel-build-raft-dask: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 @@ -104,3 +128,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: raft_dask + package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a270df1dfa..dbb40d1cb0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,11 +21,12 @@ jobs: - conda-python-build - conda-python-tests - docs-build + - wheel-build-libraft - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask - - devcontainer + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() @@ -116,10 +117,22 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibraft: + wheel-build-libraft: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: pull-request + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libraft.sh + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-build-pylibraft: + needs: [checks, wheel-build-libraft] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh @@ -132,7 +145,7 @@ jobs: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: - needs: wheel-tests-pylibraft + needs: [checks, wheel-build-libraft] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: @@ -146,13 +159,13 @@ jobs: with: build_type: pull-request script: ci/test_wheel_raft_dask.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON --verbose; - sccache -s; + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_PRIMS_BENCH=ON --verbose; + # sccache -s; diff --git a/build.sh b/build.sh index a95cb8ee23..de3ebfa3c5 100755 --- a/build.sh +++ b/build.sh @@ -347,13 +347,8 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" -fi # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS -SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') +SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${EXTRA_CMAKE_ARGS} | sed 's/ /;/g') # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 326ee9a4c7..4c295c416e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 underscore_package_name=$(echo "${package_name}" | tr "-" "_") # Clear out system ucx files to ensure that we're getting ucx from the wheel. @@ -39,6 +40,12 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac +if [[ ${package_name} != "libraft" ]]; then + EXCLUDE_ARGS+=( + --exclude "libraft.so" + ) +fi + sccache --zero-stats rapids-logger "Building '${package_name}' wheel" @@ -55,4 +62,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_type} final_dist diff --git a/ci/build_wheel_libraft.sh b/ci/build_wheel_libraft.sh new file mode 100755 index 0000000000..27c7270a1c --- /dev/null +++ b/ci/build_wheel_libraft.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libraft" +package_dir="python/libraft" + +rapids-logger "Generating build requirements" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "${matrix_selectors}" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF${EXTRA_CMAKE_ARGS}" + +ci/build_wheel.sh libraft ${package_dir} cpp +ci/validate_wheel.sh ${package_dir} final_dist libraft diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index dd62ab5399..f88aaaa6e1 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -5,17 +5,19 @@ set -euo pipefail package_dir="python/pylibraft" -case "${RAPIDS_CUDA_VERSION}" in - 12.*) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" - ;; - 11.*) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" - ;; -esac +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Downloads libraft wheels from this current build, +# then ensures 'pylibraft' wheel builds always use the 'libraft' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist +echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF" -ci/build_wheel.sh pylibraft ${package_dir} +ci/build_wheel.sh pylibraft ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist pylibraft diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index d49d131abf..79334c9f45 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -5,8 +5,19 @@ set -euo pipefail package_dir="python/raft-dask" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Downloads libraft wheels from this current build, +# then ensures 'raft-dask' wheel builds always use the 'libraft' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist +echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF" -ci/build_wheel.sh raft-dask ${package_dir} +ci/build_wheel.sh raft-dask ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist raft-dask diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index b38f5a690b..1e0b34d609 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,9 +5,13 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libraft-dep +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist + # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] +python -m pip install \ + ./local-libraft-dep/libraft*.whl \ + "$(echo ./dist/pylibraft*.whl)[test]" python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a778a3ec51..011de4d409 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -5,13 +5,13 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist - -# Download the pylibraft built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libraft-dep +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-pylibraft-dep +RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install -v \ + ./local-libraft-dep/libraft*.whl \ ./local-pylibraft-dep/pylibraft*.whl \ "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5ef72ad895..ca506af004 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,23 +10,17 @@ package_name=$3 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 -if [[ "${package_name}" == "raft-dask" ]]; then - PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '200M' - ) -elif [[ "${package_name}" == "pylibraft" ]]; then +PYDISTCHECK_ARGS=() +if [[ "${package_name}" == "libraft" ]]; then if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then - PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '600M' + PYDISTCHECK_ARGS+=( + --max-allowed-size-compressed '750M' ) else - PYDISTCHECK_ARGS=( + PYDISTCHECK_ARGS+=( --max-allowed-size-compressed '100M' ) fi -else - echo "Unsupported package name: ${package_name}" - exit 1 fi cd "${package_dir}" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 621f9fcef2..eb7e8540f0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -65,9 +65,12 @@ set(RAFT_COMPILE_LIBRARY_DEFAULT OFF) if(BUILD_TESTS OR BUILD_PRIMS_BENCH) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) +option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the shared library and skip the +static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF +) # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures @@ -311,17 +314,23 @@ if(RAFT_COMPILE_LIBRARY) # Make sure not to add the rmm logger twice since it will be brought in as an interface source by # the rmm::rmm_logger_impl target. add_library(raft_lib SHARED $,EXCLUDE,rmm.*logger>) - add_library(raft_lib_static STATIC $,EXCLUDE,rmm.*logger>) + + set(_raft_lib_targets raft_lib) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $,EXCLUDE,rmm.*logger>) + list(APPEND _raft_lib_targets raft_lib_static) + endif() set_target_properties( - raft_lib raft_lib_static + ${_raft_lib_targets} PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - foreach(target raft_lib raft_lib_static raft_objs) + list(APPEND _raft_lib_targets raft_objs) + foreach(target IN LISTS _raft_lib_targets) target_link_libraries( ${target} PUBLIC raft::raft @@ -336,7 +345,9 @@ if(RAFT_COMPILE_LIBRARY) target_link_options(${target} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") endforeach() target_link_libraries(raft_lib PRIVATE rmm::rmm_logger_impl raft_logger_impl) - target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl raft_logger_impl) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl raft_logger_impl) + endif() endif() if(TARGET raft_lib AND (NOT TARGET raft::raft_lib)) @@ -348,20 +359,22 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ -) + target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ + ) +endif() # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- @@ -410,8 +423,12 @@ install( EXPORT raft-exports ) +set(_raft_compiled_install_targets raft_compiled) +if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND _raft_compiled_install_targets raft_compiled_static) +endif() install( - TARGETS raft_compiled raft_compiled_static + TARGETS ${_raft_compiled_install_targets} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -424,12 +441,14 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) + endif() install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -500,8 +519,12 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled compiled-static) - list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + list(APPEND raft_components compiled) + list(APPEND raft_export_sets raft-compiled-lib-exports) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_components compiled-static) + list(APPEND raft_export_sets raft-compiled-static-lib-exports) + endif() endif() string( diff --git a/dependencies.yaml b/dependencies.yaml index 689cf8414c..44c240b6ce 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,6 +6,8 @@ files: cuda: ["11.8", "12.5"] arch: [x86_64, aarch64] includes: + - build_common + - build_cython - checks - cuda - cuda_version @@ -15,7 +17,6 @@ files: - depends_on_rmm - develop - docs - - rapids_build - rapids_build_skbuild - run_pylibraft - run_raft_dask @@ -48,6 +49,29 @@ files: - docs - py_version - test_pylibraft + py_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: build-system + includes: + - rapids_build_skbuild + py_rapids_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_common + - depends_on_librmm + py_run_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: project + includes: + - cuda_wheels py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -62,16 +86,19 @@ files: table: tool.rapids-build-backend key: requires includes: + - build_common + - build_cython + - depends_on_libraft + - depends_on_librmm - depends_on_cuda_python - depends_on_rmm - - rapids_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft extras: table: project includes: - - cuda_wheels + - depends_on_libraft - depends_on_cuda_python - depends_on_rmm - run_pylibraft @@ -99,8 +126,11 @@ files: table: tool.rapids-build-backend key: requires includes: + - build_common + - build_cython + - depends_on_libraft + - depends_on_librmm - depends_on_ucx_build - - rapids_build py_run_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -108,6 +138,7 @@ files: table: project includes: - depends_on_distributed_ucxx + - depends_on_libraft - run_raft_dask py_test_raft_dask: output: pyproject @@ -135,12 +166,11 @@ dependencies: - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.10.0 - rapids_build: + build_common: common: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 - - cython>=3.0.0,<3.1.0a0 - ninja - output_types: [conda] packages: @@ -182,7 +212,11 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - + build_cython: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0,<3.1.0a0 checks: common: - output_types: [conda, requirements] @@ -471,6 +505,55 @@ dependencies: packages: - distributed-ucxx-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} + depends_on_libraft: + common: + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.2.*,>=0.0.0a0 + - matrix: + packages: + - libraft==25.2.*,>=0.0.0a0 + depends_on_librmm: + common: + - output_types: conda + packages: + - &librmm_unsuffixed librmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.2.*,>=0.0.0a0 + - matrix: + packages: + - *librmm_unsuffixed depends_on_rmm: common: - output_types: conda diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt new file mode 100644 index 0000000000..57efcd61ab --- /dev/null +++ b/python/libraft/CMakeLists.txt @@ -0,0 +1,65 @@ +# ============================================================================= +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libraft-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) + +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. +find_package(raft "${RAPIDS_VERSION}") + +if(raft_FOUND) + return() +endif() + +unset(raft_FOUND) + +# --- CUDA --- # +find_package(CUDAToolkit REQUIRED) +set(CUDA_STATIC_RUNTIME ON) +set(CUDA_STATIC_MATH_LIBRARIES ON) +if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) +elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") +endif() + +# --- RAFT ---# +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) +set(RAFT_COMPILE_LIBRARY ON) + +add_subdirectory(../../cpp raft-cpp) + +if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set_property( + TARGET raft_lib + PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) +endif() diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libraft/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libraft/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py new file mode 100644 index 0000000000..9260f4e67c --- /dev/null +++ b/python/libraft/libraft/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libraft._version import __git_commit__, __version__ +from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py new file mode 100644 index 0000000000..530bf8bea6 --- /dev/null +++ b/python/libraft/libraft/_version.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py new file mode 100644 index 0000000000..ad3db9e09c --- /dev/null +++ b/python/libraft/libraft/load.py @@ -0,0 +1,80 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libraft that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libraft.so and its dependencies""" + prefer_system_installation = ( + os.getenv("RAPIDS_LIBRAFT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + soname = "libraft.so" + libraft_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libraft_lib = _load_system_installation(soname) + except OSError: + libraft_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt + # before packaging the wheel), look for a system installation. + try: + libraft_lib = _load_wheel_installation(soname) + if libraft_lib is None: + libraft_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libraft was loaded from. + return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml new file mode 100644 index 0000000000..549a1bf651 --- /dev/null +++ b/python/libraft/pyproject.toml @@ -0,0 +1,115 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "rapids_build_backend.build" + +[project] +name = "libraft" +dynamic = ["version"] +description = "RAFT: Reusable Algorithms Functions and other Tools (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +dependencies = [ + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +classifiers = [ + "Intended Audience :: Developers", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/raft" +Documentation = "https://docs.rapids.ai/api/raft/stable/" + +[project.entry-points."cmake.prefix"] +libraft = "libraft" + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_first_party = [ + "libraft", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.install-dir = "libraft" +wheel.packages = ["libraft"] +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libraft/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "librmm==25.2.*,>=0.0.0a0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" + +[tool.pydistcheck] +select = [ + # NOTE: size threshold is managed via CLI args in CI scripts + "distro-too-large-compressed", +] diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 758c1e4711..83c262dc10 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,68 +27,13 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) - - endif() -else() - set(raft_FOUND OFF) -endif() +# an installed version of raft contains the other necessary targets (like CCCL and cuco) +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft compiled) include(rapids-cython-core) -if(NOT raft_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set_property( - TARGET raft_lib - PROPERTY INSTALL_RPATH - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - endif() - - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibraft/common) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/sparse) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index b0869501f3..a01e02ec33 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -13,4 +13,15 @@ # limitations under the License. # +# If libraft was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + from pylibraft._version import __git_commit__, __version__ diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 53279bfaf7..d1c1acb3aa 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 10ff776471..7d61855111 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt index ef16981644..7b2c9f6162 100644 --- a/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt +++ b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX sparse_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX sparse_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index ba454af591..912f1ad947 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -32,11 +32,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ "cuda-python", + "libraft==25.2.*,>=0.0.0a0", "numpy>=1.23,<3.0a0", - "nvidia-cublas", - "nvidia-curand", - "nvidia-cusolver", - "nvidia-cusparse", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -124,18 +121,22 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", "cython>=3.0.0,<3.1.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' + [tool.pytest.ini_options] filterwarnings = [ "error", diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 9ebbaa5298..c8c5c5ba83 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,38 +25,16 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF -) - rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) -else() - set(raft_FOUND OFF) -endif() - -if(NOT raft_FOUND) - # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all - # library compilation and we don't need to install anything here. - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARIES OFF) - set(RAFT_COMPILE_DIST_LIBRARY OFF) - set(RAFT_COMPILE_NN_LIBRARY OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(RAFT_DASK_UCXX_STATIC ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) - find_package(NCCL REQUIRED) -endif() +# why these components: +# +# * 'raft' = the headers, needed to link against libraft +# * 'distributed' = needed for NCCL +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft distributed) include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index f5daf70f92..e6b9c4aa0e 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -45,8 +45,8 @@ function(find_and_configure_ucxx) endfunction() # Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft +# To use a different ucxx locally, set the CMake variable +# CPM_ucxx_SOURCE=/path/to/local/ucxx find_and_configure_ucxx(VERSION 0.42 FORK rapidsai PINNED_TAG branch-0.42 diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index cabe8e72a6..d3a26db282 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -33,6 +33,7 @@ requires-python = ">=3.10" dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "distributed-ucxx==0.42.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", "pylibraft==25.2.*,>=0.0.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", "ucx-py==0.42.*,>=0.0.0a0", @@ -119,6 +120,8 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0,<3.1.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 19a037ae75..78248fad7a 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -13,8 +13,6 @@ # limitations under the License. # -from raft_dask._version import __git_commit__, __version__ - # If libucx was installed as a wheel, we must request it to load the library symbols. # Otherwise, we assume that the library was installed in a system path that ld can find. try: @@ -24,3 +22,16 @@ else: libucx.load_library() del libucx + +# If libraft was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + +from raft_dask._version import __git_commit__, __version__ diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 65d5f06577..49dee15d8f 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 2ff1cd9150..8475bcaa93 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX )