From 24a4db88687c8a02ed6faf23f4d20b3ca56cf8b3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 16 Dec 2024 08:30:56 -0800 Subject: [PATCH 01/25] start adding libraft wheels --- build.sh | 4 +- python/libraft/CMakeLists.txt | 63 +++++++++++++++ python/libraft/LICENSE | 1 + python/libraft/README.md | 1 + python/libraft/libraft/VERSION | 1 + python/libraft/libraft/__init__.py | 16 ++++ python/libraft/libraft/_version.py | 30 +++++++ python/libraft/libraft/load.py | 77 ++++++++++++++++++ python/libraft/pyproject.toml | 125 +++++++++++++++++++++++++++++ python/pylibraft/CMakeLists.txt | 42 ---------- 10 files changed, 315 insertions(+), 45 deletions(-) create mode 100644 python/libraft/CMakeLists.txt create mode 120000 python/libraft/LICENSE create mode 120000 python/libraft/README.md create mode 120000 python/libraft/libraft/VERSION create mode 100644 python/libraft/libraft/__init__.py create mode 100644 python/libraft/libraft/_version.py create mode 100644 python/libraft/libraft/load.py create mode 100644 python/libraft/pyproject.toml diff --git a/build.sh b/build.sh index a95cb8ee23..558e6ca8d6 100755 --- a/build.sh +++ b/build.sh @@ -349,9 +349,7 @@ fi # Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" -fi + # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt new file mode 100644 index 0000000000..fde6f34ee3 --- /dev/null +++ b/python/libraft/CMakeLists.txt @@ -0,0 +1,63 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +include(rapids-cuda) +rapids_cuda_init_architectures(libraft-python) + +project( + libraft-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) + +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. +find_package(raft "${RAPIDS_VERSION}") + +if(raft_FOUND) + return() +endif() + +unset(raft_FOUND) + +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(RAFT_COMPILE_LIBRARY ON) +set(CUDA_STATIC_RUNTIME ON) +set(CUDA_STATIC_MATH_LIBRARIES ON) +if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) +elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") +endif() + +add_subdirectory(../../cpp raft-cpp) + +if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set_property( + TARGET raft_lib + PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) +endif() diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libraft/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libraft/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py new file mode 100644 index 0000000000..2df69256fa --- /dev/null +++ b/python/libraft/libraft/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libraft._version import __git_commit__, __version__ +from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py new file mode 100644 index 0000000000..a5171f19f4 --- /dev/null +++ b/python/libraft/libraft/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py new file mode 100644 index 0000000000..4cdd499323 --- /dev/null +++ b/python/libraft/libraft/load.py @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libraft that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile(lib := os.path.join(os.path.dirname(__file__), "lib64", soname)): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcugraph.so and its dependencies""" + prefer_system_installation = ( + os.getenv("RAPIDS_LIBRAFT_PREFER_SYSTEM_LIBRARY", "false").lower() != "false" + ) + + soname = "libraft.so" + libraft_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libraft_lib = _load_system_installation(soname) + except OSError: + libraft_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libraft_lib = _load_wheel_installation(soname) + if libraft_lib is None: + libraft_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcugraph was loaded from. + return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml new file mode 100644 index 0000000000..4df05fa784 --- /dev/null +++ b/python/libraft/pyproject.toml @@ -0,0 +1,125 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "rapids_build_backend.build" + +[project] +name = "libraft" +dynamic = ["version"] +description = "RAFT: Reusable Algorithms Functions and other Tools (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +dependencies = [ + "cuda-python", + "numpy>=1.23,<3.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", + "rmm==25.2.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +classifiers = [ + "Intended Audience :: Developers", +] + +[project.optional-dependencies] +test = [ + "cupy-cuda11x>=12.0.0", + "pytest-cov", + "pytest==7.*", + "scikit-learn", + "scipy", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/raft" +Documentation = "https://docs.rapids.ai/api/raft/stable/" + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_first_party = [ + "libraft", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libraft"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libraft/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "cuda-python", + "cython>=3.0.0,<3.1.0a0", + "ninja", + "rmm==25.2.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" + +[tool.pydistcheck] +select = [ + # NOTE: size threshold is managed via CLI args in CI scripts + "distro-too-large-compressed", +] diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 758c1e4711..faec24f0e3 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,9 +27,6 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) # If the user requested it we attempt to find RAFT. @@ -48,47 +45,8 @@ endif() include(rapids-cython-core) -if(NOT raft_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set_property( - TARGET raft_lib - PROPERTY INSTALL_RPATH - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - endif() - - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibraft/common) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/sparse) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() From c0f059cdca5c4964e376b77a181ccf9b7bb19e30 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 16 Dec 2024 10:27:16 -0800 Subject: [PATCH 02/25] libraft minimally working --- ci/build_wheel.sh | 3 ++- ci/build_wheel_libraft.sh | 43 ++++++++++++++++++++++++++++++ ci/build_wheel_pylibraft.sh | 2 +- ci/build_wheel_raft_dask.sh | 2 +- dependencies.yaml | 43 ++++++++++++++++++++++++++++++ python/libraft/CMakeLists.txt | 4 +++ python/libraft/libraft/_version.py | 5 +++- python/libraft/libraft/load.py | 11 +++++--- 8 files changed, 105 insertions(+), 8 deletions(-) create mode 100755 ci/build_wheel_libraft.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 326ee9a4c7..f2b56ff225 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 underscore_package_name=$(echo "${package_name}" | tr "-" "_") # Clear out system ucx files to ensure that we're getting ucx from the wheel. @@ -55,4 +56,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_type} final_dist diff --git a/ci/build_wheel_libraft.sh b/ci/build_wheel_libraft.sh new file mode 100755 index 0000000000..27c7270a1c --- /dev/null +++ b/ci/build_wheel_libraft.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libraft" +package_dir="python/libraft" + +rapids-logger "Generating build requirements" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "${matrix_selectors}" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF${EXTRA_CMAKE_ARGS}" + +ci/build_wheel.sh libraft ${package_dir} cpp +ci/validate_wheel.sh ${package_dir} final_dist libraft diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index dd62ab5399..1212dc58d5 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -17,5 +17,5 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh pylibraft ${package_dir} +ci/build_wheel.sh pylibraft ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist pylibraft diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index d49d131abf..92f364b052 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -8,5 +8,5 @@ package_dir="python/raft-dask" # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" -ci/build_wheel.sh raft-dask ${package_dir} +ci/build_wheel.sh raft-dask ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist raft-dask diff --git a/dependencies.yaml b/dependencies.yaml index 37ea223a01..5f6b9ff32f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -48,6 +48,23 @@ files: - docs - py_version - test_pylibraft + py_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: build-system + includes: + - rapids_build_skbuild + py_rapids_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_pylibraft + - depends_on_librmm + - rapids_build py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -537,6 +554,32 @@ dependencies: packages: - distributed-ucxx-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} + depends_on_librmm: + common: + - output_types: conda + packages: + - &librmm_unsuffixed librmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.2.*,>=0.0.0a0 + - matrix: + packages: + - *librmm_unsuffixed depends_on_ucx_build: common: - output_types: conda diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index fde6f34ee3..33793df402 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -29,6 +29,10 @@ option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libr # Check if raft is already available. If so, it is the user's responsibility to ensure that the # CMake package is also available at build time of the Python raft package. + +find_package(CUDAToolkit REQUIRED) + +# TODO(jameslamb): make this work find_package(raft "${RAPIDS_VERSION}") if(raft_FOUND) diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py index a5171f19f4..7dd732b490 100644 --- a/python/libraft/libraft/_version.py +++ b/python/libraft/libraft/_version.py @@ -15,7 +15,10 @@ import importlib.resources __version__ = ( - importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() ) try: __git_commit__ = ( diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py index 4cdd499323..f05120224a 100644 --- a/python/libraft/libraft/load.py +++ b/python/libraft/libraft/load.py @@ -36,15 +36,18 @@ def _load_wheel_installation(soname: str): """Try to dlopen() the library indicated by ``soname`` Returns ``None`` if the library cannot be loaded. """ - if os.path.isfile(lib := os.path.join(os.path.dirname(__file__), "lib64", soname)): + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) return None def load_library(): - """Dynamically load libcugraph.so and its dependencies""" + """Dynamically load libraft.so and its dependencies""" prefer_system_installation = ( - os.getenv("RAPIDS_LIBRAFT_PREFER_SYSTEM_LIBRARY", "false").lower() != "false" + os.getenv("RAPIDS_LIBRAFT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" ) soname = "libraft.so" @@ -73,5 +76,5 @@ def load_library(): # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle - # to inspect where libcugraph was loaded from. + # to inspect where libraft was loaded from. return libraft_lib From d11641756c212e39de1d42795afb5c5bcd86c8c4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 16 Dec 2024 12:15:02 -0800 Subject: [PATCH 03/25] pylibraft builds mostly working --- ci/build_wheel_pylibraft.sh | 11 +----- dependencies.yaml | 34 +++++++++++++++++++ python/libraft/CMakeLists.txt | 3 ++ python/libraft/pyproject.toml | 9 +++-- python/pylibraft/CMakeLists.txt | 16 +-------- python/pylibraft/pylibraft/__init__.py | 11 ++++++ .../pylibraft/pylibraft/common/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 2 +- .../pylibraft/sparse/linalg/CMakeLists.txt | 2 +- python/pylibraft/pyproject.toml | 4 ++- python/raft-dask/pyproject.toml | 2 ++ python/raft-dask/raft_dask/__init__.py | 15 ++++++-- .../raft-dask/raft_dask/common/CMakeLists.txt | 2 +- .../raft_dask/include_test/CMakeLists.txt | 2 +- 14 files changed, 79 insertions(+), 36 deletions(-) diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 1212dc58d5..3740a6099a 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -5,17 +5,8 @@ set -euo pipefail package_dir="python/pylibraft" -case "${RAPIDS_CUDA_VERSION}" in - 12.*) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" - ;; - 11.*) - EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" - ;; -esac - # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" ci/build_wheel.sh pylibraft ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist pylibraft diff --git a/dependencies.yaml b/dependencies.yaml index 5f6b9ff32f..91687bc4c1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -65,6 +65,13 @@ files: - build_pylibraft - depends_on_librmm - rapids_build + py_run_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: project + includes: + - cuda_wheels py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -79,6 +86,7 @@ files: table: tool.rapids-build-backend key: requires includes: + - depends_on_libraft - rapids_build - build_pylibraft py_run_pylibraft: @@ -88,6 +96,7 @@ files: table: project includes: - cuda_wheels + - depends_on_libraft - run_pylibraft py_test_pylibraft: output: pyproject @@ -113,6 +122,7 @@ files: table: tool.rapids-build-backend key: requires includes: + - depends_on_libraft - rapids_build - depends_on_ucx_build py_run_raft_dask: @@ -121,6 +131,7 @@ files: extras: table: project includes: + - depends_on_libraft - run_raft_dask - depends_on_distributed_ucxx py_test_raft_dask: @@ -554,6 +565,29 @@ dependencies: packages: - distributed-ucxx-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} + depends_on_libraft: + common: + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.2.*,>=0.0.0a0 + - matrix: + packages: + - libraft==25.2.*,>=0.0.0a0 depends_on_librmm: common: - output_types: conda diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 33793df402..274638046e 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -52,6 +52,9 @@ elseif(USE_CUDA_MATH_WHEELS) message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") endif() +# ensure installs are inside the libraft/ folder in site-packages/ +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + add_subdirectory(../../cpp raft-cpp) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 4df05fa784..ab0fe11879 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -31,13 +31,10 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "cuda-python", - "numpy>=1.23,<3.0a0", "nvidia-cublas", "nvidia-curand", "nvidia-cusolver", "nvidia-cusparse", - "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -56,6 +53,9 @@ test = [ Homepage = "https://github.com/rapidsai/raft" Documentation = "https://docs.rapids.ai/api/raft/stable/" +[project.entry-points."cmake.prefix"] +libraft = "libraft" + [tool.isort] line_length = 79 multi_line_output = 3 @@ -100,6 +100,8 @@ ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libraft"] +wheel.install-dir = "libraft" +wheel.py-api = "py3" [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" @@ -112,6 +114,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", "cython>=3.0.0,<3.1.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index faec24f0e3..9eb658e55e 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,21 +27,7 @@ project( LANGUAGES CXX CUDA ) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) - - endif() -else() - set(raft_FOUND OFF) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft compiled) include(rapids-cython-core) diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index b0869501f3..a01e02ec33 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -13,4 +13,15 @@ # limitations under the License. # +# If libraft was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + from pylibraft._version import __git_commit__, __version__ diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 53279bfaf7..d1c1acb3aa 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 10ff776471..7d61855111 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt index ef16981644..7b2c9f6162 100644 --- a/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt +++ b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX sparse_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX sparse_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index ba454af591..a41bd879f7 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -32,6 +32,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ "cuda-python", + "libraft==25.2.*,>=0.0.0a0", "numpy>=1.23,<3.0a0", "nvidia-cublas", "nvidia-curand", @@ -124,11 +125,12 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", "cython>=3.0.0,<3.1.0a0", + "libraft==25.2.*,>=0.0.0a0", "ninja", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 33643c481e..cc76c07d2b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "distributed-ucxx==0.42.*,>=0.0.0a0", "joblib>=0.11", + "libraft==25.2.*,>=0.0.0a0", "numba>=0.57", "pylibraft==25.2.*,>=0.0.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", @@ -121,6 +122,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0,<3.1.0a0", + "libraft==25.2.*,>=0.0.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 19a037ae75..78248fad7a 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -13,8 +13,6 @@ # limitations under the License. # -from raft_dask._version import __git_commit__, __version__ - # If libucx was installed as a wheel, we must request it to load the library symbols. # Otherwise, we assume that the library was installed in a system path that ld can find. try: @@ -24,3 +22,16 @@ else: libucx.load_library() del libucx + +# If libraft was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + +from raft_dask._version import __git_commit__, __version__ diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 65d5f06577..49dee15d8f 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 2ff1cd9150..8475bcaa93 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) From eb177350bb82f16f3541665a3b57f9725c42cbf8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 08:47:53 -0800 Subject: [PATCH 04/25] more changes --- .github/workflows/build.yaml | 20 ++++++++++++++++++++ .github/workflows/pr.yaml | 14 ++++++++++++-- build.sh | 4 +--- ci/validate_wheel.sh | 15 +++++++++++---- dependencies.yaml | 2 ++ python/libraft/pyproject.toml | 1 + python/pylibraft/pyproject.toml | 2 ++ python/raft-dask/CMakeLists.txt | 30 ++++-------------------------- python/raft-dask/pyproject.toml | 3 +++ 9 files changed, 56 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5f80d8cfda..eeb6ac8f24 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,27 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libraft: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcugraph.sh + wheel-publish-libraft: + needs: wheel-publish-libcugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libraft wheel-build-pylibraft: + needs: wheel-publish-libraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 965943e726..5adb2b65ed 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -103,10 +103,20 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibraft: + wheel-build-libraft: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libraft.sh + wheel-build-pylibraft: + needs: [checks, wheel-build-libraft] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh @@ -119,7 +129,7 @@ jobs: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: - needs: wheel-tests-pylibraft + needs: [checks, wheel-build-libraft] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: diff --git a/build.sh b/build.sh index 558e6ca8d6..671625a65f 100755 --- a/build.sh +++ b/build.sh @@ -347,10 +347,8 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" - # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS +SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') # If clean given, run it prior to any other steps diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5ef72ad895..986be879d5 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,10 +10,17 @@ package_name=$3 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 -if [[ "${package_name}" == "raft-dask" ]]; then - PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '200M' - ) +# TODO(jameslamb): adjust these thresholds +if [[ "${package_name}" == "libraft" ]]; then + if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then + PYDISTCHECK_ARGS=( + --max-allowed-size-compressed '5.0G' + ) + else + PYDISTCHECK_ARGS=( + --max-allowed-size-compressed '5.0G' + ) + fi elif [[ "${package_name}" == "pylibraft" ]]; then if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( diff --git a/dependencies.yaml b/dependencies.yaml index 91687bc4c1..d4576a5ca2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -87,6 +87,7 @@ files: key: requires includes: - depends_on_libraft + - depends_on_librmm - rapids_build - build_pylibraft py_run_pylibraft: @@ -123,6 +124,7 @@ files: key: requires includes: - depends_on_libraft + - depends_on_librmm - rapids_build - depends_on_ucx_build py_run_raft_dask: diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index ab0fe11879..e51f707847 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -124,5 +124,6 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pydistcheck] select = [ # NOTE: size threshold is managed via CLI args in CI scripts + # TODO(jameslamb): adjust this "distro-too-large-compressed", ] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a41bd879f7..bb0ca3c5aa 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -126,6 +126,7 @@ requires = [ "cuda-python", "cython>=3.0.0,<3.1.0a0", "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -135,6 +136,7 @@ matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ # NOTE: size threshold is managed via CLI args in CI scripts + # TODO(jameslamb): adjust this "distro-too-large-compressed", ] diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 9ebbaa5298..f3f588ea8a 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,38 +25,16 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF -) - rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) -else() - set(raft_FOUND OFF) -endif() - -if(NOT raft_FOUND) - # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all - # library compilation and we don't need to install anything here. - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARIES OFF) - set(RAFT_COMPILE_DIST_LIBRARY OFF) - set(RAFT_COMPILE_NN_LIBRARY OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(RAFT_DASK_UCXX_STATIC ON) +# raft-dask doesn't actually use raft libraries, it just needs the headers +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft) - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) - find_package(NCCL REQUIRED) -endif() +# list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) +# find_package(NCCL REQUIRED) include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index cc76c07d2b..c7678328b0 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -30,6 +30,7 @@ authors = [ ] license = { text = "Apache 2.0" } requires-python = ">=3.10" +# TODO(jameslamb): does this really need joblib and numba? dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "distributed-ucxx==0.42.*,>=0.0.0a0", @@ -123,6 +124,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0,<3.1.0a0", "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -135,4 +137,5 @@ select = [ ] # detect when package size grows significantly +# TODO(jameslamb): adjust this max_allowed_size_compressed = '300M' From d317235a6e6853a70bfa1dffa8df012ab0a2d430 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 08:48:36 -0800 Subject: [PATCH 05/25] comment out more CI --- .github/workflows/pr.yaml | 122 +++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5adb2b65ed..3ff4109739 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,17 +14,17 @@ jobs: needs: - changed-files - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-python-build - - conda-python-tests - - docs-build + # - conda-cpp-build + # - conda-cpp-tests + # - conda-cpp-checks + # - conda-python-build + # - conda-python-tests + # - docs-build - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask - - devcontainer + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() @@ -59,50 +59,50 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 with: enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 - with: - build_type: pull-request - node_type: cpu16 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 - with: - build_type: pull-request - enable_check_symbols: true - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + # with: + # build_type: pull-request + # node_type: cpu16 + # conda-cpp-tests: + # needs: [conda-cpp-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 + # with: + # build_type: pull-request + # enable_check_symbols: true + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 + # with: + # build_type: pull-request + # conda-python-tests: + # needs: [conda-python-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-libraft: needs: checks secrets: inherit @@ -143,13 +143,13 @@ jobs: with: build_type: pull-request script: ci/test_wheel_raft_dask.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON --verbose; - sccache -s; + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_PRIMS_BENCH=ON --verbose; + # sccache -s; From 679d2890c1718bb4ad41a339233a3c2d696eab6b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 09:12:21 -0800 Subject: [PATCH 06/25] more changes --- ci/build_wheel_raft_dask.sh | 2 +- python/raft-dask/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index 92f364b052..4570d9c68d 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -6,7 +6,7 @@ set -euo pipefail package_dir="python/raft-dask" # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" ci/build_wheel.sh raft-dask ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist raft-dask diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index f3f588ea8a..cd00ada043 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -26,6 +26,7 @@ project( ) rapids_cpm_init() +# TODO(jameslamb): could this just use the libucxx wheel? # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) From 51e7041f7e50e668aa31436551533d009f221090 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 09:54:40 -0800 Subject: [PATCH 07/25] more dependencies.yaml fiddling --- dependencies.yaml | 7 ++++++- python/raft-dask/CMakeLists.txt | 7 +++++-- python/raft-dask/pyproject.toml | 1 - 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d4576a5ca2..8a562924c3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -122,9 +122,14 @@ files: extras: table: tool.rapids-build-backend key: requires + # TODO(jameslamb): use libucxx? includes: - depends_on_libraft - - depends_on_librmm + # CMake Error at /pyenv/versions/3.12.7/lib/python3.12/site-packages/librmm/lib64/cmake/rmm/rmm-targets.cmake:42 (message): + # Some (but not all) targets in this export set were already defined. + # Targets Defined: rmm::rmm + # Targets not yet defined: rmm::rmm_logger, rmm::rmm_logger_im + # - depends_on_librmm - rapids_build - depends_on_ucx_build py_run_raft_dask: diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index cd00ada043..0311912bc5 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -31,8 +31,11 @@ rapids_cpm_init() find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -# raft-dask doesn't actually use raft libraries, it just needs the headers -find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft) +# why these components: +# +# * 'raft' = the headers, needed to link against libraft +# * 'distributed = needed for 'nccl' +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft distributed) # list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) # find_package(NCCL REQUIRED) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index c7678328b0..8be374cce6 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -124,7 +124,6 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0,<3.1.0a0", "libraft==25.2.*,>=0.0.0a0", - "librmm==25.2.*,>=0.0.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 6d38c19ee31b9e8fdc7eccd217e9f8fe374b2c99 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 09:56:37 -0800 Subject: [PATCH 08/25] actually run builds --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3ff4109739..9f0f8dbdce 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,6 +20,7 @@ jobs: # - conda-python-build # - conda-python-tests # - docs-build + - wheel-build-libraft - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask From 717df6fe61e5f4c2f24f259ee8a03905bd84c166 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 10:12:16 -0800 Subject: [PATCH 09/25] pre-commit --- python/libraft/libraft/load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py index f05120224a..8cc9035fa9 100644 --- a/python/libraft/libraft/load.py +++ b/python/libraft/libraft/load.py @@ -62,8 +62,8 @@ def load_library(): libraft_lib = _load_wheel_installation(soname) else: # Prefer the libraries bundled in this package. If they aren't found - # (which might be the case in builds where the library was prebuilt before - # packaging the wheel), look for a system installation. + # (which might be the case in builds where the library was prebuilt + # before packaging the wheel), look for a system installation. try: libraft_lib = _load_wheel_installation(soname) if libraft_lib is None: From 11168a1ed9441f128b9e4916b9c58d8985ba4e9c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 10:45:30 -0800 Subject: [PATCH 10/25] install libraft wheels --- ci/build_wheel_pylibraft.sh | 9 +++++++++ ci/build_wheel_raft_dask.sh | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 3740a6099a..f20d88f2f4 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -5,6 +5,15 @@ set -euo pipefail package_dir="python/pylibraft" +# Downloads libraft wheels from this current build, +# then ensures 'pylibraft' wheel builds always use the 'libraft' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist +echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index 4570d9c68d..afd286006d 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -5,6 +5,17 @@ set -euo pipefail package_dir="python/raft-dask" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Downloads libraft wheels from this current build, +# then ensures 'raft-dask' wheel builds always use the 'libraft' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist +echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" From 46bfcfaf64a2e0a8056a9bbfa7cb4a7d2d6e281b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 11:06:16 -0800 Subject: [PATCH 11/25] fewer builds, fix libraft downloads --- .github/workflows/build.yaml | 4 +++- .github/workflows/pr.yaml | 2 ++ ci/build_wheel_pylibraft.sh | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index eeb6ac8f24..bce27e9eb7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,9 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_libcugraph.sh + script: ci/build_wheel_libraft.sh + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-publish-libraft: needs: wheel-publish-libcugraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9f0f8dbdce..d91fc82178 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -114,6 +114,8 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_libraft.sh + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-build-pylibraft: needs: [checks, wheel-build-libraft] secrets: inherit diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index f20d88f2f4..b8da15ffaf 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -5,6 +5,8 @@ set -euo pipefail package_dir="python/pylibraft" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + # Downloads libraft wheels from this current build, # then ensures 'pylibraft' wheel builds always use the 'libraft' just built in the same CI run. # From 97f5d31f4be70c47fde79abf6fa8caa39cf91243 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 11:35:37 -0800 Subject: [PATCH 12/25] fix branches --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bce27e9eb7..d00ba45f58 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,7 +78,7 @@ jobs: # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-publish-libraft: - needs: wheel-publish-libcugraph + needs: wheel-build-libraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d91fc82178..73dbea68b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -109,7 +109,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: - build_type: ${{ inputs.build_type || 'branch' }} + build_type: pull-request branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} From f492d59978af3390e418796228aedb2601d03efc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 12:13:08 -0800 Subject: [PATCH 13/25] more fixes to CI scripts --- ci/build_wheel.sh | 6 ++++++ ci/validate_wheel.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f2b56ff225..4c295c416e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,6 +40,12 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac +if [[ ${package_name} != "libraft" ]]; then + EXCLUDE_ARGS+=( + --exclude "libraft.so" + ) +fi + sccache --zero-stats rapids-logger "Building '${package_name}' wheel" diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 986be879d5..018abcba23 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -21,7 +21,7 @@ if [[ "${package_name}" == "libraft" ]]; then --max-allowed-size-compressed '5.0G' ) fi -elif [[ "${package_name}" == "pylibraft" ]]; then +elif [[ "${package_name}" == "pylibraft" ]] || [[ "${package_name}" == "raft-dask" ]]; then if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( --max-allowed-size-compressed '600M' From ad13d08045a4309f489ad2dc540885c33073ef30 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 12:55:03 -0800 Subject: [PATCH 14/25] download libraft in wheel test jobs, change file-size limits --- ci/test_wheel_pylibraft.sh | 8 ++++++-- ci/test_wheel_raft_dask.sh | 8 ++++---- ci/validate_wheel.sh | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index b38f5a690b..1e0b34d609 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,9 +5,13 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libraft-dep +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist + # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] +python -m pip install \ + ./local-libraft-dep/libraft*.whl \ + "$(echo ./dist/pylibraft*.whl)[test]" python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a778a3ec51..011de4d409 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -5,13 +5,13 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist - -# Download the pylibraft built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libraft-dep +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-pylibraft-dep +RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install -v \ + ./local-libraft-dep/libraft*.whl \ ./local-pylibraft-dep/pylibraft*.whl \ "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 018abcba23..708aeb029c 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -28,7 +28,7 @@ elif [[ "${package_name}" == "pylibraft" ]] || [[ "${package_name}" == "raft-das ) else PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '100M' + --max-allowed-size-compressed '400M' ) fi else From 6bf5ebacd362a898d2580e88e17113ddcfeafdae Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 13:19:57 -0800 Subject: [PATCH 15/25] pylibraft no longer needs CUDA-wheels runtime dependency --- dependencies.yaml | 1 - python/pylibraft/pyproject.toml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 58d09ad0be..924ef1f24b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -97,7 +97,6 @@ files: extras: table: project includes: - - cuda_wheels - depends_on_libraft - depends_on_cuda_python - depends_on_rmm diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index bb0ca3c5aa..a3b1fa9372 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -34,10 +34,6 @@ dependencies = [ "cuda-python", "libraft==25.2.*,>=0.0.0a0", "numpy>=1.23,<3.0a0", - "nvidia-cublas", - "nvidia-curand", - "nvidia-cusolver", - "nvidia-cusparse", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 978324442bd2347bee78acee1ad5463a9e99ded7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Dec 2024 13:32:33 -0800 Subject: [PATCH 16/25] update thresholds --- ci/validate_wheel.sh | 18 +++--------------- python/libraft/CMakeLists.txt | 6 ++---- python/libraft/pyproject.toml | 1 - python/pylibraft/pyproject.toml | 5 +++-- python/raft-dask/pyproject.toml | 2 -- 5 files changed, 8 insertions(+), 24 deletions(-) diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 708aeb029c..7f1dae43f2 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,30 +10,18 @@ package_name=$3 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 -# TODO(jameslamb): adjust these thresholds if [[ "${package_name}" == "libraft" ]]; then if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '5.0G' + --max-allowed-size-compressed '750M' ) else PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '5.0G' - ) - fi -elif [[ "${package_name}" == "pylibraft" ]] || [[ "${package_name}" == "raft-dask" ]]; then - if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then - PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '600M' - ) - else - PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '400M' + --max-allowed-size-compressed '100M' ) fi else - echo "Unsupported package name: ${package_name}" - exit 1 + PYDISTCHECK_ARGS=() fi cd "${package_dir}" diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 274638046e..bb9a87d36f 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -27,12 +27,10 @@ project( option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) -# Check if raft is already available. If so, it is the user's responsibility to ensure that the -# CMake package is also available at build time of the Python raft package. - find_package(CUDAToolkit REQUIRED) -# TODO(jameslamb): make this work +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. find_package(raft "${RAPIDS_VERSION}") if(raft_FOUND) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 9479f0a832..e01692552c 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -123,6 +123,5 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pydistcheck] select = [ # NOTE: size threshold is managed via CLI args in CI scripts - # TODO(jameslamb): adjust this "distro-too-large-compressed", ] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a3b1fa9372..912f1ad947 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -131,11 +131,12 @@ matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts - # TODO(jameslamb): adjust this "distro-too-large-compressed", ] +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' + [tool.pytest.ini_options] filterwarnings = [ "error", diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 8be374cce6..cc76c07d2b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -30,7 +30,6 @@ authors = [ ] license = { text = "Apache 2.0" } requires-python = ">=3.10" -# TODO(jameslamb): does this really need joblib and numba? dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "distributed-ucxx==0.42.*,>=0.0.0a0", @@ -136,5 +135,4 @@ select = [ ] # detect when package size grows significantly -# TODO(jameslamb): adjust this max_allowed_size_compressed = '300M' From ee9970292907fd91a1d12f9037c48cc0abc9bd5f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 24 Dec 2024 10:20:16 -0800 Subject: [PATCH 17/25] try re-enabling conda CI --- .github/workflows/pr.yaml | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 73dbea68b6..3482588cf2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,11 +14,11 @@ jobs: needs: - changed-files - checks - # - conda-cpp-build - # - conda-cpp-tests + - conda-cpp-build + - conda-cpp-tests # - conda-cpp-checks - # - conda-python-build - # - conda-python-tests + - conda-python-build + - conda-python-tests # - docs-build - wheel-build-libraft - wheel-build-pylibraft @@ -60,20 +60,20 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 with: enable_check_generated_files: false - # conda-cpp-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 - # with: - # build_type: pull-request - # node_type: cpu16 - # conda-cpp-tests: - # needs: [conda-cpp-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - # with: - # build_type: pull-request + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request # conda-cpp-checks: # needs: conda-cpp-build # secrets: inherit @@ -81,19 +81,19 @@ jobs: # with: # build_type: pull-request # enable_check_symbols: true - # conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 - # with: - # build_type: pull-request - # conda-python-tests: - # needs: [conda-python-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - # with: - # build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request # docs-build: # needs: conda-python-build # secrets: inherit From 48c98b8e918080a070e98fe70af32db492c2a261 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 6 Jan 2025 10:08:55 -0800 Subject: [PATCH 18/25] more changes --- cpp/cmake/thirdparty/get_rmm.cmake | 5 +++-- python/pylibraft/CMakeLists.txt | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake index 0e93363039..f16644670d 100644 --- a/cpp/cmake/thirdparty/get_rmm.cmake +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -16,8 +16,9 @@ function(find_and_configure_rmm) include(${rapids-cmake-dir}/cpm/rmm.cmake) - rapids_cpm_rmm(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports) + # intentionally not re-exporting + # TODO(jameslamb): make this description better + rapids_cpm_rmm() endfunction() find_and_configure_rmm() diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 9eb658e55e..f5b9be16fa 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,6 +27,17 @@ project( LANGUAGES CXX CUDA ) +# CCCL before rmm/cuco so we get the right version of CCCL +# TODO(jameslamb): is this necessary? +include(rapids-cmake) +include(rapids-cpm) +include(rapids-export) +include(rapids-find) +# add third party dependencies using CPM +rapids_cpm_init() +include(../../cpp/cmake/thirdparty/get_cccl.cmake) +include(../../cpp/cmake/thirdparty/get_rmm.cmake) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft compiled) include(rapids-cython-core) From 4c657b2817c29fbd1d6855697fe9181730db2d93 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 7 Jan 2025 14:24:46 -0800 Subject: [PATCH 19/25] comments --- python/libraft/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index bb9a87d36f..993760638a 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -53,6 +53,7 @@ endif() # ensure installs are inside the libraft/ folder in site-packages/ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) +# TODO(jameslamb): do not install the static library in the wheel add_subdirectory(../../cpp raft-cpp) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) From c6579e1b53c5eea35d1b2cb41719f82dd194164f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 8 Jan 2025 16:39:42 -0800 Subject: [PATCH 20/25] exclude static raft from wheels --- cpp/CMakeLists.txt | 22 +++++++++++++--------- python/libraft/CMakeLists.txt | 13 ++++++++++++- python/libraft/pyproject.toml | 18 +++++++++++++++++- python/pylibraft/CMakeLists.txt | 3 +++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 621f9fcef2..86cc0af0ad 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -411,7 +411,8 @@ install( ) install( - TARGETS raft_compiled raft_compiled_static + # TARGETS raft_compiled raft_compiled_static + TARGETS raft_compiled DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -424,12 +425,13 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) + # TODO(jameslamb): put all these static-skipping things behind an option like RAFT_BUILD_STATIC_LIB + # install( + # TARGETS raft_lib_static + # DESTINATION ${lib_dir} + # COMPONENT compiled-static + # EXPORT raft-compiled-static-lib-exports + # ) install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -500,8 +502,10 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled compiled-static) - list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + #list(APPEND raft_components compiled compiled-static) + #list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + list(APPEND raft_components compiled) + list(APPEND raft_export_sets raft-compiled-lib-exports) endif() string( diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 993760638a..5325437d6f 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -53,9 +53,20 @@ endif() # ensure installs are inside the libraft/ folder in site-packages/ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) -# TODO(jameslamb): do not install the static library in the wheel +# TODO(jameslamb): figure out how to keep the static library (libraft.a) out of wheels. +# +# things tried: +# +# - add_subdirectory(../../cpp raft-cpp) +# install( +# TARGETS +# cuco +# CUTLASS +# COMPONENT libraft_wheel_interface_dependencies +# ) + if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set_property( TARGET raft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index e01692552c..7556e6343e 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -99,8 +99,24 @@ minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true -wheel.packages = ["libraft"] +# exclude static libraries from the wheel +# +# TODO(jameslamb): if we agree on this as the mechanism, open an issue about updating install +# rules and using 'build.targets = ["raft_lib"]' instead. +# wheel.exclude = ["*.a"] +# build.targets = [ +# "raft" +# ] +# cannot reliably us 'install.components' / 'build.targets' to limit what gets +# installed because RAFT is installing dependencies like cutlass and cuco +# by including them in "all" +#install.components = [ +# "raft", +# "compiled", +# "libraft_wheel_interface_dependencies" +#] wheel.install-dir = "libraft" +wheel.packages = ["libraft"] wheel.py-api = "py3" [tool.scikit-build.metadata.version] diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index f5b9be16fa..a7c7f7cc10 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -38,6 +38,9 @@ rapids_cpm_init() include(../../cpp/cmake/thirdparty/get_cccl.cmake) include(../../cpp/cmake/thirdparty/get_rmm.cmake) +# TODO(jameslamb): remove this... just testing my understanding +# include(../../cpp/cmake/thirdparty/get_cutlass.cmake) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft compiled) include(rapids-cython-core) From 1fd760372bd9fa2be27fcd9663920501ea852a24 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 9 Jan 2025 14:44:50 -0800 Subject: [PATCH 21/25] add a RAFT_BUILD_STATIC_LIB option --- cpp/CMakeLists.txt | 29 ++++++++++++++++++----------- python/libraft/CMakeLists.txt | 24 +++++++++--------------- python/libraft/pyproject.toml | 25 ------------------------- 3 files changed, 27 insertions(+), 51 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 86cc0af0ad..7ed47b8a70 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -60,6 +60,7 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable deprecaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) +option(RAFT_BUILD_STATIC_LIB "Build static library and write install rules for it." ON) set(RAFT_COMPILE_LIBRARY_DEFAULT OFF) if(BUILD_TESTS OR BUILD_PRIMS_BENCH) @@ -410,9 +411,12 @@ install( EXPORT raft-exports ) +set(_raft_compiled_targets_to_install "raft_compiled") +if(RAFT_BUILD_STATIC_LIB) + list(APPEND _raft_compiled_targets_to_install "raft_compiled_static") +endif() install( - # TARGETS raft_compiled raft_compiled_static - TARGETS raft_compiled + TARGETS ${_raft_compiled_targets_to_install} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -425,13 +429,14 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - # TODO(jameslamb): put all these static-skipping things behind an option like RAFT_BUILD_STATIC_LIB - # install( - # TARGETS raft_lib_static - # DESTINATION ${lib_dir} - # COMPONENT compiled-static - # EXPORT raft-compiled-static-lib-exports - # ) + if(RAFT_BUILD_STATIC_LIB) + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) + endif() install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -502,10 +507,12 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - #list(APPEND raft_components compiled compiled-static) - #list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) list(APPEND raft_components compiled) list(APPEND raft_export_sets raft-compiled-lib-exports) + if(RAFT_BUILD_STATIC_LIB) + list(APPEND raft_components compiled-static) + list(APPEND raft_export_sets raft-compiled-static-lib-export) + endif() endif() string( diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 5325437d6f..8424acf697 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -16,8 +16,8 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../rapids_config.cmake) -include(rapids-cuda) -rapids_cuda_init_architectures(libraft-python) +# include(rapids-cuda) +# rapids_cuda_init_architectures(libraft-python) project( libraft-python @@ -27,7 +27,7 @@ project( option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) -find_package(CUDAToolkit REQUIRED) +# find_package(CUDAToolkit REQUIRED) # Check if raft is already available. If so, it is the user's responsibility to ensure that the # CMake package is also available at build time of the Python raft package. @@ -41,7 +41,13 @@ unset(raft_FOUND) set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) + +# --- RAFT ---# +set(RAFT_BUILD_STATIC_LIB OFF) set(RAFT_COMPILE_LIBRARY ON) + +# --- CUDA --- # +find_package(CUDAToolkit REQUIRED) set(CUDA_STATIC_RUNTIME ON) set(CUDA_STATIC_MATH_LIBRARIES ON) if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) @@ -53,20 +59,8 @@ endif() # ensure installs are inside the libraft/ folder in site-packages/ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) -# TODO(jameslamb): figure out how to keep the static library (libraft.a) out of wheels. -# -# things tried: -# -# - add_subdirectory(../../cpp raft-cpp) -# install( -# TARGETS -# cuco -# CUTLASS -# COMPONENT libraft_wheel_interface_dependencies -# ) - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set_property( TARGET raft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 7556e6343e..241acda1de 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -40,15 +40,6 @@ classifiers = [ "Intended Audience :: Developers", ] -[project.optional-dependencies] -test = [ - "cupy-cuda11x>=12.0.0", - "pytest-cov", - "pytest==7.*", - "scikit-learn", - "scipy", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - [project.urls] Homepage = "https://github.com/rapidsai/raft" Documentation = "https://docs.rapids.ai/api/raft/stable/" @@ -99,22 +90,6 @@ minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true -# exclude static libraries from the wheel -# -# TODO(jameslamb): if we agree on this as the mechanism, open an issue about updating install -# rules and using 'build.targets = ["raft_lib"]' instead. -# wheel.exclude = ["*.a"] -# build.targets = [ -# "raft" -# ] -# cannot reliably us 'install.components' / 'build.targets' to limit what gets -# installed because RAFT is installing dependencies like cutlass and cuco -# by including them in "all" -#install.components = [ -# "raft", -# "compiled", -# "libraft_wheel_interface_dependencies" -#] wheel.install-dir = "libraft" wheel.packages = ["libraft"] wheel.py-api = "py3" From a1a905d11d03722ec997f32bd6d7d5644f2f1c15 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 10 Jan 2025 07:44:22 -0800 Subject: [PATCH 22/25] be more thorough in skipping the static library --- cpp/CMakeLists.txt | 60 +++++++++++-------- cpp/cmake/thirdparty/get_rmm.cmake | 5 +- python/libraft/CMakeLists.txt | 2 +- python/pylibraft/CMakeLists.txt | 15 +---- python/raft-dask/CMakeLists.txt | 6 +- .../raft-dask/cmake/thirdparty/get_ucxx.cmake | 4 +- 6 files changed, 43 insertions(+), 49 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7ed47b8a70..eb7e8540f0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -60,15 +60,17 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable deprecaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) -option(RAFT_BUILD_STATIC_LIB "Build static library and write install rules for it." ON) set(RAFT_COMPILE_LIBRARY_DEFAULT OFF) if(BUILD_TESTS OR BUILD_PRIMS_BENCH) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) +option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the shared library and skip the +static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF +) # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures @@ -312,17 +314,23 @@ if(RAFT_COMPILE_LIBRARY) # Make sure not to add the rmm logger twice since it will be brought in as an interface source by # the rmm::rmm_logger_impl target. add_library(raft_lib SHARED $,EXCLUDE,rmm.*logger>) - add_library(raft_lib_static STATIC $,EXCLUDE,rmm.*logger>) + + set(_raft_lib_targets raft_lib) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $,EXCLUDE,rmm.*logger>) + list(APPEND _raft_lib_targets raft_lib_static) + endif() set_target_properties( - raft_lib raft_lib_static + ${_raft_lib_targets} PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - foreach(target raft_lib raft_lib_static raft_objs) + list(APPEND _raft_lib_targets raft_objs) + foreach(target IN LISTS _raft_lib_targets) target_link_libraries( ${target} PUBLIC raft::raft @@ -337,7 +345,9 @@ if(RAFT_COMPILE_LIBRARY) target_link_options(${target} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") endforeach() target_link_libraries(raft_lib PRIVATE rmm::rmm_logger_impl raft_logger_impl) - target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl raft_logger_impl) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl raft_logger_impl) + endif() endif() if(TARGET raft_lib AND (NOT TARGET raft::raft_lib)) @@ -349,20 +359,22 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ -) + target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ + ) +endif() # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- @@ -411,12 +423,12 @@ install( EXPORT raft-exports ) -set(_raft_compiled_targets_to_install "raft_compiled") -if(RAFT_BUILD_STATIC_LIB) - list(APPEND _raft_compiled_targets_to_install "raft_compiled_static") +set(_raft_compiled_install_targets raft_compiled) +if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND _raft_compiled_install_targets raft_compiled_static) endif() install( - TARGETS ${_raft_compiled_targets_to_install} + TARGETS ${_raft_compiled_install_targets} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -429,7 +441,7 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - if(RAFT_BUILD_STATIC_LIB) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) install( TARGETS raft_lib_static DESTINATION ${lib_dir} @@ -509,9 +521,9 @@ set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) list(APPEND raft_components compiled) list(APPEND raft_export_sets raft-compiled-lib-exports) - if(RAFT_BUILD_STATIC_LIB) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) list(APPEND raft_components compiled-static) - list(APPEND raft_export_sets raft-compiled-static-lib-export) + list(APPEND raft_export_sets raft-compiled-static-lib-exports) endif() endif() diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake index f16644670d..0e93363039 100644 --- a/cpp/cmake/thirdparty/get_rmm.cmake +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -16,9 +16,8 @@ function(find_and_configure_rmm) include(${rapids-cmake-dir}/cpm/rmm.cmake) - # intentionally not re-exporting - # TODO(jameslamb): make this description better - rapids_cpm_rmm() + rapids_cpm_rmm(BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports) endfunction() find_and_configure_rmm() diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 8424acf697..aad89e0c06 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -43,7 +43,7 @@ set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) # --- RAFT ---# -set(RAFT_BUILD_STATIC_LIB OFF) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) set(RAFT_COMPILE_LIBRARY ON) # --- CUDA --- # diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index a7c7f7cc10..83c262dc10 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,20 +27,7 @@ project( LANGUAGES CXX CUDA ) -# CCCL before rmm/cuco so we get the right version of CCCL -# TODO(jameslamb): is this necessary? -include(rapids-cmake) -include(rapids-cpm) -include(rapids-export) -include(rapids-find) -# add third party dependencies using CPM -rapids_cpm_init() -include(../../cpp/cmake/thirdparty/get_cccl.cmake) -include(../../cpp/cmake/thirdparty/get_rmm.cmake) - -# TODO(jameslamb): remove this... just testing my understanding -# include(../../cpp/cmake/thirdparty/get_cutlass.cmake) - +# an installed version of raft contains the other necessary targets (like CCCL and cuco) find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft compiled) include(rapids-cython-core) diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 0311912bc5..c8c5c5ba83 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -26,7 +26,6 @@ project( ) rapids_cpm_init() -# TODO(jameslamb): could this just use the libucxx wheel? # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) @@ -34,12 +33,9 @@ include(cmake/thirdparty/get_ucxx.cmake) # why these components: # # * 'raft' = the headers, needed to link against libraft -# * 'distributed = needed for 'nccl' +# * 'distributed' = needed for NCCL find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS raft distributed) -# list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) -# find_package(NCCL REQUIRED) - include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index f5daf70f92..e6b9c4aa0e 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -45,8 +45,8 @@ function(find_and_configure_ucxx) endfunction() # Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft +# To use a different ucxx locally, set the CMake variable +# CPM_ucxx_SOURCE=/path/to/local/ucxx find_and_configure_ucxx(VERSION 0.42 FORK rapidsai PINNED_TAG branch-0.42 From c224e27c59ca2e1e76c930f2aec89f94917798d0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 10 Jan 2025 08:28:41 -0800 Subject: [PATCH 23/25] more fixes --- build.sh | 3 +-- ci/validate_wheel.sh | 7 +++---- dependencies.yaml | 9 +-------- python/libraft/CMakeLists.txt | 4 +--- python/libraft/libraft/__init__.py | 2 +- python/libraft/libraft/_version.py | 2 +- python/libraft/libraft/load.py | 2 +- python/libraft/pyproject.toml | 3 +-- python/raft-dask/pyproject.toml | 1 + 9 files changed, 11 insertions(+), 22 deletions(-) diff --git a/build.sh b/build.sh index 671625a65f..de3ebfa3c5 100755 --- a/build.sh +++ b/build.sh @@ -348,8 +348,7 @@ if [[ ${CMAKE_TARGET} == "" ]]; then fi # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') +SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${EXTRA_CMAKE_ARGS} | sed 's/ /;/g') # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 7f1dae43f2..ca506af004 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,18 +10,17 @@ package_name=$3 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 +PYDISTCHECK_ARGS=() if [[ "${package_name}" == "libraft" ]]; then if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then - PYDISTCHECK_ARGS=( + PYDISTCHECK_ARGS+=( --max-allowed-size-compressed '750M' ) else - PYDISTCHECK_ARGS=( + PYDISTCHECK_ARGS+=( --max-allowed-size-compressed '100M' ) fi -else - PYDISTCHECK_ARGS=() fi cd "${package_dir}" diff --git a/dependencies.yaml b/dependencies.yaml index 884347de57..7bbd91f2fe 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -124,15 +124,9 @@ files: extras: table: tool.rapids-build-backend key: requires - # TODO(jameslamb): use libucxx? includes: - depends_on_libraft - # CMake Error at /pyenv/versions/3.12.7/lib/python3.12/site-packages/librmm/lib64/cmake/rmm/rmm-targets.cmake:42 (message): - # Some (but not all) targets in this export set were already defined. - # Targets Defined: rmm::rmm - # Targets not yet defined: rmm::rmm_logger, rmm::rmm_logger_im - # - depends_on_librmm - - rapids_build + - depends_on_librmm - depends_on_ucx_build - rapids_build py_run_raft_dask: @@ -537,7 +531,6 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index aad89e0c06..ab6810f842 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,8 +27,6 @@ project( option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) -# find_package(CUDAToolkit REQUIRED) - # Check if raft is already available. If so, it is the user's responsibility to ensure that the # CMake package is also available at build time of the Python raft package. find_package(raft "${RAPIDS_VERSION}") diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py index 2df69256fa..9260f4e67c 100644 --- a/python/libraft/libraft/__init__.py +++ b/python/libraft/libraft/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py index 7dd732b490..530bf8bea6 100644 --- a/python/libraft/libraft/_version.py +++ b/python/libraft/libraft/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py index 8cc9035fa9..ad3db9e09c 100644 --- a/python/libraft/libraft/load.py +++ b/python/libraft/libraft/load.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 241acda1de..f47eddf759 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -88,7 +88,6 @@ cmake.build-type = "Release" cmake.version = "CMakeLists.txt" minimum-version = "build-system.requires" ninja.make-fallback = true -sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.install-dir = "libraft" wheel.packages = ["libraft"] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index a622bf6e2a..d3a26db282 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -121,6 +121,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0,<3.1.0a0", "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 76d788bfb688335e9d0fab3ee83afc71a1599a7c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 10 Jan 2025 09:43:35 -0800 Subject: [PATCH 24/25] clean up dependencies --- .github/workflows/build.yaml | 3 +++ dependencies.yaml | 21 +++++++++++++-------- python/libraft/CMakeLists.txt | 8 +------- python/libraft/pyproject.toml | 2 -- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d00ba45f58..af6cb910e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -87,6 +87,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: libraft + package-type: cpp wheel-build-pylibraft: needs: wheel-publish-libraft secrets: inherit @@ -107,6 +108,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibraft + package-type: python wheel-build-raft-dask: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 @@ -126,3 +128,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: raft_dask + package-type: python diff --git a/dependencies.yaml b/dependencies.yaml index 7bbd91f2fe..44c240b6ce 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,6 +6,8 @@ files: cuda: ["11.8", "12.5"] arch: [x86_64, aarch64] includes: + - build_common + - build_cython - checks - cuda - cuda_version @@ -15,7 +17,6 @@ files: - depends_on_rmm - develop - docs - - rapids_build - rapids_build_skbuild - run_pylibraft - run_raft_dask @@ -62,9 +63,8 @@ files: table: tool.rapids-build-backend key: requires includes: - - depends_on_cuda_python + - build_common - depends_on_librmm - - rapids_build py_run_libraft: output: pyproject pyproject_dir: python/libraft @@ -86,11 +86,12 @@ files: table: tool.rapids-build-backend key: requires includes: + - build_common + - build_cython - depends_on_libraft - depends_on_librmm - depends_on_cuda_python - depends_on_rmm - - rapids_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -125,10 +126,11 @@ files: table: tool.rapids-build-backend key: requires includes: + - build_common + - build_cython - depends_on_libraft - depends_on_librmm - depends_on_ucx_build - - rapids_build py_run_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -164,12 +166,11 @@ dependencies: - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.10.0 - rapids_build: + build_common: common: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 - - cython>=3.0.0,<3.1.0a0 - ninja - output_types: [conda] packages: @@ -211,7 +212,11 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - + build_cython: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0,<3.1.0a0 checks: common: - output_types: [conda, requirements] diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index ab6810f842..9cbb87caf4 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -16,13 +16,10 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../rapids_config.cmake) -# include(rapids-cuda) -# rapids_cuda_init_architectures(libraft-python) - project( libraft-python VERSION "${RAPIDS_VERSION}" - LANGUAGES CXX CUDA + LANGUAGES CXX ) option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) @@ -54,9 +51,6 @@ elseif(USE_CUDA_MATH_WHEELS) message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") endif() -# ensure installs are inside the libraft/ folder in site-packages/ -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) - add_subdirectory(../../cpp raft-cpp) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index f47eddf759..549a1bf651 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -102,8 +102,6 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", - "cuda-python", - "cython>=3.0.0,<3.1.0a0", "librmm==25.2.*,>=0.0.0a0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From b2a5107373a67403546d7455ce5469524bf66197 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 13 Jan 2025 21:00:44 -0800 Subject: [PATCH 25/25] uncomment more CI --- .github/workflows/pr.yaml | 38 +++++++++++++++++------------------ ci/build_wheel_pylibraft.sh | 2 +- ci/build_wheel_raft_dask.sh | 2 +- python/libraft/CMakeLists.txt | 13 ++++++------ 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 38be1e1f36..dbb40d1cb0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,10 +17,10 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - # - conda-cpp-checks + - conda-cpp-checks - conda-python-build - conda-python-tests - # - docs-build + - docs-build - wheel-build-libraft - wheel-build-pylibraft - wheel-tests-pylibraft @@ -87,13 +87,13 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request - # conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 - # with: - # build_type: pull-request - # enable_check_symbols: true + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 + with: + build_type: pull-request + enable_check_symbols: true conda-python-build: needs: conda-cpp-build secrets: inherit @@ -107,16 +107,16 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request - # docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-libraft: needs: checks secrets: inherit diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index b8da15ffaf..f88aaaa6e1 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -17,7 +17,7 @@ echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft export PIP_CONSTRAINT="/tmp/constraints.txt" # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF" ci/build_wheel.sh pylibraft ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist pylibraft diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index afd286006d..79334c9f45 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -17,7 +17,7 @@ echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libraft_dist/libraft export PIP_CONSTRAINT="/tmp/constraints.txt" # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF" ci/build_wheel.sh raft-dask ${package_dir} python ci/validate_wheel.sh ${package_dir} final_dist raft-dask diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 9cbb87caf4..57efcd61ab 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -34,13 +34,6 @@ endif() unset(raft_FOUND) -set(BUILD_TESTS OFF) -set(BUILD_PRIMS_BENCH OFF) - -# --- RAFT ---# -set(RAFT_COMPILE_DYNAMIC_ONLY ON) -set(RAFT_COMPILE_LIBRARY ON) - # --- CUDA --- # find_package(CUDAToolkit REQUIRED) set(CUDA_STATIC_RUNTIME ON) @@ -51,6 +44,12 @@ elseif(USE_CUDA_MATH_WHEELS) message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") endif() +# --- RAFT ---# +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) +set(RAFT_COMPILE_LIBRARY ON) + add_subdirectory(../../cpp raft-cpp) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS)