From 2898e786576f04f3a813ba3dca695fe9d585dd25 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Dec 2024 15:12:26 -0800 Subject: [PATCH 01/29] distribute libcugraph wheels --- .github/workflows/build.yaml | 22 +- .github/workflows/pr.yaml | 218 ++++++++++--------- .gitignore | 7 +- ci/build_wheel.sh | 3 +- ci/build_wheel_cugraph.sh | 15 +- ci/build_wheel_libcugraph.sh | 44 ++++ ci/build_wheel_pylibcugraph.sh | 19 +- ci/release/update-version.sh | 1 + ci/test_wheel.sh | 9 - ci/test_wheel_cugraph.sh | 13 +- ci/test_wheel_pylibcugraph.sh | 10 + ci/validate_wheel.sh | 2 +- dependencies.yaml | 82 ++++++- python/cugraph/CMakeLists.txt | 45 +--- python/cugraph/cugraph/__init__.py | 11 + python/cugraph/pyproject.toml | 2 + python/libcugraph/CMakeLists.txt | 70 ++++++ python/libcugraph/LICENSE | 1 + python/libcugraph/README.md | 1 + python/libcugraph/libcugraph/VERSION | 1 + python/libcugraph/libcugraph/__init__.py | 16 ++ python/libcugraph/libcugraph/_version.py | 30 +++ python/libcugraph/libcugraph/load.py | 78 +++++++ python/libcugraph/pyproject.toml | 82 +++++++ python/pylibcugraph/pylibcugraph/__init__.py | 11 + python/pylibcugraph/pyproject.toml | 2 + 26 files changed, 621 insertions(+), 174 deletions(-) create mode 100755 ci/build_wheel_libcugraph.sh create mode 100644 python/libcugraph/CMakeLists.txt create mode 120000 python/libcugraph/LICENSE create mode 120000 python/libcugraph/README.md create mode 120000 python/libcugraph/libcugraph/VERSION create mode 100644 python/libcugraph/libcugraph/__init__.py create mode 100644 python/libcugraph/libcugraph/_version.py create mode 100644 python/libcugraph/libcugraph/load.py create mode 100644 python/libcugraph/pyproject.toml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 357d0d9bcae..0bf7960e543 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,28 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libcugraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcugraph.sh + node_type: cpu32 + wheel-publish-libcugraph: + needs: wheel-publish-libcugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcugraph wheel-build-pylibcugraph: + needs: wheel-publish-libcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: @@ -76,7 +97,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_pylibcugraph.sh - node_type: cpu32 wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 633d964a3bf..02e6d726c92 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,35 +15,36 @@ jobs: needs: - changed-files - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-notebook-tests - - conda-python-build - - conda-python-tests + # - conda-cpp-build + # - conda-cpp-tests + # - conda-cpp-checks + # - conda-notebook-tests + # - conda-python-build + # - conda-python-tests - docs-build + - wheel-build-libcugraph - wheel-build-pylibcugraph - - wheel-tests-pylibcugraph + # - wheel-tests-pylibcugraph - wheel-build-cugraph - - wheel-tests-cugraph - - telemetry-setup + # - wheel-tests-cugraph + # - telemetry-setup - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() with: needs: ${{ toJSON(needs) }} - telemetry-setup: - runs-on: ubuntu-latest - continue-on-error: true - env: - OTEL_SERVICE_NAME: "pr-cugraph" - steps: - - name: Telemetry setup - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main + # telemetry-setup: + # runs-on: ubuntu-latest + # continue-on-error: true + # env: + # OTEL_SERVICE_NAME: "pr-cugraph" + # steps: + # - name: Telemetry setup + # uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main changed-files: secrets: inherit - needs: telemetry-setup + # needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.02 with: files_yaml: | @@ -74,101 +75,110 @@ jobs: - '!notebooks/**' checks: secrets: inherit - needs: telemetry-setup + # needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 with: enable_check_generated_files: false ignored_pr_jobs: telemetry-summarize - conda-cpp-build: + # conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + # with: + # build_type: pull-request + # node_type: cpu32 + # conda-cpp-tests: + # needs: [conda-cpp-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 + # with: + # build_type: pull-request + # enable_check_symbols: true + # symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 + # with: + # build_type: pull-request + # conda-python-tests: + # needs: [conda-python-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # conda-notebook-tests: + # needs: [conda-python-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + # run_script: "ci/test_notebooks.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + # run_script: "ci/build_docs.sh" + wheel-build-libcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request + script: ci/build_wheel_libcugraph.sh node_type: cpu32 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - conda-notebook-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: - needs: checks + needs: wheel-build-libcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh - node_type: cpu32 - wheel-tests-pylibcugraph: - needs: [wheel-build-pylibcugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_pylibcugraph.sh + # wheel-tests-pylibcugraph: + # needs: [wheel-build-pylibcugraph, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: - needs: wheel-tests-pylibcugraph + needs: [wheel-build-libcugraph] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_cugraph.sh - wheel-tests-cugraph: - needs: [wheel-build-cugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph.sh + # wheel-tests-cugraph: + # needs: [wheel-build-cugraph, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel_cugraph.sh devcontainer: secrets: inherit - needs: telemetry-setup + # needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 with: arch: '["amd64"]' @@ -178,17 +188,17 @@ jobs: sccache -z; build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; sccache -s; - telemetry-summarize: - runs-on: ubuntu-latest - needs: pr-builder - if: always() - continue-on-error: true - steps: - - name: Load stashed telemetry env vars - uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main - with: - load_service_name: true - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main - with: - cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}" + # telemetry-summarize: + # runs-on: ubuntu-latest + # needs: pr-builder + # if: always() + # continue-on-error: true + # steps: + # - name: Load stashed telemetry env vars + # uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main + # with: + # load_service_name: true + # - name: Telemetry summarize + # uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main + # with: + # cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}" diff --git a/.gitignore b/.gitignore index 9480c2618bf..fe6dc4b1031 100644 --- a/.gitignore +++ b/.gitignore @@ -32,17 +32,16 @@ test-results ## Python build directories & artifacts dask-worker-space/ htmlcov +*.conda dist/ -*.egg-info/ +.egg-info/ python/build python/cugraph/bindings/*.cpp wheels/ wheelhouse/ _skbuild/ cufile.log - -## pylibcugraph build directories & artifacts -python/pylibcugraph/pylibcugraph.egg-info +*.whl ## Patching *.diff diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9a77e6b3021..7fbb1a702a1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 source rapids-configure-sccache source rapids-date-string @@ -47,4 +48,4 @@ esac mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 11ab9acc58c..9e444bcc0ae 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -7,16 +7,21 @@ package_dir="python/cugraph" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph wheel built in the previous step and make it +# Download the libcugraph and pylibcugraph wheesl built in the previous steps and make them # available for pip to find. # -# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run +# ensure 'cugraph' wheel builds always use the 'libcugraph' and 'pylibcugraph' just built in the same CI run # # using env variable PIP_CONSTRAINT is necessary to ensure the constraints # are used when creating the isolated build environment -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) +LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) +PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) + +cat > ./constraints.txt < ./constraints.txt export PIP_CONSTRAINT="${PWD}/constraints.txt" PARALLEL_LEVEL=$(python -c \ @@ -33,5 +38,5 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" -./ci/build_wheel.sh cugraph ${package_dir} +./ci/build_wheel.sh cugraph ${package_dir} python ./ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh new file mode 100755 index 00000000000..1d0a53b9ad1 --- /dev/null +++ b/ci/build_wheel_libcugraph.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libcugraph" +package_dir="python/libcugraph" + +rapids-logger "Generating build requirements" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "${matrix_selectors}" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" +export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" + +./ci/build_wheel.sh libcugraph ${package_dir} cpp +./ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index 8bdfa20ee77..33b214f91d2 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -5,6 +5,23 @@ set -euo pipefail package_dir="python/pylibcugraph" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Download the libcugraph and wheel built in the previous step and make it +# available for pip to find. +# +# ensure 'cugraph' wheel builds always use the 'libcugraph' just built in the same CI run +# +# using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment +LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) + +cat > ./constraints.txt <=0.0.0a0 - libraft-headers==25.2.*,>=0.0.0a0 - libraft==25.2.*,>=0.0.0a0 - - librmm==25.2.*,>=0.0.0a0 - openmpi # Required for building cpp-mgtests (multi-GPU tests) specific: - output_types: [conda] @@ -554,6 +584,31 @@ dependencies: - pylibwholegraph-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_librmm: + common: + - output_types: conda + packages: + - &librmm_unsuffixed librmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} + depends_on_rmm: common: - output_types: conda @@ -679,6 +734,31 @@ dependencies: - raft-dask-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*raft_dask_unsuffixed]} + depends_on_libcugraph: + common: + - output_types: conda + packages: + - &libcugraph_unsuffixed libcugraph==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcugraph-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcugraph-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libcugraph_unsuffixed]} + depends_on_pylibcugraph: common: - output_types: conda diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 117f7cf5142..30eb883aa08 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -29,55 +29,12 @@ project( ################################################################################ # - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - -# If the user requested it, we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) -if(NOT cugraph_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set(rpaths - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/cufft/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - set(cython_lib_dir cugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(cugraph/components) diff --git a/python/cugraph/cugraph/__init__.py b/python/cugraph/cugraph/__init__.py index ada1fec74cb..4f3980e71e7 100644 --- a/python/cugraph/cugraph/__init__.py +++ b/python/cugraph/cugraph/__init__.py @@ -11,6 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + from cugraph.community import ( ecg, induced_subgraph, diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 9a465d789e2..c2fb3a06e08 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "dask-cudf==25.2.*,>=0.0.0a0", "fsspec[http]>=0.6.0", + "libcugraph==25.2.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<3.0a0", "pylibcugraph==25.2.*,>=0.0.0a0", @@ -81,6 +82,7 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", + "libcugraph==25.2.*,>=0.0.0a0", "ninja", "pylibcugraph==25.2.*,>=0.0.0a0", "pylibraft==25.2.*,>=0.0.0a0", diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt new file mode 100644 index 00000000000..10121d4363e --- /dev/null +++ b/python/libcugraph/CMakeLists.txt @@ -0,0 +1,70 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcugraph-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF) + +# Check if cugraph is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cugraph package. +find_package(cugraph "${RAPIDS_VERSION}") + +if(cugraph_FOUND) + return() +endif() + +unset(cugraph_FOUND) + +# Find Python early so that later commands can use it +find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) + +set(BUILD_TESTS OFF) +set(BUILD_CUGRAPH_MG_TESTS OFF) +set(CMAKE_CUDA_LINEINFO OFF) +set(CUDA_STATIC_RUNTIME ON) +set(CUDA_STATIC_MATH_LIBRARIES ON) +set(USE_RAFT_STATIC ON) +set(CUGRAPH_COMPILE_RAFT_LIB ON) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + +find_package(CUDAToolkit REQUIRED) + +if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) +elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") +endif() + +add_subdirectory(../../cpp cugraph-cpp) + +if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set(rpaths + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/cufft/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) + set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) +endif() diff --git a/python/libcugraph/LICENSE b/python/libcugraph/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/libcugraph/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcugraph/README.md b/python/libcugraph/README.md new file mode 120000 index 00000000000..fe840054137 --- /dev/null +++ b/python/libcugraph/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcugraph/libcugraph/VERSION b/python/libcugraph/libcugraph/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/libcugraph/libcugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libcugraph/libcugraph/__init__.py b/python/libcugraph/libcugraph/__init__.py new file mode 100644 index 00000000000..87b7916c8a3 --- /dev/null +++ b/python/libcugraph/libcugraph/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcugraph._version import __git_commit__, __version__ +from libcugraph.load import load_library diff --git a/python/libcugraph/libcugraph/_version.py b/python/libcugraph/libcugraph/_version.py new file mode 100644 index 00000000000..a5171f19f4b --- /dev/null +++ b/python/libcugraph/libcugraph/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py new file mode 100644 index 00000000000..4daaa11c22c --- /dev/null +++ b/python/libcugraph/libcugraph/load.py @@ -0,0 +1,78 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcudf that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile(lib := os.path.join(os.path.dirname(__file__), "lib64", soname)): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcugraph.so and its dependencies""" + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUGRAPH_PREFER_SYSTEM_LIBRARY", "false").lower() != "false" + ) + + soname = "libcugraph.so" + libcugraph_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcugraph_lib = _load_system_installation(soname) + except OSError: + libcugraph_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libcugraph_lib = _load_wheel_installation(soname) + if libcugraph_lib is None: + libcugraph_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcugraph was loaded from. + return libcugraph_lib diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml new file mode 100644 index 00000000000..5f680c44eef --- /dev/null +++ b/python/libcugraph/pyproject.toml @@ -0,0 +1,82 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "cython>=3.0.0", + "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcugraph" +dynamic = ["version"] +description = "cuGraph - RAPIDS GPU Graph Analytics (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "rmm==25.2.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cugraph" + +[project.entry-points."cmake.prefix"] +libcugraph = "libcugraph" + +[tool.pydistcheck] +select = [ + # NOTE: size threshold is managed via CLI args in CI scripts + "distro-too-large-compressed", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcugraph"] +wheel.install-dir = "libcugraph" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcugraph/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "librmm==25.2.*,>=0.0.0a0", + "ninja", + "pylibraft==25.2.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 9047144c13a..ccd9c73ac6c 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -11,6 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + from pylibcugraph.components._connectivity import ( strongly_connected_components, ) diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 72a5e19c702..1b70fd89916 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -23,6 +23,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ + "libcugraph==25.2.*,>=0.0.0a0", "nvidia-cublas", "nvidia-curand", "nvidia-cusolver", @@ -73,6 +74,7 @@ build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4,!=3.30.0", + "libcugraph==25.2.*,>=0.0.0a0", "ninja", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", From 0ecd5a5a2942e84da2a9e5a570e68691f27da619 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 4 Dec 2024 15:44:32 -0800 Subject: [PATCH 02/29] exclude libcugraph.so from cugraph/pylibcugraph --- ci/build_wheel.sh | 15 +++++++++ cpp/cmake/thirdparty/get_raft.cmake | 2 +- python/cugraph/CMakeLists.txt | 4 --- python/pylibcugraph/CMakeLists.txt | 52 +---------------------------- 4 files changed, 17 insertions(+), 56 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 7fbb1a702a1..afe20bfeaea 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -46,6 +46,21 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac +case "${package_dir}" in + python/pylibcugraph) + EXCLUDE_ARGS+=( + --exclude "libcugraph_c.so" + --exclude "libcugraph.so" + ) + ;; + python/cugraph) + EXCLUDE_ARGS+=( + --exclude "libcugraph_c.so" + --exclude "libcugraph.so" + ) + ;; +esac + mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 8f56372c81a..c04ed2caf71 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -44,7 +44,7 @@ function(find_and_configure_raft) INSTALL_EXPORT_SET cugraph-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS - EXCLUDE_FROM_ALL TRUE + # EXCLUDE_FROM_ALL TRUE GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git GIT_TAG ${PKG_PINNED_TAG} SOURCE_SUBDIR cpp diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 30eb883aa08..39e26c84339 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -46,7 +46,3 @@ add_subdirectory(cugraph/linear_assignment) add_subdirectory(cugraph/structure) add_subdirectory(cugraph/tree) add_subdirectory(cugraph/utilities) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 22d788ff253..1b78714b32f 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -27,60 +27,10 @@ project( LANGUAGES CXX CUDA ) -################################################################################ -# - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - -# If the user requested it we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) -if (NOT cugraph_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set(rpaths - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - set(cython_lib_dir pylibcugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) - install(TARGETS cugraph_c DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibcugraph) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() From f1645ac7cfc480108f93c65905326d5173497e25 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 5 Dec 2024 09:01:59 -0800 Subject: [PATCH 03/29] fix pr-builder --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 02e6d726c92..2c257567f8e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,7 +21,7 @@ jobs: # - conda-notebook-tests # - conda-python-build # - conda-python-tests - - docs-build + # - docs-build - wheel-build-libcugraph - wheel-build-pylibcugraph # - wheel-tests-pylibcugraph From 6e1d3d93ee4ad75d3cc932ce2bd2d2a9d32e5784 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 5 Dec 2024 11:46:10 -0800 Subject: [PATCH 04/29] remove more cugraph-ops, fix variables --- build.sh | 5 ----- ci/build_wheel_cugraph.sh | 5 +---- ci/build_wheel_libcugraph.sh | 5 ++++- ci/build_wheel_pylibcugraph.sh | 3 +-- ci/validate_wheel.sh | 1 + python/cugraph/pyproject.toml | 4 +++- python/pylibcugraph/pyproject.toml | 4 +++- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index 0282948659e..20611e58bfa 100755 --- a/build.sh +++ b/build.sh @@ -175,11 +175,6 @@ SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') -# Append `-DFIND_CUGRAPH_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUGRAPH_CPP=ON" -fi - # If clean or uninstall targets given, run them prior to any other steps if hasArg uninstall; then if [[ "$INSTALL_PREFIX" != "" ]]; then diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 9e444bcc0ae..099c93e5228 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -24,8 +24,6 @@ EOF export PIP_CONSTRAINT="${PWD}/constraints.txt" -PARALLEL_LEVEL=$(python -c \ - "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -35,8 +33,7 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF${EXTRA_CMAKE_ARGS}" -export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF${EXTRA_CMAKE_ARGS}" ./ci/build_wheel.sh cugraph ${package_dir} python ./ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 1d0a53b9ad1..d5be5acebee 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -28,6 +28,9 @@ export PIP_NO_BUILD_ISOLATION=0 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +PARALLEL_LEVEL=$(python -c \ + "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -37,7 +40,7 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" ./ci/build_wheel.sh libcugraph ${package_dir} cpp diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index 33b214f91d2..f1f3ceacd0b 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -34,8 +34,7 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF${EXTRA_CMAKE_ARGS}" -export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF${EXTRA_CMAKE_ARGS}" ./ci/build_wheel.sh pylibcugraph ${package_dir} python ./ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5cbdef9e2d6..f33d691ea5e 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,6 +10,7 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 if [[ "${package_dir}" == "python/libcugraph" ]]; then + # TODO: before merging, modify these thresholds if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( --max-allowed-size-compressed '1.5G' diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index c2fb3a06e08..ceff4aeca89 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -93,6 +93,8 @@ matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] + +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 1b70fd89916..be5efda7e70 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -83,6 +83,8 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] + +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' From 3aa693c7e68b3b8f90e5da6d92d7ba1925f6cabc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 08:33:58 -0800 Subject: [PATCH 05/29] remove Python interpreter stuff --- python/libcugraph/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index 10121d4363e..545b490e147 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -34,9 +34,6 @@ endif() unset(cugraph_FOUND) -# Find Python early so that later commands can use it -find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) - set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) set(CMAKE_CUDA_LINEINFO OFF) @@ -45,7 +42,7 @@ set(CUDA_STATIC_MATH_LIBRARIES ON) set(USE_RAFT_STATIC ON) set(CUGRAPH_COMPILE_RAFT_LIB ON) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) +# set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) find_package(CUDAToolkit REQUIRED) From c5cb53df38da35ebb624f7a65a0c6a4118e4bdc0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 08:34:55 -0800 Subject: [PATCH 06/29] temporarily raise size threshold --- ci/validate_wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index f33d691ea5e..1b8ce4b57ee 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -13,11 +13,11 @@ if [[ "${package_dir}" == "python/libcugraph" ]]; then # TODO: before merging, modify these thresholds if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '1.5G' + --max-allowed-size-compressed '5.0G' ) else PYDISTCHECK_ARGS=( - --max-allowed-size-compressed '975M' + --max-allowed-size-compressed '5.0G' ) fi else From e73606c65bcf1c673db16d7b22e2909518ebad55 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 10:11:20 -0800 Subject: [PATCH 07/29] declare librmm dependency, fix CI dependency tree --- .github/workflows/pr.yaml | 2 +- dependencies.yaml | 2 ++ python/cugraph/pyproject.toml | 1 + python/pylibcugraph/pyproject.toml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2c257567f8e..94fea971466 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -162,7 +162,7 @@ jobs: # build_type: pull-request # script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: - needs: [wheel-build-libcugraph] + needs: [wheel-build-pylibcugraph] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: diff --git a/dependencies.yaml b/dependencies.yaml index 91239803049..a3ab343f076 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -81,6 +81,7 @@ files: includes: - common_build - depends_on_libcugraph + - depends_on_librmm - depends_on_rmm - depends_on_pylibraft - depends_on_pylibcugraph @@ -151,6 +152,7 @@ files: includes: - common_build - depends_on_libcugraph + - depends_on_librmm - depends_on_rmm - depends_on_pylibraft py_run_pylibcugraph: diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index ceff4aeca89..ef989c3c0c8 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -83,6 +83,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "libcugraph==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibcugraph==25.2.*,>=0.0.0a0", "pylibraft==25.2.*,>=0.0.0a0", diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index be5efda7e70..5f7c7dd409c 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -75,6 +75,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4,!=3.30.0", "libcugraph==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", From fc3702084c68e3db085da85ab2c7d50375679d95 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 10:57:51 -0800 Subject: [PATCH 08/29] run a subet of wheel-testing CI --- .github/workflows/pr.yaml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 94fea971466..e76f1efb1c0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: - wheel-build-pylibcugraph # - wheel-tests-pylibcugraph - wheel-build-cugraph - # - wheel-tests-cugraph + - wheel-tests-cugraph # - telemetry-setup - devcontainer secrets: inherit @@ -168,14 +168,16 @@ jobs: with: build_type: pull-request script: ci/build_wheel_cugraph.sh - # wheel-tests-cugraph: - # needs: [wheel-build-cugraph, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - # with: - # build_type: pull-request - # script: ci/test_wheel_cugraph.sh + wheel-tests-cugraph: + needs: [wheel-build-cugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph.sh + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] devcontainer: secrets: inherit # needs: telemetry-setup From e80d5664fe4d034496f5109f6bba82ce98cd0bd6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 10:58:30 -0800 Subject: [PATCH 09/29] skip devcontainer jobs --- .github/workflows/pr.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e76f1efb1c0..565142491a4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -28,7 +28,7 @@ jobs: - wheel-build-cugraph - wheel-tests-cugraph # - telemetry-setup - - devcontainer + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() @@ -178,18 +178,18 @@ jobs: script: ci/test_wheel_cugraph.sh # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] - devcontainer: - secrets: inherit - # needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - node_type: cpu32 - build_command: | - sccache -z; - build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; - sccache -s; + # devcontainer: + # secrets: inherit + # # needs: telemetry-setup + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # node_type: cpu32 + # build_command: | + # sccache -z; + # build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; + # sccache -s; # telemetry-summarize: # runs-on: ubuntu-latest # needs: pr-builder From 695f627dc41037a6c9dce710c3c52f753893673f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 11:00:55 -0800 Subject: [PATCH 10/29] skip 'checks' job too --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 565142491a4..170aa2bfc6a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,7 +14,7 @@ jobs: pr-builder: needs: - changed-files - - checks + # - checks # - conda-cpp-build # - conda-cpp-tests # - conda-cpp-checks @@ -73,13 +73,13 @@ jobs: - '!docs/**' - '!img/**' - '!notebooks/**' - checks: - secrets: inherit - # needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 - with: - enable_check_generated_files: false - ignored_pr_jobs: telemetry-summarize + # checks: + # secrets: inherit + # # needs: telemetry-setup + # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 + # with: + # enable_check_generated_files: false + # ignored_pr_jobs: telemetry-summarize # conda-cpp-build: # needs: checks # secrets: inherit @@ -137,7 +137,7 @@ jobs: # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" # run_script: "ci/build_docs.sh" wheel-build-libcugraph: - needs: checks + # needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: From 486bf96ea297891aaa3ec7acf8ad1bbe7ed1762f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 11:29:24 -0800 Subject: [PATCH 11/29] just run Python 3.12 builds --- .github/workflows/pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 170aa2bfc6a..bc2434a5d9e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -153,6 +153,9 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh + # build for every combination of arch and CUDA version, but only for the latest Python + # TODO: remove this before merging + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) # wheel-tests-pylibcugraph: # needs: [wheel-build-pylibcugraph, changed-files] # secrets: inherit @@ -168,6 +171,9 @@ jobs: with: build_type: pull-request script: ci/build_wheel_cugraph.sh + # build for every combination of arch and CUDA version, but only for the latest Python + # TODO: remove this before merging + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-tests-cugraph: needs: [wheel-build-cugraph, changed-files] secrets: inherit From fce4fcf20de3d28808a3dcb00d092070f44d8292 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 13:20:09 -0800 Subject: [PATCH 12/29] move CUDA libs deps to wheels, ignore deprecation warning --- dependencies.yaml | 2 +- python/cugraph/pytest.ini | 1 + python/libcugraph/pyproject.toml | 4 ++++ python/pylibcugraph/pyproject.toml | 4 ---- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index a3ab343f076..51a0f5e0c28 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -134,6 +134,7 @@ files: extras: table: project includes: + - cuda_wheels - depends_on_rmm py_build_pylibcugraph: output: pyproject @@ -161,7 +162,6 @@ files: extras: table: project includes: - - cuda_wheels - depends_on_libcugraph - depends_on_rmm - depends_on_pylibraft diff --git a/python/cugraph/pytest.ini b/python/cugraph/pytest.ini index bf6e6bdd802..7bdef2d9771 100644 --- a/python/cugraph/pytest.ini +++ b/python/cugraph/pytest.ini @@ -62,6 +62,7 @@ filterwarnings = error::FutureWarning error::DeprecationWarning # TODO + ignore:.*cuda..* module is deprecated.*:DeprecationWarning ignore:Multi is deprecated and the removal of multi edges will no longer be supported:FutureWarning ignore:The legacy column names:FutureWarning ignore:The include_hop_column flag is deprecated and will be removed:FutureWarning diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 5f680c44eef..4f4b9ebbb37 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -39,6 +39,10 @@ classifiers = [ "Environment :: GPU :: NVIDIA CUDA", ] dependencies = [ + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 5f7c7dd409c..b0175b16ed5 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -24,10 +24,6 @@ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ "libcugraph==25.2.*,>=0.0.0a0", - "nvidia-cublas", - "nvidia-curand", - "nvidia-cusolver", - "nvidia-cusparse", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 796b54ead9cf46f00b3b743ff5a0148ee7d79aef Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Dec 2024 13:49:24 -0800 Subject: [PATCH 13/29] more dependency fiddling --- dependencies.yaml | 1 - python/libcugraph/pyproject.toml | 3 +-- python/pylibcugraph/pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 51a0f5e0c28..22de9773dc9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -126,7 +126,6 @@ files: key: requires includes: - common_build - - depends_on_pylibraft - depends_on_librmm py_run_libcugraph: output: pyproject diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 4f4b9ebbb37..8be028e1cd1 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -77,10 +77,9 @@ regex = "(?P.*)" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.26.4,!=3.30.0", "librmm==25.2.*,>=0.0.0a0", "ninja", - "pylibraft==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index b0175b16ed5..d7738b56074 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -76,7 +76,7 @@ requires = [ "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ From 157f63ed4ada7dd5057847ce24b4e1252f7f97b3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Dec 2024 08:03:47 -0800 Subject: [PATCH 14/29] fix CI configs, remove unnecessary stuff from pylibcugraph CMake --- .github/workflows/pr.yaml | 6 ------ python/libcugraph/CMakeLists.txt | 2 -- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bc2434a5d9e..170aa2bfc6a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -153,9 +153,6 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh - # build for every combination of arch and CUDA version, but only for the latest Python - # TODO: remove this before merging - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) # wheel-tests-pylibcugraph: # needs: [wheel-build-pylibcugraph, changed-files] # secrets: inherit @@ -171,9 +168,6 @@ jobs: with: build_type: pull-request script: ci/build_wheel_cugraph.sh - # build for every combination of arch and CUDA version, but only for the latest Python - # TODO: remove this before merging - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-tests-cugraph: needs: [wheel-build-cugraph, changed-files] secrets: inherit diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index 545b490e147..9624d0bcb92 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -22,8 +22,6 @@ project( LANGUAGES CXX ) -option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF) - # Check if cugraph is already available. If so, it is the user's responsibility to ensure that the # CMake package is also available at build time of the Python cugraph package. find_package(cugraph "${RAPIDS_VERSION}") From 4b4c4b2fe1d8b799288bd5c157f061636ba14772 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Dec 2024 10:02:29 -0800 Subject: [PATCH 15/29] load libcugraph_c too --- python/libcugraph/libcugraph/load.py | 48 +++++++++++++++------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py index 4daaa11c22c..c0cf71e9008 100644 --- a/python/libcugraph/libcugraph/load.py +++ b/python/libcugraph/libcugraph/load.py @@ -48,31 +48,33 @@ def load_library(): os.getenv("RAPIDS_LIBCUGRAPH_PREFER_SYSTEM_LIBRARY", "false").lower() != "false" ) - soname = "libcugraph.so" - libcugraph_lib = None - if prefer_system_installation: - # Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. - try: - libcugraph_lib = _load_system_installation(soname) - except OSError: - libcugraph_lib = _load_wheel_installation(soname) - else: - # Prefer the libraries bundled in this package. If they aren't found - # (which might be the case in builds where the library was prebuilt before - # packaging the wheel), look for a system installation. - try: - libcugraph_lib = _load_wheel_installation(soname) - if libcugraph_lib is None: + for soname in ["libcugraph.so", "libcugraph_c.so"]: + libcugraph_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: libcugraph_lib = _load_system_installation(soname) - except OSError: - # If none of the searches above succeed, just silently return None - # and rely on other mechanisms (like RPATHs on other DSOs) to - # help the loader find the library. - pass + except OSError: + libcugraph_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libcugraph_lib = _load_wheel_installation(soname) + if libcugraph_lib is None: + libcugraph_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle # to inspect where libcugraph was loaded from. - return libcugraph_lib + + # TODO: return something here? + # return libcugraph_lib From 089109f0e3e7514e78a19d6e7debf69eafeeb9e2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Dec 2024 10:04:07 -0800 Subject: [PATCH 16/29] fix typo --- ci/build_wheel_cugraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 099c93e5228..0423a6ac80e 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -7,7 +7,7 @@ package_dir="python/cugraph" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the libcugraph and pylibcugraph wheesl built in the previous steps and make them +# Download the libcugraph and pylibcugraph wheels built in the previous steps and make them # available for pip to find. # # ensure 'cugraph' wheel builds always use the 'libcugraph' and 'pylibcugraph' just built in the same CI run From 881dc25cf3d34aa1396d0609faade447d7286d58 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 10 Dec 2024 08:55:26 -0800 Subject: [PATCH 17/29] header install sort of working --- cpp/CMakeLists.txt | 18 ++++++++++++++ cpp/cmake/thirdparty/get_raft.cmake | 38 ++++++++++++++++++++++++++--- python/cugraph/CMakeLists.txt | 3 +++ python/libcugraph/CMakeLists.txt | 25 +++++++++++++++++++ python/pylibcugraph/CMakeLists.txt | 3 +++ 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8ecab358dd0..9579a7b44ae 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -609,6 +609,21 @@ install(TARGETS cugraph_c DESTINATION ${lib_dir} EXPORT cugraph-exports) +install( + TARGETS raft raft_lib_static + COMPONENT raft + DESTINATION include/raft-static/ + INCLUDES DESTINATION include/raft + EXPORT cugraph-raft-exports +) + +get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) +install( + DIRECTORY ${_raft_interface_include_dirs} + DESTINATION include/raft-headers/ + # EXPORT cugraph-raft-exports +) + install(DIRECTORY include/cugraph_c/ DESTINATION include/cugraph_c) @@ -658,3 +673,6 @@ if(Doxygen_FOUND) add_custom_target(docs_cugraph DEPENDS CUGRAPH_DOXYGEN) endif() + +# TODO: revert before merging +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index c04ed2caf71..458591e868b 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -39,12 +39,12 @@ function(find_and_configure_raft) endif() rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET cugraph-exports - INSTALL_EXPORT_SET cugraph-exports + GLOBAL_TARGETS raft raft_compiled_static + BUILD_EXPORT_SET cugraph-raft-exports + INSTALL_EXPORT_SET cugraph-raft-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS - # EXCLUDE_FROM_ALL TRUE + EXCLUDE_FROM_ALL TRUE GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git GIT_TAG ${PKG_PINNED_TAG} SOURCE_SUBDIR cpp @@ -53,6 +53,36 @@ function(find_and_configure_raft) "BUILD_TESTS OFF" "BUILD_BENCH OFF" "BUILD_CAGRA_HNSWLIB OFF" + "BUILD_PRIMS_BENCH OFF" + ) + + # _raft_include_dirs: + # _raft_include_dirs-NOTFOUND + # get_target_property(_raft_include_dirs raft INCLUDE_DIRECTORIES) + # message(STATUS "_raft_include_dirs: ${_raft_include_dirs}") + + # _raft_interface_include_dirs: + # $;$ + # get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) + # message(FATAL_ERROR "_raft_interface_include_dirs: ${_raft_interface_include_dirs}") + + # TODO: move this under if(raft_ADDED)? + # TODO: install just the headers + install( + TARGETS raft raft_lib_static + COMPONENT raft + DESTINATION include/raft-static/ + INCLUDES DESTINATION include/raft + EXPORT cugraph-raft-exports + ) + + # TODO: maybe raft should be using target_sources instead? + # ref: https://cmake.org/cmake/help/latest/command/install.html#directory + get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) + install( + DIRECTORY ${_raft_interface_include_dirs} + DESTINATION include/raft-headers/ + # EXPORT cugraph-raft-exports ) if(raft_ADDED) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 39e26c84339..9c6d09478f2 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -46,3 +46,6 @@ add_subdirectory(cugraph/linear_assignment) add_subdirectory(cugraph/structure) add_subdirectory(cugraph/tree) add_subdirectory(cugraph/utilities) + +# TODO: revert before merging +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index 9624d0bcb92..d022e25b949 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -52,6 +52,28 @@ endif() add_subdirectory(../../cpp cugraph-cpp) +install( + TARGETS raft raft_lib_static + COMPONENT raft + DESTINATION include/raft-static/ + INCLUDES DESTINATION include/raft + EXPORT cugraph-raft-exports +) + +# install( +# DIRECTORY include/raft +# COMPONENT raft +# DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +# ) + +get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) +install( + DIRECTORY ${_raft_interface_include_dirs} + DESTINATION include/raft-headers/ + # EXPORT cugraph-raft-exports +) + + if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set(rpaths "$ORIGIN/../nvidia/cublas/lib" @@ -63,3 +85,6 @@ if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) ) set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) endif() + +# TODO: revert before merging +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 1b78714b32f..46e0742097b 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -34,3 +34,6 @@ include(rapids-cython-core) rapids_cython_init() add_subdirectory(pylibcugraph) + +# TODO: revert before merging +#set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) From ead60e4608eb86e0c6522fe3d7c337fff2ed27da Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 12 Dec 2024 09:03:14 -0800 Subject: [PATCH 18/29] install working (no libraft.a/libraft.so, just headers) ... export not working yet --- cpp/CMakeLists.txt | 15 ---- cpp/cmake/thirdparty/get_raft.cmake | 106 +++++++++++++++++++++++----- python/libcugraph/CMakeLists.txt | 22 ------ 3 files changed, 89 insertions(+), 54 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9579a7b44ae..c982dc76e24 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -609,21 +609,6 @@ install(TARGETS cugraph_c DESTINATION ${lib_dir} EXPORT cugraph-exports) -install( - TARGETS raft raft_lib_static - COMPONENT raft - DESTINATION include/raft-static/ - INCLUDES DESTINATION include/raft - EXPORT cugraph-raft-exports -) - -get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) -install( - DIRECTORY ${_raft_interface_include_dirs} - DESTINATION include/raft-headers/ - # EXPORT cugraph-raft-exports -) - install(DIRECTORY include/cugraph_c/ DESTINATION include/cugraph_c) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 458591e868b..25b25d66a62 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -32,21 +32,21 @@ function(find_and_configure_raft) if(PKG_COMPILE_RAFT_LIB) if(NOT PKG_USE_RAFT_STATIC) - string(APPEND RAFT_COMPONENTS " compiled") + string(APPEND RAFT_COMPONENTS " raft compiled") else() - string(APPEND RAFT_COMPONENTS " compiled_static") + string(APPEND RAFT_COMPONENTS " raft compiled_static") endif() endif() rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft raft_compiled_static - BUILD_EXPORT_SET cugraph-raft-exports - INSTALL_EXPORT_SET cugraph-raft-exports + GLOBAL_TARGETS raft::raft + # BUILD_EXPORT_SET cugraph-raft-exports + # INSTALL_EXPORT_SET cugraph-raft-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS EXCLUDE_FROM_ALL TRUE - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} + GIT_REPOSITORY https://github.com/jameslamb/raft.git + GIT_TAG use-filesets SOURCE_SUBDIR cpp OPTIONS "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_RAFT_LIB}" @@ -68,23 +68,95 @@ function(find_and_configure_raft) # TODO: move this under if(raft_ADDED)? # TODO: install just the headers - install( - TARGETS raft raft_lib_static - COMPONENT raft - DESTINATION include/raft-static/ - INCLUDES DESTINATION include/raft - EXPORT cugraph-raft-exports - ) + # install( + # TARGETS raft raft_lib_static + # COMPONENT raft + # DESTINATION include/raft-static/ + # INCLUDES DESTINATION include/raft + # EXPORT cugraph-raft-exports + # ) # TODO: maybe raft should be using target_sources instead? # ref: https://cmake.org/cmake/help/latest/command/install.html#directory + message(STATUS "--- CMAKE_INSTALL_INCLUDEDIR: ${CMAKE_INSTALL_INCLUDEDIR}") get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) + message(STATUS "--- _raft_interface_include_dirs: ${_raft_interface_include_dirs}") + # install( + # #DIRECTORY ${_raft_interface_include_dirs} + # DIRECTORY "/tmp/cugraph/python/libcugraph/build/py3-none-linux_x86_64/_deps/raft-src/cpp/include/" + # #DESTINATION include/raft-headers/ + # DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # # EXPORT cugraph-raft-exports + # ) + + # TODO: add to PR description: https://cmake.org/cmake/help/latest/command/install.html#example-install-targets-with-per-artifact-components + + #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft + # result: include/raft/include/raft + # DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # result: include/include/raft + + # rapids_export_package( + # BUILD raft cugraph-raft-exports + # COMPONENTS + # raft + # GLOBAL_TARGETS + # raft::raft raft + # ) + # rapids_export_package( + # INSTALL raft cugraph-raft-exports + # COMPONENTS + # raft + # GLOBAL_TARGETS + # raft::raft raft + # ) + + # install the RAFT headers install( - DIRECTORY ${_raft_interface_include_dirs} - DESTINATION include/raft-headers/ - # EXPORT cugraph-raft-exports + TARGETS raft + #EXPORT cugraph-raft-exports + # these files already come through with a relative path like 'include/raft', + # so should end up alongside other cugraph includes + DESTINATION ${CMAKE_INSTALL_PREFIX} + FILE_SET raft_headers + # COMPONENT raft + PUBLIC_HEADER + #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft + #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # COMPONENT raft + FILE_SET raft_headers + INCLUDES + #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft + #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # COMPONENT raft ) + # rapids_export( + # BUILD raft + # EXPORT_SET cugraph-raft-exports + # COMPONENTS raft + # COMPONENTS_EXPORT_SET cugraph-raft-exports + # GLOBAL_TARGETS raft + # NAMESPACE raft:: + # ) + # rapids_export( + # INSTALL raft + # EXPORT_SET cugraph-raft-exports + # COMPONENTS raft + # COMPONENTS_EXPORT_SET cugraph-raft-exports + # GLOBAL_TARGETS raft + # NAMESPACE raft:: + # ) + + get_target_property(_raft_sources raft SOURCES) + message(STATUS "_raft_sources: ${_raft_sources}") + get_target_property(_raft_raft_sources raft::raft SOURCES) + message(STATUS "_raft_raft_sources: ${_raft_sources}") + foreach(source_file IN LISTS _raft_sources) + message(STATUS "-- ${source_file}") + endforeach() + # message(FATAL_ERROR "--- here beep boop ---") + if(raft_ADDED) message(VERBOSE "CUGRAPH: Using RAFT located in ${raft_SOURCE_DIR}") else() diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index d022e25b949..beb3f5a1e2c 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -52,28 +52,6 @@ endif() add_subdirectory(../../cpp cugraph-cpp) -install( - TARGETS raft raft_lib_static - COMPONENT raft - DESTINATION include/raft-static/ - INCLUDES DESTINATION include/raft - EXPORT cugraph-raft-exports -) - -# install( -# DIRECTORY include/raft -# COMPONENT raft -# DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -# ) - -get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) -install( - DIRECTORY ${_raft_interface_include_dirs} - DESTINATION include/raft-headers/ - # EXPORT cugraph-raft-exports -) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set(rpaths "$ORIGIN/../nvidia/cublas/lib" From 7dd440940dc2fcbedd8bc31c9340596211c96861 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 12 Dec 2024 10:24:55 -0800 Subject: [PATCH 19/29] install working, exports still not quite working --- .github/workflows/pr.yaml | 38 +++++++++++----------- cpp/cmake/thirdparty/get_raft.cmake | 49 ++++++++++++++++++----------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7d394916f47..39b10932731 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,8 +25,8 @@ jobs: - wheel-build-libcugraph - wheel-build-pylibcugraph # - wheel-tests-pylibcugraph - - wheel-build-cugraph - - wheel-tests-cugraph + # - wheel-build-cugraph + # - wheel-tests-cugraph # - telemetry-setup # - devcontainer secrets: inherit @@ -161,23 +161,23 @@ jobs: # with: # build_type: pull-request # script: ci/test_wheel_pylibcugraph.sh - wheel-build-cugraph: - needs: wheel-build-pylibcugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 - with: - build_type: pull-request - script: ci/build_wheel_cugraph.sh - wheel-tests-cugraph: - needs: [wheel-build-cugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph.sh - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] + # wheel-build-cugraph: + # needs: wheel-build-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + # with: + # build_type: pull-request + # script: ci/build_wheel_cugraph.sh + # wheel-tests-cugraph: + # needs: [wheel-build-cugraph, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + # # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel_cugraph.sh + # # This selects "ARCH=amd64 + the latest supported Python + CUDA". + # matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] # devcontainer: # secrets: inherit # # needs: telemetry-setup diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 25b25d66a62..5b3bd7cb0bf 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -40,8 +40,8 @@ function(find_and_configure_raft) rapids_cpm_find(raft ${PKG_VERSION} GLOBAL_TARGETS raft::raft - # BUILD_EXPORT_SET cugraph-raft-exports - # INSTALL_EXPORT_SET cugraph-raft-exports + # BUILD_EXPORT_SET cugraph-exports + # INSTALL_EXPORT_SET cugraph-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS EXCLUDE_FROM_ALL TRUE @@ -97,14 +97,14 @@ function(find_and_configure_raft) # result: include/include/raft # rapids_export_package( - # BUILD raft cugraph-raft-exports + # BUILD raft cugraph-exports # COMPONENTS # raft # GLOBAL_TARGETS # raft::raft raft # ) # rapids_export_package( - # INSTALL raft cugraph-raft-exports + # INSTALL raft cugraph-exports # COMPONENTS # raft # GLOBAL_TARGETS @@ -114,22 +114,35 @@ function(find_and_configure_raft) # install the RAFT headers install( TARGETS raft - #EXPORT cugraph-raft-exports - # these files already come through with a relative path like 'include/raft', - # so should end up alongside other cugraph includes - DESTINATION ${CMAKE_INSTALL_PREFIX} FILE_SET raft_headers - # COMPONENT raft - PUBLIC_HEADER - #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft - #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # COMPONENT raft - FILE_SET raft_headers - INCLUDES - #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft - #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # COMPONENT raft + DESTINATION ${CMAKE_INSTALL_PREFIX} ) + # install( + # TARGETS raft + # # --- notes --- + # # + # # If `EXPORT raft-exports` is included here, resuls in this: + # # + # # CMake Error: install(EXPORT "raft-exports" ...) includes target "raft" more than once in the export set. + # # + # #EXPORT raft-exports + # # EXPORT cugraph-exports + # # these files already come through with a relative path like 'include/raft', + # # so should end up alongside other cugraph includes + # DESTINATION ${CMAKE_INSTALL_PREFIX} + # FILE_SET raft_headers + # # COMPONENT raft + # PUBLIC_HEADER + # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft + # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # # COMPONENT raft + # FILE_SET raft_headers + # INCLUDES + # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft + # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + # # COMPONENT raft + # ) + # install(EXPORT raft-exports DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}) # rapids_export( # BUILD raft From 1d60712d6214e6c7f3a2db845a7fc11cbc9b2567 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 12 Dec 2024 10:34:39 -0800 Subject: [PATCH 20/29] remove debugging stuff --- cpp/cmake/thirdparty/get_raft.cmake | 109 +--------------------------- python/cugraph/CMakeLists.txt | 3 - python/libcugraph/CMakeLists.txt | 5 -- python/pylibcugraph/CMakeLists.txt | 3 - 4 files changed, 2 insertions(+), 118 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 5b3bd7cb0bf..da264b5bcef 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -56,119 +56,14 @@ function(find_and_configure_raft) "BUILD_PRIMS_BENCH OFF" ) - # _raft_include_dirs: - # _raft_include_dirs-NOTFOUND - # get_target_property(_raft_include_dirs raft INCLUDE_DIRECTORIES) - # message(STATUS "_raft_include_dirs: ${_raft_include_dirs}") - - # _raft_interface_include_dirs: - # $;$ - # get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) - # message(FATAL_ERROR "_raft_interface_include_dirs: ${_raft_interface_include_dirs}") - - # TODO: move this under if(raft_ADDED)? - # TODO: install just the headers - # install( - # TARGETS raft raft_lib_static - # COMPONENT raft - # DESTINATION include/raft-static/ - # INCLUDES DESTINATION include/raft - # EXPORT cugraph-raft-exports - # ) - - # TODO: maybe raft should be using target_sources instead? - # ref: https://cmake.org/cmake/help/latest/command/install.html#directory - message(STATUS "--- CMAKE_INSTALL_INCLUDEDIR: ${CMAKE_INSTALL_INCLUDEDIR}") - get_target_property(_raft_interface_include_dirs raft INTERFACE_INCLUDE_DIRECTORIES) - message(STATUS "--- _raft_interface_include_dirs: ${_raft_interface_include_dirs}") - # install( - # #DIRECTORY ${_raft_interface_include_dirs} - # DIRECTORY "/tmp/cugraph/python/libcugraph/build/py3-none-linux_x86_64/_deps/raft-src/cpp/include/" - # #DESTINATION include/raft-headers/ - # DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # # EXPORT cugraph-raft-exports - # ) - - # TODO: add to PR description: https://cmake.org/cmake/help/latest/command/install.html#example-install-targets-with-per-artifact-components - - #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft - # result: include/raft/include/raft - # DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # result: include/include/raft - - # rapids_export_package( - # BUILD raft cugraph-exports - # COMPONENTS - # raft - # GLOBAL_TARGETS - # raft::raft raft - # ) - # rapids_export_package( - # INSTALL raft cugraph-exports - # COMPONENTS - # raft - # GLOBAL_TARGETS - # raft::raft raft - # ) - # install the RAFT headers + # TODO(jameslamb): get exporting working + # TODO(jameslamb): move this under if(raft_ADDED)? install( TARGETS raft FILE_SET raft_headers DESTINATION ${CMAKE_INSTALL_PREFIX} ) - # install( - # TARGETS raft - # # --- notes --- - # # - # # If `EXPORT raft-exports` is included here, resuls in this: - # # - # # CMake Error: install(EXPORT "raft-exports" ...) includes target "raft" more than once in the export set. - # # - # #EXPORT raft-exports - # # EXPORT cugraph-exports - # # these files already come through with a relative path like 'include/raft', - # # so should end up alongside other cugraph includes - # DESTINATION ${CMAKE_INSTALL_PREFIX} - # FILE_SET raft_headers - # # COMPONENT raft - # PUBLIC_HEADER - # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft - # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # # COMPONENT raft - # FILE_SET raft_headers - # INCLUDES - # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft - # #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - # # COMPONENT raft - # ) - # install(EXPORT raft-exports DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}) - - # rapids_export( - # BUILD raft - # EXPORT_SET cugraph-raft-exports - # COMPONENTS raft - # COMPONENTS_EXPORT_SET cugraph-raft-exports - # GLOBAL_TARGETS raft - # NAMESPACE raft:: - # ) - # rapids_export( - # INSTALL raft - # EXPORT_SET cugraph-raft-exports - # COMPONENTS raft - # COMPONENTS_EXPORT_SET cugraph-raft-exports - # GLOBAL_TARGETS raft - # NAMESPACE raft:: - # ) - - get_target_property(_raft_sources raft SOURCES) - message(STATUS "_raft_sources: ${_raft_sources}") - get_target_property(_raft_raft_sources raft::raft SOURCES) - message(STATUS "_raft_raft_sources: ${_raft_sources}") - foreach(source_file IN LISTS _raft_sources) - message(STATUS "-- ${source_file}") - endforeach() - # message(FATAL_ERROR "--- here beep boop ---") if(raft_ADDED) message(VERBOSE "CUGRAPH: Using RAFT located in ${raft_SOURCE_DIR}") diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 9c6d09478f2..39e26c84339 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -46,6 +46,3 @@ add_subdirectory(cugraph/linear_assignment) add_subdirectory(cugraph/structure) add_subdirectory(cugraph/tree) add_subdirectory(cugraph/utilities) - -# TODO: revert before merging -# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt index beb3f5a1e2c..3c935218077 100644 --- a/python/libcugraph/CMakeLists.txt +++ b/python/libcugraph/CMakeLists.txt @@ -40,8 +40,6 @@ set(CUDA_STATIC_MATH_LIBRARIES ON) set(USE_RAFT_STATIC ON) set(CUGRAPH_COMPILE_RAFT_LIB ON) -# set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) - find_package(CUDAToolkit REQUIRED) if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) @@ -63,6 +61,3 @@ if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) ) set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) endif() - -# TODO: revert before merging -# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 46e0742097b..1b78714b32f 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -34,6 +34,3 @@ include(rapids-cython-core) rapids_cython_init() add_subdirectory(pylibcugraph) - -# TODO: revert before merging -#set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) From a18be8ca1f7181469646b440cc9b290a60295301 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 12 Dec 2024 10:46:55 -0800 Subject: [PATCH 21/29] remove more debugging stuff --- ci/validate_wheel.sh | 2 +- cpp/CMakeLists.txt | 3 --- python/libcugraph/libcugraph/load.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 1b8ce4b57ee..ef4a5bfb108 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -10,7 +10,7 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" # some packages are much larger on CUDA 11 than on CUDA 12 if [[ "${package_dir}" == "python/libcugraph" ]]; then - # TODO: before merging, modify these thresholds + # TODO(jameslamb): before merging, modify these thresholds if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then PYDISTCHECK_ARGS=( --max-allowed-size-compressed '5.0G' diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c982dc76e24..8ecab358dd0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -658,6 +658,3 @@ if(Doxygen_FOUND) add_custom_target(docs_cugraph DEPENDS CUGRAPH_DOXYGEN) endif() - -# TODO: revert before merging -# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE ON) diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py index c0cf71e9008..df98c721f18 100644 --- a/python/libcugraph/libcugraph/load.py +++ b/python/libcugraph/libcugraph/load.py @@ -76,5 +76,5 @@ def load_library(): # harm in offering the option since this object at least provides a handle # to inspect where libcugraph was loaded from. - # TODO: return something here? + # TODO(jameslamb): return something here? # return libcugraph_lib From 7f4f13b44be2492681d43dde8bb0a45bcc8a4e74 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 12 Dec 2024 11:47:09 -0800 Subject: [PATCH 22/29] try adding exports back --- cpp/cmake/thirdparty/get_raft.cmake | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index da264b5bcef..bbc9f704dda 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -32,16 +32,16 @@ function(find_and_configure_raft) if(PKG_COMPILE_RAFT_LIB) if(NOT PKG_USE_RAFT_STATIC) - string(APPEND RAFT_COMPONENTS " raft compiled") + string(APPEND RAFT_COMPONENTS " compiled") else() - string(APPEND RAFT_COMPONENTS " raft compiled_static") + string(APPEND RAFT_COMPONENTS " compiled_static") endif() endif() rapids_cpm_find(raft ${PKG_VERSION} GLOBAL_TARGETS raft::raft - # BUILD_EXPORT_SET cugraph-exports - # INSTALL_EXPORT_SET cugraph-exports + BUILD_EXPORT_SET cugraph-exports + INSTALL_EXPORT_SET cugraph-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS EXCLUDE_FROM_ALL TRUE @@ -61,6 +61,7 @@ function(find_and_configure_raft) # TODO(jameslamb): move this under if(raft_ADDED)? install( TARGETS raft + EXPORT cugraph-exports FILE_SET raft_headers DESTINATION ${CMAKE_INSTALL_PREFIX} ) From dc7dac9d17fc631da950bd52039560c296c4d696 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 13:02:12 -0800 Subject: [PATCH 23/29] use libraft wheels --- cpp/cmake/thirdparty/get_raft.cmake | 15 ++------------- dependencies.yaml | 28 ++++++++++++++++++++++++++++ python/cugraph/pyproject.toml | 1 + python/libcugraph/pyproject.toml | 1 + python/pylibcugraph/pyproject.toml | 1 + 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index bbc9f704dda..8f56372c81a 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -45,25 +45,14 @@ function(find_and_configure_raft) COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS EXCLUDE_FROM_ALL TRUE - GIT_REPOSITORY https://github.com/jameslamb/raft.git - GIT_TAG use-filesets + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} SOURCE_SUBDIR cpp OPTIONS "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_RAFT_LIB}" "BUILD_TESTS OFF" "BUILD_BENCH OFF" "BUILD_CAGRA_HNSWLIB OFF" - "BUILD_PRIMS_BENCH OFF" - ) - - # install the RAFT headers - # TODO(jameslamb): get exporting working - # TODO(jameslamb): move this under if(raft_ADDED)? - install( - TARGETS raft - EXPORT cugraph-exports - FILE_SET raft_headers - DESTINATION ${CMAKE_INSTALL_PREFIX} ) if(raft_ADDED) diff --git a/dependencies.yaml b/dependencies.yaml index 22de9773dc9..cc93e9bc3b9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -81,6 +81,7 @@ files: includes: - common_build - depends_on_libcugraph + - depends_on_libraft - depends_on_librmm - depends_on_rmm - depends_on_pylibraft @@ -127,6 +128,7 @@ files: includes: - common_build - depends_on_librmm + - depends_on_libraft py_run_libcugraph: output: pyproject pyproject_dir: python/libcugraph @@ -152,6 +154,7 @@ files: includes: - common_build - depends_on_libcugraph + - depends_on_libraft - depends_on_librmm - depends_on_rmm - depends_on_pylibraft @@ -585,6 +588,31 @@ dependencies: - pylibwholegraph-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_libraft: + common: + - output_types: conda + packages: + - &libraft_unsuffixed libraft=25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libraft_unsuffixed]} + depends_on_librmm: common: - output_types: conda diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index ef989c3c0c8..5c8c9d53903 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -83,6 +83,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "libcugraph==25.2.*,>=0.0.0a0", + "libraft=25.2.*,>=0.0.0a0", "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibcugraph==25.2.*,>=0.0.0a0", diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml index 8be028e1cd1..2b46a9eda3e 100644 --- a/python/libcugraph/pyproject.toml +++ b/python/libcugraph/pyproject.toml @@ -80,6 +80,7 @@ dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.26.4,!=3.30.0", + "libraft=25.2.*,>=0.0.0a0", "librmm==25.2.*,>=0.0.0a0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index d7738b56074..90a4e38d737 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -71,6 +71,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4,!=3.30.0", "libcugraph==25.2.*,>=0.0.0a0", + "libraft=25.2.*,>=0.0.0a0", "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibraft==25.2.*,>=0.0.0a0", From a38e5ae829487686ef1a0ea76dc2bf53dab51984 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 13:24:07 -0800 Subject: [PATCH 24/29] use libraft from PR --- ci/build_wheel_cugraph.sh | 7 +++++++ ci/build_wheel_libcugraph.sh | 7 +++++++ ci/build_wheel_pylibcugraph.sh | 7 +++++++ ci/test_wheel_cugraph.sh | 12 ++++++++++++ ci/test_wheel_pylibcugraph.sh | 12 ++++++++++++ 5 files changed, 45 insertions(+) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 0423a6ac80e..c12fc3851aa 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -17,9 +17,16 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) +# TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 +RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +LIBRAFT_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" +) + cat > ./constraints.txt <> /tmp/requirements-build.txt + rapids-logger "Installing build requirements" python -m pip install \ -v \ diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index f1f3ceacd0b..db10f54b7bc 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -16,8 +16,15 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # are used when creating the isolated build environment LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) +# TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 +RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +LIBRAFT_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" +) + cat > ./constraints.txt < ./constraints.txt < ./constraints.txt < Date: Tue, 17 Dec 2024 13:43:05 -0800 Subject: [PATCH 25/29] update RAFT reference --- .github/workflows/pr.yaml | 56 +++++++++++++++++----------------- ci/build_wheel_cugraph.sh | 2 +- ci/build_wheel_libcugraph.sh | 2 +- ci/build_wheel_pylibcugraph.sh | 2 +- ci/test_wheel_cugraph.sh | 2 +- ci/test_wheel_pylibcugraph.sh | 2 +- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 39b10932731..5023266c3b9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,9 +24,9 @@ jobs: # - docs-build - wheel-build-libcugraph - wheel-build-pylibcugraph - # - wheel-tests-pylibcugraph - # - wheel-build-cugraph - # - wheel-tests-cugraph + - wheel-tests-pylibcugraph + - wheel-build-cugraph + - wheel-tests-cugraph # - telemetry-setup # - devcontainer secrets: inherit @@ -153,31 +153,31 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh - # wheel-tests-pylibcugraph: - # needs: [wheel-build-pylibcugraph, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - # with: - # build_type: pull-request - # script: ci/test_wheel_pylibcugraph.sh - # wheel-build-cugraph: - # needs: wheel-build-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 - # with: - # build_type: pull-request - # script: ci/build_wheel_cugraph.sh - # wheel-tests-cugraph: - # needs: [wheel-build-cugraph, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - # # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - # with: - # build_type: pull-request - # script: ci/test_wheel_cugraph.sh - # # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] + wheel-tests-pylibcugraph: + needs: [wheel-build-pylibcugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_pylibcugraph.sh + wheel-build-cugraph: + needs: wheel-build-pylibcugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: pull-request + script: ci/build_wheel_cugraph.sh + wheel-tests-cugraph: + needs: [wheel-build-cugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph.sh + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] # devcontainer: # secrets: inherit # # needs: telemetry-setup diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index c12fc3851aa..daae834f65d 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -18,7 +18,7 @@ LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 17966e29ec7..47af5db8b7c 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -17,7 +17,7 @@ rapids-dependency-file-generator \ | tee /tmp/requirements-build.txt # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index db10f54b7bc..26ae2e60176 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -17,7 +17,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index f1f6ee738ba..6ded45ee861 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -11,7 +11,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-whe RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libcugraph-dep # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/test_wheel_pylibcugraph.sh b/ci/test_wheel_pylibcugraph.sh index 96e0c2dd435..b936a11990b 100755 --- a/ci/test_wheel_pylibcugraph.sh +++ b/ci/test_wheel_pylibcugraph.sh @@ -10,7 +10,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-whe RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libcugraph-dep # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="f492d59978af3390e418796228aedb2601d03efc" +RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) From 79cebbc7f42c46780d9aa7065513e347b9e63622 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 14:11:46 -0800 Subject: [PATCH 26/29] fix libcugraph build script --- ci/build_wheel_libcugraph.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 47af5db8b7c..2c4c40ba29c 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -17,6 +17,7 @@ rapids-dependency-file-generator \ | tee /tmp/requirements-build.txt # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" From 3a07f41e2a1379fc0820586eddbc238c11622c08 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 14:34:30 -0800 Subject: [PATCH 27/29] try to fix installing libraft --- ci/build_wheel_libcugraph.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 2c4c40ba29c..55e6dcc553b 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -24,6 +24,11 @@ LIBRAFT_CHANNEL=$( ) echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRAFT_CHANNEL}/libraft_*.whl)" >> /tmp/requirements-build.txt +ls ${LIBRAFT_CHANNEL}/ + +rapids-logger "build reqs:" +cat /tmp/requirements-build.txt + rapids-logger "Installing build requirements" python -m pip install \ -v \ From c753752233d5ce66b09b512ffb9e10085f032fcc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 14:46:14 -0800 Subject: [PATCH 28/29] use constraints instead --- ci/build_wheel_libcugraph.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 55e6dcc553b..47b9b9a64f3 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -22,7 +22,8 @@ RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) -echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRAFT_CHANNEL}/libraft_*.whl)" >> /tmp/requirements-build.txt +echo "libraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRAFT_CHANNEL}/libraft_*.whl)" >> /tmp/constraints.txt +export PIP_CONSTRAINT=/tmp/constraints.txt ls ${LIBRAFT_CHANNEL}/ @@ -35,6 +36,8 @@ python -m pip install \ --prefer-binary \ -r /tmp/requirements-build.txt +rapids-logger "Done build requirements" + # build with '--no-build-isolation', for better sccache hit rate # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 From c6b1d5ab8891e5e141b81e08993266faf00b943c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 20 Dec 2024 08:49:12 -0800 Subject: [PATCH 29/29] try new libraft wheels --- ci/build_wheel_cugraph.sh | 2 +- ci/build_wheel_libcugraph.sh | 2 +- ci/build_wheel_pylibcugraph.sh | 2 +- ci/test_wheel_cugraph.sh | 2 +- ci/test_wheel_pylibcugraph.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index daae834f65d..6cf741e4f34 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -18,7 +18,7 @@ LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" +RAFT_COMMIT="d0b638579757deb74912e37785df3166f3be1109" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/build_wheel_libcugraph.sh b/ci/build_wheel_libcugraph.sh index 47b9b9a64f3..868f988de16 100755 --- a/ci/build_wheel_libcugraph.sh +++ b/ci/build_wheel_libcugraph.sh @@ -18,7 +18,7 @@ rapids-dependency-file-generator \ # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" +RAFT_COMMIT="d0b638579757deb74912e37785df3166f3be1109" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index 26ae2e60176..47fb03ee132 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -17,7 +17,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" +RAFT_COMMIT="d0b638579757deb74912e37785df3166f3be1109" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index 6ded45ee861..1989aba32cc 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -11,7 +11,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-whe RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libcugraph-dep # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" +RAFT_COMMIT="d0b638579757deb74912e37785df3166f3be1109" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" ) diff --git a/ci/test_wheel_pylibcugraph.sh b/ci/test_wheel_pylibcugraph.sh index b936a11990b..4fc4ca9127f 100755 --- a/ci/test_wheel_pylibcugraph.sh +++ b/ci/test_wheel_pylibcugraph.sh @@ -10,7 +10,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-whe RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libcugraph-dep # TODO(jameslamb): remove this stuff from https://github.com/rapidsai/raft/pull/2531 -RAFT_COMMIT="6bf5ebacd362a898d2580e88e17113ddcfeafdae" +RAFT_COMMIT="d0b638579757deb74912e37785df3166f3be1109" LIBRAFT_CHANNEL=$( RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact raft 2531 cpp "${RAFT_COMMIT:0:7}" )