From 3ad4949f2e119f881361500172a04d2d4b243f31 Mon Sep 17 00:00:00 2001 From: Don Acosta <97529984+acostadon@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:07:24 -0500 Subject: [PATCH 1/4] Changed ci build_docs to just upload the xml so the cugraph-docs can create the docs (#4662) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … create the docs --------- Co-authored-by: Jake Awe Co-authored-by: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> --- ci/build_docs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index ad77c9a0e6..46a6005b8d 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -63,6 +63,8 @@ rapids-logger "Build CPP docs" pushd cpp/doxygen doxygen Doxyfile export XML_DIR_LIBCUGRAPH="$(pwd)/xml" +mkdir -p "${RAPIDS_DOCS_DIR}/libcugraph/xml_tar" +tar -czf "${RAPIDS_DOCS_DIR}/libcugraph/xml_tar"/xml.tar.gz -C xml . popd rapids-logger "Build Python docs" From 9b28c88bd49696e9507523805b7ba2d39153cbae Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Wed, 18 Dec 2024 00:27:34 -0600 Subject: [PATCH 2/4] update telemetry actions to fluent-bit friendly style (#4834) Simplifies telemetry a bit. More details at https://github.com/rapidsai/shared-actions/pull/28. Telemetry will still not be collected until @ajschmidt8 enables the `TELEMETRY_ENABLED` environment variable for this repo. Authors: - Mike Sarahan (https://github.com/msarahan) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4834 --- .github/workflows/pr.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a018544c96..aea81c152e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -180,16 +180,11 @@ jobs: build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; sccache -s; telemetry-summarize: - runs-on: ubuntu-latest + # This job must use a self-hosted runner to record telemetry traces. + runs-on: linux-amd64-cpu4 needs: pr-builder if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} continue-on-error: true steps: - - name: Load stashed telemetry env vars - uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main - with: - load_service_name: true - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main - with: - cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}" + uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main From 6783aec7a56f05d5088073c822140bc0cd7d9665 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 18 Dec 2024 09:54:48 -0600 Subject: [PATCH 3/4] Add cuda-python dependency (#4829) cuGraph depends on `cuda-python` but does not currently list that dependency. This PR adds it. Authors: - Bradley Dice (https://github.com/bdice) - Ralph Liu (https://github.com/nv-rliu) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cugraph/pull/4829 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-125_arch-x86_64.yaml | 1 + dependencies.yaml | 9 +++++++++ python/cugraph/pyproject.toml | 1 + 4 files changed, 12 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d98db2232e..c5f683afd0 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,6 +13,7 @@ dependencies: - certifi - cmake>=3.26.4,!=3.30.0 - cuda-nvtx +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cudf==25.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 34bef310fe..21eda1c098 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,6 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cudf==25.2.*,>=0.0.0a0 - cupy>=12.0.0 diff --git a/dependencies.yaml b/dependencies.yaml index b271abc627..9ee9031440 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -401,6 +401,15 @@ dependencies: # dataset APIs require [http] extras for use with cudf. - fsspec[http]>=0.6.0 specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - cuda-python>=12.6.2,<13.0a0 + - matrix: # All CUDA 11 versions + packages: + - cuda-python>=11.8.5,<12.0a0 - output_types: pyproject matrices: - matrix: diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 481f151ac2..8fa3a0938e 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -23,6 +23,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ + "cuda-python>=11.8.5,<12.0a0", "cudf==25.2.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "dask-cuda==25.2.*,>=0.0.0a0", From 3fdaba6f711e16fa59264906bc42df1f1bfd072c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 20 Dec 2024 17:29:23 -0500 Subject: [PATCH 4/4] remove 'wget' conda dependency, re-organize dependencies.yaml (#4805) In #4804, I've started working on adding `libcugraph` wheels. This includes a few fixes for things I noticed while doing that: * removes `wget` from `test_notebook` environment - *our CI images already have this system-installed, and removing it helps to remove it from the environment solve for the unified RAPIDS devcontainers* * `dependencies.yaml` re-organization: - breaks `librmm` dependency out into a `depends_on_librmm` to reduce duplication, and for consistency with other RAPIDS dependencies - uses `depends_on_pylibwholegraph` group everywhere instead of repeating `pylibwholegraph` in multiple places - removes unused YAML anchors - alphabetizes lists ## Notes for Reviewers I'd originally wanted to also add a `librmm` wheel dependency for wheel builds here, but looks like doing that resulted in a lot more `sccache` misses, I guess because of building with build isolation. That change will happen in #4804 . Authors: - James Lamb (https://github.com/jameslamb) - Ralph Liu (https://github.com/nv-rliu) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4805 --- ci/release/update-version.sh | 1 - .../all_cuda-118_arch-x86_64.yaml | 4 +- .../all_cuda-125_arch-x86_64.yaml | 4 +- conda/recipes/libcugraph/meta.yaml | 2 - dependencies.yaml | 197 +++++++++++------- python/cugraph-service/client/pyproject.toml | 2 +- python/cugraph-service/server/pyproject.toml | 2 +- 7 files changed, 128 insertions(+), 84 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a73745f2c0..2890011bc7 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -58,7 +58,6 @@ DEPENDENCIES=( dask-cudf libcudf libraft - libraft-headers librmm pylibcugraph pylibwholegraph diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c5f683afd0..7b2fddb742 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -28,7 +28,6 @@ dependencies: - graphviz - ipython - libcudf==25.2.*,>=0.0.0a0 -- libraft-headers==25.2.*,>=0.0.0a0 - libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx @@ -68,11 +67,10 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2!=0.5.0,!=0.5.1 +- thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata - torchmetrics - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 -- wget - wheel name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 21eda1c098..87a2417bbc 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -34,7 +34,6 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libraft-headers==25.2.*,>=0.0.0a0 - libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx @@ -73,11 +72,10 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2!=0.5.0,!=0.5.1 +- thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata - torchmetrics - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 -- wget - wheel name: all_cuda-125_arch-x86_64 diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 922d5e577b..eb948e08d1 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -70,7 +70,6 @@ requirements: - doxygen {{ doxygen_version }} - libcudf ={{ minor_version }} - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -114,7 +113,6 @@ outputs: - libcusparse {% endif %} - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu diff --git a/dependencies.yaml b/dependencies.yaml index 9ee9031440..eb4bb68dc9 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -11,18 +11,23 @@ files: - cpp_build - cuda - cuda_version - - docs - - python_build_rapids - - python_build_wheel - - python_build_cythonize - - depends_on_rmm - depends_on_cudf + - depends_on_cupy + - depends_on_dask_cuda - depends_on_dask_cudf + - depends_on_libcudf + - depends_on_libraft + - depends_on_librmm - depends_on_pylibraft - - depends_on_raft_dask - depends_on_pylibwholegraph - - depends_on_cupy - depends_on_pytorch + - depends_on_raft_dask + - depends_on_rmm + - depends_on_ucx_py + - docs + - python_build_cythonize + - python_build_rapids + - python_build_wheel - python_run_cugraph - test_notebook - test_python_common @@ -48,6 +53,7 @@ files: output: none includes: - cuda_version + - depends_on_pylibwholegraph - py_version - test_notebook - test_python_common @@ -69,8 +75,8 @@ files: extras: table: build-system includes: - - python_build_rapids - python_build_cythonize + - python_build_rapids py_rapids_build_cugraph: output: pyproject pyproject_dir: python/cugraph @@ -79,21 +85,23 @@ files: key: requires includes: - common_build - - depends_on_rmm - - depends_on_pylibraft - depends_on_pylibcugraph + - depends_on_pylibraft + - depends_on_rmm py_run_cugraph: output: pyproject pyproject_dir: python/cugraph extras: table: project includes: - - depends_on_rmm - depends_on_cudf + - depends_on_cupy + - depends_on_dask_cuda - depends_on_dask_cudf - - depends_on_raft_dask - depends_on_pylibcugraph - - depends_on_cupy + - depends_on_raft_dask + - depends_on_rmm + - depends_on_ucx_py - python_run_cugraph py_test_cugraph: output: pyproject @@ -102,17 +110,17 @@ files: table: project.optional-dependencies key: test includes: + - depends_on_pylibwholegraph - test_python_common - test_python_cugraph - - depends_on_pylibwholegraph py_build_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph extras: table: build-system includes: - - python_build_rapids - python_build_cythonize + - python_build_rapids py_rapids_build_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -121,8 +129,8 @@ files: key: requires includes: - common_build - - depends_on_rmm - depends_on_pylibraft + - depends_on_rmm py_run_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -130,8 +138,8 @@ files: table: project includes: - cuda_wheels - - depends_on_rmm - depends_on_pylibraft + - depends_on_rmm py_test_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -171,10 +179,14 @@ files: extras: table: project includes: - - depends_on_rmm - depends_on_cudf - - depends_on_dask_cudf + - depends_on_cugraph + - depends_on_cugraph_service_client - depends_on_cupy + - depends_on_dask_cuda + - depends_on_dask_cudf + - depends_on_rmm + - depends_on_ucx_py - python_run_cugraph_service_server py_test_cugraph_service_server: output: pyproject @@ -290,10 +302,6 @@ dependencies: packages: - c-compiler - cxx-compiler - - libcudf==25.2.*,>=0.0.0a0 - - libraft-headers==25.2.*,>=0.0.0a0 - - libraft==25.2.*,>=0.0.0a0 - - librmm==25.2.*,>=0.0.0a0 - openmpi # Required for building cpp-mgtests (multi-GPU tests) specific: - output_types: [conda] @@ -384,7 +392,6 @@ dependencies: - output_types: [conda, pyproject] packages: - &dask rapids-dask-dependency==25.2.*,>=0.0.0a0 - - &dask_cuda dask-cuda==25.2.*,>=0.0.0a0 - &numba numba>=0.57 - &numpy numpy>=1.23,<3.0a0 - output_types: conda @@ -394,7 +401,6 @@ dependencies: - requests - nccl>=2.19 - ucx-proc=*=gpu - - &ucx_py_unsuffixed ucx-py==0.42.*,>=0.0.0a0 - output_types: pyproject packages: # cudf uses fsspec but is protocol independent. cugraph @@ -410,60 +416,19 @@ dependencies: - matrix: # All CUDA 11 versions packages: - cuda-python>=11.8.5,<12.0a0 - - output_types: pyproject - matrices: - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &ucx_py_cu11 ucx-py-cu11==0.42.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &ucx_py_cu12 ucx-py-cu12==0.42.*,>=0.0.0a0 - - matrix: - packages: - - *ucx_py_unsuffixed python_run_cugraph_service_client: common: - output_types: [conda, pyproject] packages: - - &thrift thriftpy2!=0.5.0,!=0.5.1 + - &thrift thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 python_run_cugraph_service_server: common: - output_types: [conda, pyproject] packages: - *dask - - *dask_cuda - *numba - *numpy - *thrift - - output_types: conda - packages: - - *ucx_py_unsuffixed - specific: - - output_types: pyproject - matrices: - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &cugraph_cu11 cugraph-cu11==25.2.*,>=0.0.0a0 - - cugraph-service-client-cu11==25.2.*,>=0.0.0a0 - - *ucx_py_cu11 - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &cugraph_cu12 cugraph-cu12==25.2.*,>=0.0.0a0 - - cugraph-service-client-cu12==25.2.*,>=0.0.0a0 - - *ucx_py_cu12 - - matrix: - packages: - - &cugraph_unsuffixed cugraph==25.2.*,>=0.0.0a0 - - cugraph-service-client==25.2.*,>=0.0.0a0 - - *ucx_py_unsuffixed test_cpp: common: - output_types: conda @@ -476,9 +441,6 @@ dependencies: - certifi - ipython - notebook>=0.5.0 - - output_types: [conda] - packages: - - wget test_python_common: common: - output_types: [conda, pyproject] @@ -500,7 +462,6 @@ dependencies: - scikit-learn>=0.23.1 - output_types: [conda] packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==25.2.*,>=0.0.0a0 - *thrift test_python_pylibcugraph: common: @@ -508,11 +469,47 @@ dependencies: packages: - *numpy + depends_on_cugraph: + common: + - output_types: conda + packages: + - &cugraph_unsuffixed cugraph==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - cugraph-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - cugraph-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*cugraph_unsuffixed]} + + depends_on_cugraph_service_client: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cugraph-service-client==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + depends_on_pytorch: common: - output_types: [conda] packages: - - &pytorch_conda pytorch>=2.3,<2.5a0 + - pytorch>=2.3,<2.5a0 - torchdata - pydantic - ogb @@ -544,7 +541,7 @@ dependencies: common: - output_types: conda packages: - - *pylibwholegraph_unsuffixed + - &pylibwholegraph_unsuffixed pylibwholegraph==25.2.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -565,6 +562,24 @@ dependencies: - pylibwholegraph-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_libcudf: + common: + - output_types: conda + packages: + - libcudf==25.2.*,>=0.0.0a0 + + depends_on_libraft: + common: + - output_types: conda + packages: + - libraft==25.2.*,>=0.0.0a0 + + depends_on_librmm: + common: + - output_types: conda + packages: + - librmm==25.2.*,>=0.0.0a0 + depends_on_rmm: common: - output_types: conda @@ -615,6 +630,17 @@ dependencies: - cudf-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} + depends_on_dask_cuda: + common: + - output_types: [conda, pyproject, requirements] + packages: + - dask-cuda==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + depends_on_dask_cudf: common: - output_types: conda @@ -733,3 +759,28 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + + depends_on_ucx_py: + common: + - output_types: conda + packages: + - &ucx_py_unsuffixed ucx-py==0.42.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - ucx-py-cu12==0.42.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - ucx-py-cu11==0.42.*,>=0.0.0a0 + - {matrix: null, packages: [*ucx_py_unsuffixed]} diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index ac5e6bad0d..093a578db2 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -20,7 +20,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "thriftpy2!=0.5.0,!=0.5.1", + "thriftpy2>=0.4.15,!=0.5.0,!=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index adb18f5721..29ee41854f 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "numpy>=1.23,<3.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", - "thriftpy2!=0.5.0,!=0.5.1", + "thriftpy2>=0.4.15,!=0.5.0,!=0.5.1", "ucx-py==0.42.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [