WIP: introduce libcugraph wheels

.github/workflows/build.yaml

@@ -67,7 +67,28 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-libcugraph:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcugraph.sh
+      node_type: cpu32
+  wheel-publish-libcugraph:
+    needs: wheel-publish-libcugraph
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcugraph
   wheel-build-pylibcugraph:
+    needs: wheel-publish-libcugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -76,7 +97,6 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       script: ci/build_wheel_pylibcugraph.sh
-      node_type: cpu32
   wheel-publish-pylibcugraph:
     needs: wheel-build-pylibcugraph
     secrets: inherit

.github/workflows/pr.yaml

@@ -14,37 +14,37 @@ jobs:
   pr-builder:
     needs:
       - changed-files
-      - checks
-      - conda-cpp-build
-      - conda-cpp-tests
-      - conda-cpp-checks
-      - conda-notebook-tests
-      - conda-python-build
-      - conda-python-tests
-      - docs-build
+      # - checks
+      # - conda-cpp-build
+      # - conda-cpp-tests
+      # - conda-cpp-checks
+      # - conda-notebook-tests
+      # - conda-python-build
+      # - conda-python-tests
+      # - docs-build
+      - wheel-build-libcugraph
       - wheel-build-pylibcugraph
-      - wheel-tests-pylibcugraph
-      - wheel-build-cugraph
-      - wheel-tests-cugraph
-      - telemetry-setup
-      - devcontainer
+      # - wheel-tests-pylibcugraph
+      # - wheel-build-cugraph
+      # - wheel-tests-cugraph
+      # - telemetry-setup
+      # - devcontainer
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
     with:
       needs: ${{ toJSON(needs) }}
-  telemetry-setup:
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    env:
-        OTEL_SERVICE_NAME: "pr-cugraph"
-    steps:
-      - name: Telemetry setup
-        if: ${{ vars.TELEMETRY_ENABLED == 'true' }}
-        uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main
+  # telemetry-setup:
+  #   runs-on: ubuntu-latest
+  #   continue-on-error: true
+  #   env:
+  #       OTEL_SERVICE_NAME: "pr-cugraph"
+  #   steps:
+  #     - name: Telemetry setup
+  #       uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main
   changed-files:
     secrets: inherit
-    needs: telemetry-setup
+    # needs: telemetry-setup
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       files_yaml: |
@@ -73,123 +73,134 @@ jobs:
           - '!docs/**'
           - '!img/**'
           - '!notebooks/**'
-  checks:
+  # checks:
+  #   secrets: inherit
+  #   # needs: telemetry-setup
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     enable_check_generated_files: false
+  #     ignored_pr_jobs: telemetry-summarize
+  # conda-cpp-build:
+  #   needs: checks
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     node_type: cpu32
+  # conda-cpp-tests:
+  #   needs: [conda-cpp-build, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
+  #   with:
+  #     build_type: pull-request
+  # conda-cpp-checks:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     enable_check_symbols: true
+  #     symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel)
+  # conda-python-build:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  # conda-python-tests:
+  #   needs: [conda-python-build, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  # conda-notebook-tests:
+  #   needs: [conda-python-build, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
+  #     run_script: "ci/test_notebooks.sh"
+  # docs-build:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
+  #     run_script: "ci/build_docs.sh"
+  wheel-build-libcugraph:
+    # needs: checks
     secrets: inherit
-    needs: telemetry-setup
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      enable_check_generated_files: false
-      ignored_pr_jobs: telemetry-summarize
-  conda-cpp-build:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       build_type: pull-request
+      script: ci/build_wheel_libcugraph.sh
       node_type: cpu32
-  conda-cpp-tests:
-    needs: [conda-cpp-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
-    with:
-      build_type: pull-request
-  conda-cpp-checks:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      enable_check_symbols: true
-      symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel)
-  conda-python-build:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-  conda-python-tests:
-    needs: [conda-python-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-  conda-notebook-tests:
-    needs: [conda-python-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/test_notebooks.sh"
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/build_docs.sh"
   wheel-build-pylibcugraph:
-    needs: checks
+    needs: wheel-build-libcugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_pylibcugraph.sh
-      node_type: cpu32
-  wheel-tests-pylibcugraph:
-    needs: [wheel-build-pylibcugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_pylibcugraph.sh
-  wheel-build-cugraph:
-    needs: wheel-build-pylibcugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_cugraph.sh
-  wheel-tests-cugraph:
-    needs: [wheel-build-cugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph.sh
-  devcontainer:
-    secrets: inherit
-    needs: telemetry-setup
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      arch: '["amd64"]'
-      cuda: '["12.5"]'
-      node_type: cpu32
-      build_command: |
-        sccache -z;
-        build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON;
-        sccache -s;
-  telemetry-summarize:
-    runs-on: ubuntu-latest
-    needs: pr-builder
-    if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }}
-    continue-on-error: true
-    steps:
-      - name: Load stashed telemetry env vars
-        uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main
-        with:
-            load_service_name: true
-      - name: Telemetry summarize
-        uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main
-        with:
-          cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}"
+  # wheel-tests-pylibcugraph:
+  #   needs: [wheel-build-pylibcugraph, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_pylibcugraph.sh
+  # wheel-build-cugraph:
+  #   needs: wheel-build-pylibcugraph
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/build_wheel_cugraph.sh
+  # wheel-tests-cugraph:
+  #   needs: [wheel-build-cugraph, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_cugraph.sh
+  #     # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+  #     matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
+  # devcontainer:
+  #   secrets: inherit
+  #   # needs: telemetry-setup
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     arch: '["amd64"]'
+  #     cuda: '["12.5"]'
+  #     node_type: cpu32
+  #     build_command: |
+  #       sccache -z;
+  #       build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON;
+  #       sccache -s;
+  # telemetry-summarize:
+  #   runs-on: ubuntu-latest
+  #   needs: pr-builder
+  #   if: always()
+  #   continue-on-error: true
+  #   steps:
+  #     - name: Load stashed telemetry env vars
+  #       uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main
+  #       with:
+  #           load_service_name: true
+  #     - name: Telemetry summarize
+  #       uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main
+  #       with:
+  #         cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}"

.gitignore

@@ -32,17 +32,16 @@ test-results
 ## Python build directories & artifacts
 dask-worker-space/
 htmlcov
+*.conda
 dist/
-*.egg-info/
+.egg-info/
 python/build
 python/cugraph/bindings/*.cpp
 wheels/
 wheelhouse/
 _skbuild/
 cufile.log
-
-## pylibcugraph build directories & artifacts
-python/pylibcugraph/pylibcugraph.egg-info
+*.whl
 
 ## Patching
 *.diff

build.sh

@@ -175,11 +175,6 @@ SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}"
 # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS
 SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g')
 
-# Append `-DFIND_CUGRAPH_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
-if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then
-    SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUGRAPH_CPP=ON"
-fi
-
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
     if [[ "$INSTALL_PREFIX" != "" ]]; then