make conda installs in CI stricter, other packaging changes

ci/test_cpp.sh

@@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -30,7 +32,9 @@ rapids-print-env
 
 rapids-mamba-retry install \
     --channel "${CPP_CHANNEL}" \
-    libcugraph libcugraph_etl libcugraph-tests
+    "libcugraph=${RAPIDS_VERSION}" \
+    "libcugraph_etl=${RAPIDS_VERSION}" \
+    "libcugraph-tests=${RAPIDS_VERSION}"
 
 rapids-logger "Check GPU usage"
 nvidia-smi

ci/test_notebooks.sh

@@ -5,6 +5,8 @@ set -Eeuo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -27,7 +29,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  libcugraph pylibcugraph cugraph
+  "libcugraph=${RAPIDS_VERSION}" \
+  "pylibcugraph=${RAPIDS_VERSION}" \
+  "cugraph=${RAPIDS_VERSION}"
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")"

ci/test_python.sh

@@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -34,12 +36,12 @@ rapids-print-env
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  libcugraph \
-  pylibcugraph \
-  cugraph \
-  nx-cugraph \
-  cugraph-service-server \
-  cugraph-service-client
+  "libcugraph=${RAPIDS_VERSION}" \
+  "pylibcugraph=${RAPIDS_VERSION}" \
+  "cugraph=${RAPIDS_VERSION}" \
+  "nx-cugraph=${RAPIDS_VERSION}" \
+  "cugraph-service-server=${RAPIDS_VERSION}" \
+  "cugraph-service-client=${RAPIDS_VERSION}"
 
 rapids-logger "Check GPU usage"
 nvidia-smi

ci/test_wheel.sh

@@ -4,14 +4,18 @@
 set -eoxu pipefail
 
 package_name=$1
-package_dir=$2
 
 python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
+# nx-cugraph is a pure wheel, which is part of generating the download path
+if [[ "${package_name}" == "nx-cugraph" ]]; then
+    RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
+else
+    RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+fi
 # use 'ls' to expand wildcard before adding `[extra]` requires for pip
 # pip creates wheels using python package names
 python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
@@ -32,9 +36,11 @@ else
     DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \
     DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \
     DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \
+    NX_CUGRAPH_USE_COMPAT_GRAPHS=False \
     python -m pytest \
        -v \
        --import-mode=append \
        --benchmark-disable \
+       -k "not test_property_graph_mg and not test_bulk_sampler_io" \
        ./python/${package_name}/${python_package_name}/tests
 fi

ci/test_wheel_cugraph-dgl.sh

@@ -4,24 +4,16 @@
 set -eoxu pipefail
 
 package_name="cugraph-dgl"
-package_dir="python/cugraph-dgl"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
+# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
 
+# determine pytorch and DGL sources
 PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
 PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
 if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -30,20 +22,17 @@ else
   PYTORCH_CUDA_VER=$PKG_CUDA_VER
 fi
 PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
-DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html"
-
-# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and
-# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that
-# might shadow the newer NCCL required by PyTorch during import (when importing
-# `cupy` before `torch`).
-if [[ "${NCCL_VERSION}" < "2.19" ]]; then
-  PYTORCH_VER="2.1.0"
-else
-  PYTORCH_VER="2.3.0"
-fi
-
-rapids-logger "Installing PyTorch and DGL"
-rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL}
+DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html"
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${DGL_URL}" \
+    "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'dgl==2.4.0' \
+    'torch>=2.3.0,<2.4'
 
 python -m pytest python/cugraph-dgl/tests

ci/test_wheel_cugraph-pyg.sh

@@ -4,46 +4,44 @@
 set -eoxu pipefail
 
 package_name="cugraph-pyg"
-package_dir="python/cugraph-pyg"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
+# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
-# RAPIDS_DATASET_ROOT_DIR is used by test scripts
-export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
-
-# Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
 
+# determine pytorch and pyg sources
 if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then
   PYTORCH_URL="https://download.pytorch.org/whl/cu118"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html"
 else
   PYTORCH_URL="https://download.pytorch.org/whl/cu121"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html"
 fi
-rapids-logger "Installing PyTorch and PyG dependencies"
-rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install "torch-geometric>=2.5,<2.6"
-rapids-retry python -m pip install \
-  ogb \
-  pyg_lib \
-  torch_scatter \
-  torch_sparse \
-  tensordict \
-  -f ${PYG_URL}
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${PYG_URL}" \
+    "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'ogb' \
+    'pyg_lib' \
+    'torch>=2.3.0,<2.4' \
+    'torch-geometric>=2.5,<2.6' \
+    'torch_scatter' \
+    'torch_sparse'
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+
+# Used to skip certain examples in CI due to memory limitations
+export CI_RUN=1
 
 rapids-logger "pytest cugraph-pyg (single GPU)"
 pushd python/cugraph-pyg/cugraph_pyg

conda/recipes/cugraph-dgl/meta.yaml

@@ -21,15 +21,17 @@ build:
 requirements:
   host:
     - python
+    - rapids-build-backend>=0.3.1,<0.4.0.dev0
+    - setuptools>=61.0.0
   run:
     - cugraph ={{ version }}
-    - dgl >=1.1.0.cu*
+    - dgl >=2.4.0.th23.cu*
     - numba >=0.57
     - numpy >=1.23,<3.0a0
     - pylibcugraphops ={{ minor_version }}
     - tensordict >=0.1.2
     - python
-    - pytorch >=2.0
+    - pytorch >=2.3,<2.4.0a0
     - cupy >=12.0.0
 
 tests:

conda/recipes/cugraph-pyg/meta.yaml

@@ -24,18 +24,19 @@ requirements:
   host:
     - cython >=3.0.0
     - python
-    - scikit-build-core >=0.7.0
+    - rapids-build-backend>=0.3.1,<0.4.0.dev0
+    - setuptools>=61.0.0
   run:
     - rapids-dask-dependency ={{ minor_version }}
     - numba >=0.57
     - numpy >=1.23,<3.0a0
     - python
-    - pytorch >=2.0
+    - pytorch >=2.3,<2.4.0a0
     - cupy >=12.0.0
     - cugraph ={{ version }}
     - pylibcugraphops ={{ minor_version }}
     - tensordict >=0.1.2
-    - pyg >=2.5,<2.6
+    - pytorch_geometric >=2.5,<2.6
 
 tests:
   imports:

conda/recipes/libwholegraph/conda_build_config.yaml

@@ -17,7 +17,7 @@ doxygen_version:
   - ">=1.8.11"
 
 nccl_version:
-  - ">=2.9.9"
+  - ">=2.19"
 
 c_stdlib:
   - sysroot

conda/recipes/pylibwholegraph/meta.yaml

@@ -75,4 +75,6 @@ requirements:
 
 about:
   home: https://rapids.ai/
+  license: Apache-2.0
+  license_file: ../../../LICENSE
   summary: pylibwholegraph library