From dc6c0c1ffa22b61a0663ec266b4a23f1df173234 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Fri, 6 Dec 2024 09:12:46 -0800 Subject: [PATCH] Add CUDA libs in Python Conda, Consolidate Conda CI installs & use `rapids-dask-dependency` (#513) Explicitly include dependencies on `libcufile*` and `*cudart*` in the Python Conda package. These seem to be pulled in implicitly by the C++ package. However as they are dependencies of the Python built shared objects, they should also be listed here. Also consolidate the Conda environment creation and installation of nightly dependencies into one step. By doing this in one step, we guarantee that all of our constraints are taken into account during environment creation. Lastly switch from using `dask` directly to `rapids-dask-dependency` to align Dask installation and pinning with the rest of RAPIDS. xref: https://github.com/rapidsai/build-planning/issues/22 xef: https://github.com/rapidsai/kvikio/issues/378 Authors: - https://github.com/jakirkham - Mads R. B. Kristensen (https://github.com/madsbk) Approvers: - Bradley Dice (https://github.com/bdice) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/kvikio/pull/513 --- ci/build_docs.sh | 20 +++++++------------ ci/release/update-version.sh | 3 +++ ci/test_cpp.sh | 15 +++++--------- ci/test_python.sh | 20 +++++++------------ .../all_cuda-118_arch-aarch64.yaml | 2 +- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-aarch64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/kvikio/conda_build_config.yaml | 17 ++++++++++++++-- conda/recipes/kvikio/meta.yaml | 12 ++++++++++- .../recipes/libkvikio/conda_build_config.yaml | 3 +++ conda/recipes/libkvikio/meta.yaml | 4 ++-- dependencies.yaml | 17 +++++++++++++++- python/kvikio/pyproject.toml | 2 +- 14 files changed, 74 insertions(+), 47 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 8c2ee8ae95..ea408b6940 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,28 +6,22 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -export RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-dependency-file-generator \ --output conda \ --file-key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n docs +rapids-mamba-retry env create -yq -f env.yaml -n docs conda activate docs rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "kvikio=${RAPIDS_VERSION}" \ - "libkvikio=${RAPIDS_VERSION}" - export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 6b9458b273..102beaa2ba 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -36,7 +36,10 @@ function sed_runner() { echo "${NEXT_FULL_TAG}" > VERSION DEPENDENCIES=( + kvikio libkvikio + libkvikio-tests + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml; do diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7dfde2187f..ef7933f150 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -5,33 +5,28 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -SUITEERROR=0 rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "libkvikio-tests=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 1f4ff6d2f1..df16f20f9f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "kvikio=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 2de4597657..7fee941a1d 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcurl>=7.87.0 @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8b63a6726b..b89468e062 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev=1.4.0.31 @@ -35,6 +34,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 87759faf53..e12ab092ef 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index e0a10af655..131efa85d5 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/recipes/kvikio/conda_build_config.yaml b/conda/recipes/kvikio/conda_build_config.yaml index 8af5f6f9c2..639a56f509 100644 --- a/conda/recipes/kvikio/conda_build_config.yaml +++ b/conda/recipes/kvikio/conda_build_config.yaml @@ -4,6 +4,9 @@ c_compiler_version: cxx_compiler_version: - 11 +cmake_version: + - ">=3.26.4,!=3.30.0" + cuda_compiler: - cuda-nvcc @@ -16,8 +19,18 @@ c_stdlib: c_stdlib_version: - "2.17" -cmake_version: - - ">=3.26.4,!=3.30.0" +# The CTK libraries below are missing from the conda-forge::cudatoolkit package +# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages +# and the "*_run_*" version specifiers correspond to `11.x` packages. + +cuda11_libcufile_host_version: + - "1.4.0.31" + +cuda11_libcufile_run_version: + - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" nvcomp_version: - "=4.1.0.6" diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index 3c41af3310..271712ac51 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -37,6 +37,8 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} requirements: @@ -58,13 +60,17 @@ requirements: - cython >=3.0.0 {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - libkvikio ={{ version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - python - numpy >=1.23,<3.0a0 @@ -76,6 +82,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart + - libcufile # [linux] {% endif %} test: diff --git a/conda/recipes/libkvikio/conda_build_config.yaml b/conda/recipes/libkvikio/conda_build_config.yaml index 9cf2923599..b895b842f3 100644 --- a/conda/recipes/libkvikio/conda_build_config.yaml +++ b/conda/recipes/libkvikio/conda_build_config.yaml @@ -28,3 +28,6 @@ cuda11_libcufile_host_version: cuda11_libcufile_run_version: - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" diff --git a/conda/recipes/libkvikio/meta.yaml b/conda/recipes/libkvikio/meta.yaml index 999b9fc2c1..4019a55ec8 100644 --- a/conda/recipes/libkvikio/meta.yaml +++ b/conda/recipes/libkvikio/meta.yaml @@ -52,7 +52,7 @@ requirements: {% else %} - libcufile-dev # [linux] {% endif %} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} outputs: - name: libkvikio @@ -75,7 +75,7 @@ outputs: - cmake {{ cmake_version }} host: - cuda-version ={{ cuda_version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} diff --git a/dependencies.yaml b/dependencies.yaml index fe85506ce5..5c35e5192c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,12 +24,14 @@ files: output: none includes: - cuda_version + - test_libkvikio - test_cpp test_python: output: none includes: - cuda_version - py_version + - test_kvikio - test_python checks: output: none @@ -42,6 +44,7 @@ files: - cuda_version - docs - py_version + - test_kvikio py_build_kvikio: output: pyproject pyproject_dir: python/kvikio @@ -358,6 +361,18 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs !=0.12.0 - packaging + test_libkvikio: + common: + - output_types: conda + packages: + - libkvikio==24.12.*,>=0.0.0a0 + - libkvikio-tests==24.12.*,>=0.0.0a0 + test_kvikio: + common: + - output_types: conda + packages: + - libkvikio==24.12.*,>=0.0.0a0 + - kvikio==24.12.*,>=0.0.0a0 test_cpp: common: - output_types: conda @@ -367,7 +382,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &dask dask>=2022.05.2 + - rapids-dask-dependency==24.12.*,>=0.0.0a0 - pytest - pytest-cov - rangehttpserver diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index cafde598bd..4869a63fab 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -42,11 +42,11 @@ classifiers = [ test = [ "boto3>=1.21.21", "cuda-python>=11.7.1,<12.0a0,<=11.8.3", - "dask>=2022.05.2", "moto[server]>=4.0.8", "pytest", "pytest-cov", "rangehttpserver", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls]