Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Forward-merge branch-24.12 into branch-25.02 #571

Merged
merged 2 commits into from
Dec 6, 2024
Merged

Forward-merge branch-24.12 into branch-25.02 #571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dc6c0c1
Add CUDA libs in Python Conda, Consolidate Conda CI installs & use `r…
jakirkham Dec 6, 2024
76a4177
Merge branch-24.12 into branch-25.02
jakirkham Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/trigger-breaking-change-alert.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
trigger-notifier:
if: contains(github.event.pull_request.labels.*.name, 'breaking')
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-24.12
uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.02
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As part of fixing conflicts, needed to rerun the update version script. It looks like this line wasn't updated previously. So it got picked up now. Just flagging this for awareness

Note this GHA workflow came from PR: #536

with:
sender_login: ${{ github.event.sender.login }}
sender_avatar: ${{ github.event.sender.avatar_url }}
Expand Down
20 changes: 7 additions & 13 deletions ci/build_docs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,22 @@ set -euo pipefail
rapids-logger "Create test conda environment"
. /opt/conda/etc/profile.d/conda.sh

export RAPIDS_VERSION="$(rapids-version)"
rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-dependency-file-generator \
--output conda \
--file-key docs \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n docs
rapids-mamba-retry env create -yq -f env.yaml -n docs
conda activate docs

rapids-print-env

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"kvikio=${RAPIDS_VERSION}" \
"libkvikio=${RAPIDS_VERSION}"

export RAPIDS_DOCS_DIR="$(mktemp -d)"

rapids-logger "Build CPP docs"
Expand Down
9 changes: 3 additions & 6 deletions ci/release/update-version.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,10 @@ CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
# Get <major>.<minor> for next version
NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}')
NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}

# Need to distutils-normalize the original version
NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))")
PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))")

echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"

Expand All @@ -38,7 +36,10 @@ function sed_runner() {
echo "${NEXT_FULL_TAG}" > VERSION

DEPENDENCIES=(
kvikio
libkvikio
libkvikio-tests
rapids-dask-dependency
)
for DEP in "${DEPENDENCIES[@]}"; do
for FILE in dependencies.yaml conda/environments/*.yaml; do
Expand All @@ -54,10 +55,6 @@ for FILE in .github/workflows/*.yaml; do
sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
done

# Java files
NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT"
sed_runner "s|<version>.*-SNAPSHOT</version>|<version>${NEXT_FULL_JAVA_TAG}</version>|g" java/pom.xml

# .devcontainer files
find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do
sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}"
Expand Down
15 changes: 5 additions & 10 deletions ci/test_cpp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,28 @@ set -euo pipefail

. /opt/conda/etc/profile.d/conda.sh

RAPIDS_VERSION="$(rapids-version)"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

rapids-logger "Generate C++ testing dependencies"
rapids-dependency-file-generator \
--output conda \
--file-key test_cpp \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
--prepend-channel "${CPP_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n test
rapids-mamba-retry env create -qy -f env.yaml -n test

# Temporarily allow unbound variables for conda activation.
set +u
conda activate test
set -u

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
mkdir -p "${RAPIDS_TESTS_DIR}"
SUITEERROR=0

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
"libkvikio=${RAPIDS_VERSION}" \
"libkvikio-tests=${RAPIDS_VERSION}"

rapids-logger "Check GPU usage"
nvidia-smi

Expand Down
20 changes: 7 additions & 13 deletions ci/test_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../

. /opt/conda/etc/profile.d/conda.sh

RAPIDS_VERSION="$(rapids-version)"
rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-logger "Generate Python testing dependencies"
rapids-dependency-file-generator \
--output conda \
--file-key test_python \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n test
rapids-mamba-retry env create -qy -f env.yaml -n test

# Temporarily allow unbound variables for conda activation.
set +u
conda activate test
set -u

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"libkvikio=${RAPIDS_VERSION}" \
"kvikio=${RAPIDS_VERSION}"

rapids-logger "Check GPU usage"
nvidia-smi

Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-aarch64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-aarch64=11.*
- libcurl>=7.87.0
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.2.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-64=11.*
- libcufile-dev=1.4.0.31
Expand All @@ -35,6 +34,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.2.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-125_arch-aarch64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-aarch64=11.*
- libcufile-dev
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.2.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-125_arch-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-64=11.*
- libcufile-dev
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.2.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
17 changes: 15 additions & 2 deletions conda/recipes/kvikio/conda_build_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ c_compiler_version:
cxx_compiler_version:
- 11

cmake_version:
- ">=3.26.4,!=3.30.0"

cuda_compiler:
- cuda-nvcc

Expand All @@ -16,8 +19,18 @@ c_stdlib:
c_stdlib_version:
- "2.17"

cmake_version:
- ">=3.26.4,!=3.30.0"
# The CTK libraries below are missing from the conda-forge::cudatoolkit package
# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
# and the "*_run_*" version specifiers correspond to `11.x` packages.

cuda11_libcufile_host_version:
- "1.4.0.31"

cuda11_libcufile_run_version:
- ">=1.0.0.82,<=1.4.0.31"

libcurl_version:
- "==7.87.0"

nvcomp_version:
- "=4.1.0.6"
12 changes: 11 additions & 1 deletion conda/recipes/kvikio/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ build:
- {{ compiler('cuda11') }}
{% else %}
- {{ compiler('cuda') }}
- cuda-cudart-dev
- libcufile-dev # [linux]
{% endif %}

requirements:
Expand All @@ -58,13 +60,17 @@ requirements:
- cython >=3.0.0
{% if cuda_major == "11" %}
- cudatoolkit
- libcufile {{ cuda11_libcufile_run_version }} # [linux64]
{% else %}
- cuda-cudart-dev
- libcufile-dev # [linux]
{% endif %}
- cuda-version ={{ cuda_version }}
- nvcomp {{ nvcomp_version }}
- rapids-build-backend >=0.3.0,<0.4.0.dev0
- scikit-build-core >=0.10.0
- libkvikio ={{ version }}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}
run:
- python
- numpy >=1.23,<3.0a0
Expand All @@ -76,6 +82,10 @@ requirements:
- {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
{% if cuda_major == "11" %}
- cudatoolkit
- libcufile {{ cuda11_libcufile_run_version }} # [linux64]
{% else %}
- cuda-cudart
- libcufile # [linux]
{% endif %}

test:
Expand Down
3 changes: 3 additions & 0 deletions conda/recipes/libkvikio/conda_build_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ cuda11_libcufile_host_version:

cuda11_libcufile_run_version:
- ">=1.0.0.82,<=1.4.0.31"

libcurl_version:
- "==7.87.0"
4 changes: 2 additions & 2 deletions conda/recipes/libkvikio/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ requirements:
{% else %}
- libcufile-dev # [linux]
{% endif %}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}

outputs:
- name: libkvikio
Expand All @@ -75,7 +75,7 @@ outputs:
- cmake {{ cmake_version }}
host:
- cuda-version ={{ cuda_version }}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}
run:
- {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
{% if cuda_major == "11" %}
Expand Down
17 changes: 16 additions & 1 deletion dependencies.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@ files:
output: none
includes:
- cuda_version
- test_libkvikio
- test_cpp
test_python:
output: none
includes:
- cuda_version
- py_version
- test_kvikio
- test_python
checks:
output: none
Expand All @@ -42,6 +44,7 @@ files:
- cuda_version
- docs
- py_version
- test_kvikio
py_build_kvikio:
output: pyproject
pyproject_dir: python/kvikio
Expand Down Expand Up @@ -366,6 +369,18 @@ dependencies:
# See https://github.com/zarr-developers/numcodecs/pull/475
- numcodecs !=0.12.0
- packaging
test_libkvikio:
common:
- output_types: conda
packages:
- libkvikio==25.2.*,>=0.0.0a0
- libkvikio-tests==25.2.*,>=0.0.0a0
test_kvikio:
common:
- output_types: conda
packages:
- libkvikio==25.2.*,>=0.0.0a0
- kvikio==25.2.*,>=0.0.0a0
test_cpp:
common:
- output_types: conda
Expand All @@ -375,7 +390,7 @@ dependencies:
common:
- output_types: [conda, requirements, pyproject]
packages:
- &dask dask>=2022.05.2
- rapids-dask-dependency==25.2.*,>=0.0.0a0
- pytest
- pytest-cov
- rangehttpserver
Expand Down
2 changes: 1 addition & 1 deletion python/kvikio/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ classifiers = [
test = [
"boto3>=1.21.21",
"cuda-python>=11.7.1,<12.0a0",
"dask>=2022.05.2",
"moto[server]>=4.0.8",
"pytest",
"pytest-cov",
"rangehttpserver",
"rapids-dask-dependency==25.2.*,>=0.0.0a0",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

[project.urls]
Expand Down
Loading