From 70749a3b960e0f0378c2014d7b17821452482270 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Thu, 20 Jul 2023 15:48:14 -0400 Subject: [PATCH 1/9] v0.34 --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 12 ++++++------ .github/workflows/test.yaml | 4 ++-- ci/release/update-version.sh | 5 +++-- dependencies.yaml | 2 +- pyproject.toml | 4 ++-- ucp/__init__.py | 2 +- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 89a909d93..0b3121b4b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-python-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: upload-conda: needs: [conda-python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -58,7 +58,7 @@ jobs: wheel-publish: needs: wheel-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 646f0ae2e..d552eb04d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,26 +18,26 @@ jobs: - wheel-build - wheel-tests secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.10 conda-python-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.10 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.10 with: build_type: pull-request wheel-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10 with: build_type: pull-request package-name: ucx_py @@ -47,7 +47,7 @@ jobs: wheel-tests: needs: wheel-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10 with: build_type: pull-request package-name: ucx_py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 34e1c042e..9859cd290 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a9c195e6f..c4175d0ef 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -28,6 +28,7 @@ NEXT_RAPIDS_SHORT_TAG="$(curl -sL https://version.gpuci.io/ucx-py/${NEXT_SHORT_T # Need to distutils-normalize the versions for some use cases NEXT_RAPIDS_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_SHORT_TAG}'))") +NEXT_RAPIDS_FULL_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_FULL_TAG}'))") echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -49,5 +50,5 @@ for FILE in .github/workflows/*.yaml; do sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_RAPIDS_SHORT_TAG}/g" "${FILE}" done -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" pyproject.toml -sed_runner "s/^__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" ucp/__init__.py +sed_runner "s/^version = .*/version = \"${NEXT_RAPIDS_FULL_TAG_PEP440}\"/g" pyproject.toml +sed_runner "s/^__version__ = .*/__version__ = \"${NEXT_RAPIDS_FULL_TAG_PEP440}\"/g" ucp/__init__.py diff --git a/dependencies.yaml b/dependencies.yaml index e5d0cc15a..f79f72770 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -116,7 +116,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cloudpickle - - cudf==23.8.* + - cudf==23.10.* - dask - distributed - numba>=0.57 diff --git a/pyproject.toml b/pyproject.toml index c4b599f73..ca13dc910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ requires = [ [project] name = "ucx-py" -version = "0.33.0" +version = "0.34.0" description = "Python Bindings for the Unified Communication X library (UCX)" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -49,7 +49,7 @@ classifiers = [ [project.optional-dependencies] test = [ "cloudpickle", - "cudf==23.8.*", + "cudf==23.10.*", "cupy-cuda11x>=12.0.0", "dask", "distributed", diff --git a/ucp/__init__.py b/ucp/__init__.py index 154af2fee..2edd17dd1 100644 --- a/ucp/__init__.py +++ b/ucp/__init__.py @@ -85,7 +85,7 @@ def _is_mig_device(handle): os.environ["UCX_MAX_RNDV_RAILS"] = "1" -__version__ = "0.33.0" +__version__ = "0.34.0" __ucx_version__ = "%d.%d.%d" % get_ucx_version() if get_ucx_version() < (1, 11, 1): From 7d230eb2518362c7bae715cca03a80f171f2962e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 4 Aug 2023 10:07:11 -0700 Subject: [PATCH 2/9] Update to Cython 3.0.0 (#979) This PR bumps the pinning and fixes a couple of typing errors that were previously hidden. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/ucx-py/pull/979 --- dependencies.yaml | 2 +- pyproject.toml | 2 +- ucp/_libs/ucx_worker.pyx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index f79f72770..1634ae829 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -100,7 +100,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - setuptools>=64.0.0 - - cython>=0.29.14,<0.30.0a0 + - cython>=3.0.0 - tomli # Not needed for Python 3.11+ run: common: diff --git a/pyproject.toml b/pyproject.toml index ca13dc910..439632736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ [build-system] build-backend = "setuptools.build_meta" requires = [ - "cython>=0.29.14,<0.30.0a0", + "cython>=3.0.0", "setuptools>=64.0.0", "tomli", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/ucp/_libs/ucx_worker.pyx b/ucp/_libs/ucx_worker.pyx index 54b41d543..d7f6073b3 100644 --- a/ucp/_libs/ucx_worker.pyx +++ b/ucp/_libs/ucx_worker.pyx @@ -47,7 +47,7 @@ cdef _drain_worker_tag_recv(ucp_worker_h handle): buf = malloc(info.length) status = ucp_tag_msg_recv_nb( - handle, buf, info.length, ucp_dt_make_contig(1), message, _tag_recv_callback + handle, buf, info.length, ucp_dt_make_contig(1), message, _tag_recv_cb ) try: @@ -130,7 +130,7 @@ cdef class UCXWorker(UCXObject): UCP_AM_HANDLER_PARAM_FIELD_ARG ) am_handler_param.id = AM_MSG_ID - am_handler_param.cb = _am_recv_callback + am_handler_param.cb = _am_recv_callback am_handler_param.arg = self status = ucp_worker_set_am_recv_handler(self._handle, &am_handler_param) From be4cdd9db0939f06cd875c940d56a640711b9f3b Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 14 Aug 2023 17:58:04 +0200 Subject: [PATCH 3/9] Update debug tests to remove use of deprecated `np.bool` (#981) Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Benjamin Zaitlen (https://github.com/quasiben) URL: https://github.com/rapidsai/ucx-py/pull/981 --- tests/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 3c8a287b1..4348a6de6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -76,9 +76,7 @@ async def send(ep, frames): await ep.send(np.array([len(frames)], dtype=np.uint64)) await ep.send( - np.array( - [hasattr(f, "__cuda_array_interface__") for f in frames], dtype=np.bool - ) + np.array([hasattr(f, "__cuda_array_interface__") for f in frames], dtype=bool) ) await ep.send(np.array([nbytes(f) for f in frames], dtype=np.uint64)) # Send frames @@ -96,7 +94,7 @@ async def recv(ep): # Recv meta data nframes = np.empty(1, dtype=np.uint64) await ep.recv(nframes) - is_cudas = np.empty(nframes[0], dtype=np.bool) + is_cudas = np.empty(nframes[0], dtype=bool) await ep.recv(is_cudas) sizes = np.empty(nframes[0], dtype=np.uint64) await ep.recv(sizes) From 72d2ec90285f7a84006e6737b4d31ee3d2b71238 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Thu, 24 Aug 2023 02:46:54 -0500 Subject: [PATCH 4/9] Handle ``NVMLError_NotSupported`` exception (#983) Closes https://github.com/rapidsai/ucx-py/issues/982 Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) URL: https://github.com/rapidsai/ucx-py/pull/983 --- ucp/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ucp/__init__.py b/ucp/__init__.py index 2edd17dd1..7486492c0 100644 --- a/ucp/__init__.py +++ b/ucp/__init__.py @@ -64,9 +64,14 @@ def _is_mig_device(handle): if _is_mig_device(handle): continue - total_memory = pynvml.nvmlDeviceGetMemoryInfo(handle).total - bar1_total = pynvml.nvmlDeviceGetBAR1MemoryInfo(handle).bar1Total + try: + bar1_total = pynvml.nvmlDeviceGetBAR1MemoryInfo(handle).bar1Total + except pynvml.nvml.NVMLError_NotSupported: + # Bar1 access not supported on this device, set it to + # zero (always lower than device memory). + bar1_total = 0 + total_memory = pynvml.nvmlDeviceGetMemoryInfo(handle).total if total_memory <= bar1_total: large_bar1[dev_idx] = True From 9ba056f9f2b1af169c6312b178e9853b066928bd Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 28 Aug 2023 09:40:30 -0400 Subject: [PATCH 5/9] Use `copy-pr-bot` (#984) This PR replaces the `copy_prs` functionality from the `ops-bot` with the new dedicated `copy-pr-bot` GitHub application. Thorough documentation for the new `copy-pr-bot` application can be viewed below. - https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ **Important**: `copy-pr-bot` enforces signed commits. If an organization member opens a PR that contains unsigned commits, it will be deemed untrusted and therefore require an `/ok to test` comment. See the GitHub docs [here](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) for information on how to set up commit signing. Any time a PR is deemed untrusted, it will receive a comment that looks like this: https://github.com/rapidsai/ci-imgs/pull/63#issuecomment-1688973208. Every subsequent commit on an untrusted PR will require an additional `/ok to test` comment. Any existing PRs that have unsigned commits after this change is merged will require an `/ok to test` comment for each subsequent commit _or_ the PR can be rebased to include signed commits as mentioned in the docs below: https://docs.gha-runners.nvidia.com/cpr/contributors. This information is all included on the documentation page linked above. _I've skipped CI on this PR since it's not a change that is tested._ [skip ci] --- .github/copy-pr-bot.yaml | 4 ++++ .github/ops-bot.yaml | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .github/copy-pr-bot.yaml diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml new file mode 100644 index 000000000..895ba83ee --- /dev/null +++ b/.github/copy-pr-bot.yaml @@ -0,0 +1,4 @@ +# Configuration file for `copy-pr-bot` GitHub App +# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ + +enabled: true diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml index 490610fa1..30009aba0 100644 --- a/.github/ops-bot.yaml +++ b/.github/ops-bot.yaml @@ -5,5 +5,4 @@ auto_merger: true branch_checker: false label_checker: false release_drafter: false -copy_prs: true recently_updated: true From 54fad39dcc6824795a107de93bd75c5318ac685c Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 7 Sep 2023 13:24:13 -0400 Subject: [PATCH 6/9] Simplify wheel build scripts and allow alphas of RAPIDS dependencies (#986) This PR: 1. Removes `ci/apply_wheel_modifications.sh` and uses it inline in wheel build scripts 2. Allows for specifying alpha versioned dependencies of RAPIDS projects Authors: - Divye Gala (https://github.com/divyegala) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/ucx-py/pull/986 --- ci/build_wheel.sh | 47 +++++++++++++++++++------ ci/release/apply_wheel_modifications.sh | 18 ---------- conda/recipes/ucx-py/meta.yaml | 15 ++++++++ dependencies.yaml | 16 +++++++-- 4 files changed, 64 insertions(+), 32 deletions(-) delete mode 100755 ci/release/apply_wheel_modifications.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 77c964f9b..e7592f289 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -3,6 +3,9 @@ set -euo pipefail +package_name="ucx-py" +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + source rapids-configure-sccache source rapids-date-string @@ -12,9 +15,31 @@ version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" -echo "The package name and/or version was modified in the package source. The git diff is:" -git diff +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="pyproject.toml" + +sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check @@ -27,7 +52,7 @@ python -m auditwheel repair -w final_dist dist/* # what these libraries are, we mimic the behaviour of auditwheel by using the # same hash-based uniqueness scheme and rewriting the link paths. -WHL=$(realpath final_dist/ucx_py*manylinux*.whl) +WHL=$(realpath final_dist/${underscore_package_name}*manylinux*.whl) # first grab the auditwheel hashes for libuc{tms} LIBUCM=$(unzip -l $WHL | awk 'match($4, /libucm-[^\.]+\./) { print substr($4, RSTART) }') @@ -36,11 +61,11 @@ LIBUCS=$(unzip -l $WHL | awk 'match($4, /libucs-[^\.]+\./) { print substr($4, RS LIBNUMA=$(unzip -l $WHL | awk 'match($4, /libnuma-[^\.]+\./) { print substr($4, RSTART) }') # Extract the libraries that have already been patched in by auditwheel -mkdir -p repair_dist/ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.libs/ucx -unzip $WHL "ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.libs/*.so*" -d repair_dist/ +mkdir -p repair_dist/${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.libs/ucx +unzip $WHL "${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.libs/*.so*" -d repair_dist/ # Patch the RPATH to include ORIGIN for each library -pushd repair_dist/ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.libs +pushd repair_dist/${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.libs for f in libu*.so* do if [[ -f $f ]]; then @@ -51,10 +76,10 @@ done popd # Now copy in all the extra libraries that are only ever loaded at runtime -pushd repair_dist/ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.libs/ucx +pushd repair_dist/${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.libs/ucx cp -P /usr/lib/ucx/* . -# we link against /lib/site-packages/ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.lib/libuc{ptsm} +# we link against /lib/site-packages/${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.lib/libuc{ptsm} # we also amend the rpath to search one directory above to *find* libuc{tsm} for f in libu*.so* do @@ -94,7 +119,7 @@ patchelf --add-rpath '$ORIGIN' libuct_cuda.so popd pushd repair_dist -zip -r $WHL ucx_py_${RAPIDS_PY_CUDA_SUFFIX}.libs/ +zip -r $WHL ${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}.libs/ popd -RAPIDS_PY_WHEEL_NAME="ucx_py_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh deleted file mode 100755 index d0e6bba43..000000000 --- a/ci/release/apply_wheel_modifications.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Usage: bash apply_wheel_modifications.sh - -VERSION=${1} -CUDA_SUFFIX=${2} - -# pyproject.toml versions -sed -i "s/^version = .*/version = \"${VERSION}\"/g" pyproject.toml - -# pyproject.toml cuda suffixes -sed -i "s/^name = \"ucx-py\"/name = \"ucx-py${CUDA_SUFFIX}\"/g" pyproject.toml -sed -i "s/cudf/cudf${CUDA_SUFFIX}/g" pyproject.toml - -if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" pyproject.toml -fi diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 1b88500a7..6c49d4e87 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -5,6 +5,7 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} package: @@ -27,7 +28,13 @@ requirements: host: - python - pip + {% if cuda_major == "11" %} - ucx + {% else %} + # Temporary pin because the subsequent build (h64cca9d_3) is missing proper + # CUDA support + - ucx==1.14.1=h195a15c_3 + {% endif %} {% for r in data.get("build-system", {}).get("requires", []) %} - {{ r }} {% endfor %} @@ -36,6 +43,14 @@ requirements: {% for r in data.get("project", {}).get("dependencies", []) %} - {{ r }} {% endfor %} + # Something seems wrong with the run export as well now + {% if cuda_major == "11" %} + - ucx + {% else %} + # Temporary pin because the subsequent build (h64cca9d_3) is missing proper + # CUDA support + - ucx==1.14.1=h195a15c_3 + {% endif %} test: imports: diff --git a/dependencies.yaml b/dependencies.yaml index 1634ae829..8b90862b5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -108,9 +108,19 @@ dependencies: packages: - numpy>=1.21 - pynvml>=11.4.1 - - output_types: [conda] - packages: - - ucx + specific: + - output_types: conda + matrices: + - matrix: + cuda: "12.0" + packages: + # Temporary pin because the subsequent build (h64cca9d_3) is + # missing proper CUDA support + - ucx==1.14.1=h195a15c_3 + - matrix: + cuda: "11.8" + packages: + - ucx test_python: common: - output_types: [conda, requirements, pyproject] From d380ce77c7f537d6032e20417089020786667a23 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 8 Sep 2023 13:05:51 -0500 Subject: [PATCH 7/9] Use `conda mambabuild` not `mamba mambabuild` (#988) With the release of conda 23.7.3, `mamba mambabuild` stopped working. With boa installed, `conda mambabuild` uses the mamba solver, so just use that instead. See also https://github.com/rapidsai/cudf/issues/14068. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/ucx-py/pull/988 --- ci/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 0c23034a8..de97365a4 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,7 +9,7 @@ rapids-print-env rapids-logger "Begin py build" -rapids-mamba-retry mambabuild \ +rapids-conda-retry mambabuild \ conda/recipes/ucx-py rapids-upload-conda-to-s3 python From cdaac6d638803ae5d0099cf9b741007c3e9c57f3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 8 Sep 2023 11:41:10 -0700 Subject: [PATCH 8/9] Undo manual pinning (#987) This PR is a test to validate that upstream ucx builds are no longer broken, at which point we will also be able to remove this pinning. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Benjamin Zaitlen (https://github.com/quasiben) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/ucx-py/pull/987 --- conda/recipes/ucx-py/meta.yaml | 14 -------------- dependencies.yaml | 14 ++------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 6c49d4e87..3599a900d 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -5,7 +5,6 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} package: @@ -28,13 +27,7 @@ requirements: host: - python - pip - {% if cuda_major == "11" %} - ucx - {% else %} - # Temporary pin because the subsequent build (h64cca9d_3) is missing proper - # CUDA support - - ucx==1.14.1=h195a15c_3 - {% endif %} {% for r in data.get("build-system", {}).get("requires", []) %} - {{ r }} {% endfor %} @@ -43,14 +36,7 @@ requirements: {% for r in data.get("project", {}).get("dependencies", []) %} - {{ r }} {% endfor %} - # Something seems wrong with the run export as well now - {% if cuda_major == "11" %} - ucx - {% else %} - # Temporary pin because the subsequent build (h64cca9d_3) is missing proper - # CUDA support - - ucx==1.14.1=h195a15c_3 - {% endif %} test: imports: diff --git a/dependencies.yaml b/dependencies.yaml index 8b90862b5..53fd9a84d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -108,19 +108,9 @@ dependencies: packages: - numpy>=1.21 - pynvml>=11.4.1 - specific: - output_types: conda - matrices: - - matrix: - cuda: "12.0" - packages: - # Temporary pin because the subsequent build (h64cca9d_3) is - # missing proper CUDA support - - ucx==1.14.1=h195a15c_3 - - matrix: - cuda: "11.8" - packages: - - ucx + packages: + - ucx test_python: common: - output_types: [conda, requirements, pyproject] From 8050cd0c63bdc5de424ce3e9d742c83cc2d4b19d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 21 Sep 2023 06:14:16 -0700 Subject: [PATCH 9/9] Remove ucx from explicit run reqs (#990) The ucx run requirement should be inferred from the run exports of ucx since it is part of the host requirements. The run explicit requirement was added to fix upstream issues with ucx packaging that have since been fixed. All other patches were removed in https://github.com/rapidsai/ucx-py/pull/987, but this reversion was missed. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/ucx-py/pull/990 --- conda/recipes/ucx-py/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 3599a900d..1b88500a7 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -36,7 +36,6 @@ requirements: {% for r in data.get("project", {}).get("dependencies", []) %} - {{ r }} {% endfor %} - - ucx test: imports: