From 65c21498f57ad5d4d9d1a12fa8ff2169a12e4ac8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Sep 2024 13:06:43 -0700 Subject: [PATCH] reduce config duplication, update third-party actions (#101) * add Python 3.12, reduce config duplication, update third-party actions * revert Python 3.12 changes * remove RAPIDS_VER from dask_image build --- .github/workflows/update-gpuci.yml | 12 +++--- .gitignore | 60 ++++++++++++++++++++++++++++++ README.md | 3 ++ ci/gpuci/run.sh | 15 +------- dask/Dockerfile | 14 +++---- dask_image/Dockerfile | 12 +++--- distributed/Dockerfile | 14 +++---- 7 files changed, 90 insertions(+), 40 deletions(-) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index 6afd8ad..ac8ccd1 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -11,10 +11,10 @@ jobs: if: github.repository == 'rapidsai/dask-build-environment' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Parse current axis YAML - uses: the-coding-turtle/ga-yaml-parser@v0.1.1 + uses: the-coding-turtle/ga-yaml-parser@v0.1.2 with: file: ci/axis/dask.yaml @@ -58,7 +58,7 @@ jobs: regex: false - name: Update old RAPIDS versions - uses: jacobtomlinson/gha-find-replace@v2 + uses: jacobtomlinson/gha-find-replace@v3 with: include: 'ci/**' find: "${{ env.OLD_RAPIDS_VER }}" @@ -66,7 +66,7 @@ jobs: regex: false - name: Update new UCX-Py versions - uses: jacobtomlinson/gha-find-replace@v2 + uses: jacobtomlinson/gha-find-replace@v3 with: include: 'ci/**' find: "${{ env.UCX_PY_VER }}" @@ -74,7 +74,7 @@ jobs: regex: false - name: Update old UCX-Py versions - uses: jacobtomlinson/gha-find-replace@v2 + uses: jacobtomlinson/gha-find-replace@v3 with: include: 'ci/**' find: "${{ env.OLD_UCX_PY_VER }}" @@ -83,7 +83,7 @@ jobs: - name: Create pull request if: ${{ env.RAPIDS_VER != env.NEW_RAPIDS_VER && env.UCX_PY_VER != env.NEW_UCX_PY_VER }} - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "Update gpuCI `RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb8b3e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,60 @@ +*.7zip +*.a +*.bak +*.bz +*.bz2 +*.conda +*.core +*.coverage +*.css +*.csv +*.dat +*.db +dist/ +*.dll +*.doc +*.docx +*.docm +.DS_Store +*.dylib +*.egg-info/ +*.env +*.exe +*.feather +*.html +htmlcov/ +.idea/ +*.js +*.json +*.lzma +.mypy_cache/ +*.npy +*.o +*.pdf +*.pem +*.ppt +*.pptx +*.pptm +*.pq +*.pub +*.pyc +__pycache__/ +.pytest_cache/ +*.rda +*.rds +*.Rdata +*.rsa +.ruff_cache/ +*.snappy-*.tar.gz +*.so +*.sqlite +*.swp +*.tar.gz +*.tgz +*.tmp +*.whl +*.xls +*.xlsx +*.xlsm +*.zip +*.zstd diff --git a/README.md b/README.md new file mode 100644 index 0000000..2bd3fd0 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# dask-build-environment + +Build environments for various dask related projects on gpuCI. diff --git a/ci/gpuci/run.sh b/ci/gpuci/run.sh index 312e1d4..2620bf9 100644 --- a/ci/gpuci/run.sh +++ b/ci/gpuci/run.sh @@ -29,18 +29,7 @@ case ${BUILD_NAME} in esac # Setup BUILD_ARGS -case $RAPIDS_VER in - "24.08") - UCX_PY_VER="0.39" - ;; - "24.10") - UCX_PY_VER="0.40" - ;; - *) - echo "Unrecognized RAPIDS_VER: ${RAPIDS_VER}" - exit 1 - ;; -esac +UCX_PY_VER="$(curl -sL https://version.gpuci.io/rapids/${RAPIDS_VER})" BUILD_IMAGE="gpuci/${BUILD_NAME}" BUILD_DIR="${WORKSPACE}/${BUILD_NAME}" @@ -48,7 +37,7 @@ BUILD_DIR="${WORKSPACE}/${BUILD_NAME}" case ${BUILD_NAME} in "dask_image") # doesn't depend on RAPIDS / ucx-py for gpuCI BUILD_TAG="cuda${CUDA_VER}-devel-${LINUX_VER}-py${PYTHON_VER}" - BUILD_ARGS="--squash --build-arg RAPIDS_VER=$RAPIDS_VER --build-arg CUDA_VER=$CUDA_VER --build-arg LINUX_VER=$LINUX_VER --build-arg PYTHON_VER=$PYTHON_VER" + BUILD_ARGS="--squash --build-arg CUDA_VER=$CUDA_VER --build-arg LINUX_VER=$LINUX_VER --build-arg PYTHON_VER=$PYTHON_VER" ;; *) BUILD_TAG="${RAPIDS_VER}-cuda${CUDA_VER}-devel-${LINUX_VER}-py${PYTHON_VER}" diff --git a/dask/Dockerfile b/dask/Dockerfile index aae65a9..f0687d0 100644 --- a/dask/Dockerfile +++ b/dask/Dockerfile @@ -1,13 +1,13 @@ -ARG CUDA_VER=11.8.0 -ARG LINUX_VER=ubuntu20.04 -ARG PYTHON_VER=3.10 +ARG CUDA_VER=unset +ARG LINUX_VER=unset +ARG PYTHON_VER=unset FROM rapidsai/miniforge-cuda:cuda$CUDA_VER-base-$LINUX_VER-py$PYTHON_VER -ARG CUDA_VER=11.8.0 -ARG PYTHON_VER=3.10 -ARG RAPIDS_VER=23.12 -ARG UCX_PY_VER=0.35 +ARG CUDA_VER=unset +ARG PYTHON_VER=unset +ARG RAPIDS_VER=unset +ARG UCX_PY_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml diff --git a/dask_image/Dockerfile b/dask_image/Dockerfile index 0f93e7c..87e52ed 100644 --- a/dask_image/Dockerfile +++ b/dask_image/Dockerfile @@ -1,13 +1,11 @@ -ARG CUDA_VER=11.8.0 -ARG LINUX_VER=ubuntu20.04 -ARG PYTHON_VER=3.10 +ARG CUDA_VER=unset +ARG LINUX_VER=unset +ARG PYTHON_VER=unset FROM rapidsai/miniforge-cuda:cuda$CUDA_VER-base-$LINUX_VER-py$PYTHON_VER -ARG CUDA_VER=11.8.0 -ARG PYTHON_VER=3.10 -# RAPIDS_VER isn't used but is part of the matrix so must be included -ARG RAPIDS_VER=23.12 +ARG CUDA_VER=unset +ARG PYTHON_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask-image/main/continuous_integration/environment-$PYTHON_VER.yml /dask.yml diff --git a/distributed/Dockerfile b/distributed/Dockerfile index 3e3460f..e86083b 100644 --- a/distributed/Dockerfile +++ b/distributed/Dockerfile @@ -1,13 +1,13 @@ -ARG CUDA_VER=11.8.0 -ARG LINUX_VER=ubuntu20.04 -ARG PYTHON_VER=3.10 +ARG CUDA_VER=unset +ARG LINUX_VER=unset +ARG PYTHON_VER=unset FROM rapidsai/miniforge-cuda:cuda$CUDA_VER-base-$LINUX_VER-py$PYTHON_VER -ARG CUDA_VER=11.8.0 -ARG PYTHON_VER=3.10 -ARG RAPIDS_VER=23.12 -ARG UCX_PY_VER=0.35 +ARG CUDA_VER=unset +ARG PYTHON_VER=unset +ARG RAPIDS_VER=unset +ARG UCX_PY_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/distributed/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml