add workflow to upload conda packages to legate channel

.github/workflows/build.yaml

@@ -10,9 +10,8 @@ on:
     branches:
       - "main"
       - "release/v[0-9][0-9].[0-9][0-9].[0-9][0-9]"
-    # run on pushes of new release tags
+    # run on pushes of any tags
     tags:
-      - v[0-9][0-9].[0-9][0-9].[0-9][0-9]
   # run by clicking buttons in the GitHub Actions UI
   workflow_dispatch:
 
@@ -22,6 +21,11 @@ jobs:
     with:
       script: "ci/build_python.sh"
     secrets: inherit
+  upload-conda:
+    needs:
+      - conda-python-build
+    uses: ./.github/workflows/conda-upload-packages.yaml
+    secrets: inherit
   docs-build:
     needs:
       - conda-python-build

.github/workflows/conda-upload-packages.yaml

@@ -0,0 +1,39 @@
+# adopted from https://github.com/rapidsai/shared-workflows/blob/branch-24.12/.github/workflows/conda-upload-packages.yaml
+# with notable exceptions:
+#
+#   * assumes packages were uploaded to GitHub artifact store, not Amazon S3
+#   * always publishes to the same channel, but uses different label for non-release packages
+#
+
+on:
+  # run only when called by other workflows
+  workflow_call:
+
+env:
+  # where jobs that download conda packages store the local channel
+  RAPIDS_LOCAL_CONDA_CHANNEL: /tmp/local-conda-packages
+
+jobs:
+  upload:
+    runs-on: linux-amd64-cpu4
+    container:
+      image: rapidsai/ci-conda:latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          # the notebooks and PNG files stored in git LFS aren't necessary for package uploads
+          lfs: false
+      - name: download conda packages
+        uses: actions/download-artifact@v4
+        with:
+          # omitting 'name' here means "download all artifacts from this run"... useful to
+          # avoid having to list the matrix of CUDA / Python versions here
+          path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.run_id }}
+      - name: Upload packages
+        run: "ci/upload-to-anaconda.sh"
+        env:
+          CONDA_LEGATE_TOKEN: ${{ secrets.CONDA_LEGATE_TOKEN }}

.github/workflows/github-actions.yml

@@ -23,8 +23,6 @@ jobs:
   # group together all jobs that must pass for a PR to be merged
   # (for use by branch protections)
   pr-builder:
-    # skip on builds from merges to main
-    if: github.ref != 'refs/heads/main'
     needs:
       - pre-commit
       - conda-python-build
@@ -47,92 +45,99 @@ jobs:
       script: "ci/build_python.sh"
     secrets: inherit
 
-  conda-python-tests-cpu:
+  # TODO: remove this from the PR workflow before merging
+  upload-conda:
     needs:
-      - pre-commit
       - conda-python-build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - ARCH: "amd64"
-            CUDA_VER: "12.5.1"
-            PY_VER: "3.11"
-    runs-on: linux-${{ matrix.ARCH }}-cpu16
-    container:
-      image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}"
-    steps:
-      - name: install git-lfs
-        run: |
-          conda install --yes \
-            -c conda-forge \
-              git-lfs
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: download conda packages
-        uses: actions/download-artifact@v4
-        with:
-          name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }}
-          path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.run_id }}
-      - name: test
-        run: |
-          ci/test_python_cpu.sh
+    uses: ./.github/workflows/conda-upload-packages.yaml
+    secrets: inherit
 
-  conda-python-tests-gpu:
-    needs:
-      - pre-commit
-      - conda-python-build
-    strategy:
-      fail-fast: false
-      # As of the last time this was updated, legate / cunumeric packages were published for only:
-      #
-      #  * architectures: amd64 only
-      #  * CUDA: >=12.2
-      #  * Python: 3.10, 3.11, 3.12
-      #
-      # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags
-      matrix:
-        include:
-          - ARCH: "amd64"
-            CUDA_VER: "12.5.1"
-            PY_VER: "3.10"
-    runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1
-    container:
-      image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}"
-      env:
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-    steps:
-      - name: install git-lfs
-        run: |
-          conda install --yes \
-            -c conda-forge \
-              git-lfs
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: download conda packages
-        uses: actions/download-artifact@v4
-        with:
-          name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }}
-          path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.run_id }}
-      - name: test
-        run: |
-          ci/test_python_gpu.sh
+  # conda-python-tests-cpu:
+  #   needs:
+  #     - pre-commit
+  #     - conda-python-build
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include:
+  #         - ARCH: "amd64"
+  #           CUDA_VER: "12.5.1"
+  #           PY_VER: "3.11"
+  #   runs-on: linux-${{ matrix.ARCH }}-cpu16
+  #   container:
+  #     image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}"
+  #   steps:
+  #     - name: install git-lfs
+  #       run: |
+  #         conda install --yes \
+  #           -c conda-forge \
+  #             git-lfs
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+  #         lfs: true
+  #     - name: download conda packages
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }}
+  #         path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }}
+  #         github-token: ${{ secrets.GITHUB_TOKEN }}
+  #         repository: ${{ github.repository }}
+  #         run-id: ${{ github.run_id }}
+  #     - name: test
+  #       run: |
+  #         ci/test_python_cpu.sh
 
-  docs-build:
-    needs:
-      - conda-python-build
-    uses: ./.github/workflows/docs-build.yaml
-    with:
-      script: "ci/build_docs.sh"
-      deploy: false
-    secrets: inherit
+  # conda-python-tests-gpu:
+  #   needs:
+  #     - pre-commit
+  #     - conda-python-build
+  #   strategy:
+  #     fail-fast: false
+  #     # As of the last time this was updated, legate / cunumeric packages were published for only:
+  #     #
+  #     #  * architectures: amd64 only
+  #     #  * CUDA: >=12.2
+  #     #  * Python: 3.10, 3.11, 3.12
+  #     #
+  #     # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags
+  #     matrix:
+  #       include:
+  #         - ARCH: "amd64"
+  #           CUDA_VER: "12.5.1"
+  #           PY_VER: "3.10"
+  #   runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1
+  #   container:
+  #     image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}"
+  #     env:
+  #       NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+  #   steps:
+  #     - name: install git-lfs
+  #       run: |
+  #         conda install --yes \
+  #           -c conda-forge \
+  #             git-lfs
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+  #         lfs: true
+  #     - name: download conda packages
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }}
+  #         path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }}
+  #         github-token: ${{ secrets.GITHUB_TOKEN }}
+  #         repository: ${{ github.repository }}
+  #         run-id: ${{ github.run_id }}
+  #     - name: test
+  #       run: |
+  #         ci/test_python_gpu.sh
+
+  # docs-build:
+  #   needs:
+  #     - conda-python-build
+  #   uses: ./.github/workflows/docs-build.yaml
+  #   with:
+  #     script: "ci/build_docs.sh"
+  #     deploy: false
+  #   secrets: inherit

ci/upload-to-anaconda.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# adopted from https://github.com/rapidsai/gha-tools/blob/main/tools/rapids-upload-to-anaconda, with some exceptions:
+#
+#  * assumes artifacts are on GitHub Actions artifact store, not Amazon S3
+#  * assumes packages have been unpacked to env variable RAPIDS_LOCAL_CONDA_CHANNEL
+#  * does not differentiate between pull request and nightly branches
+#    (relies on workflow triggers to just not run this script when it isn't needed)
+
+# publish to the 'experimental' label on the 'legate' channel, for all cases except
+# releases (builds triggered by pushing a tag matching this regex exactly, e.g. 'v24.09.00')
+if [[ "${GITHUB_REF}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]];
+    declare -r conda_label="main"
+else
+    declare -r conda_label="experimental"
+fi
+
+PKGS_TO_UPLOAD=$(rapids-find-anaconda-uploads.py "${RAPIDS_LOCAL_CONDA_CHANNEL}")
+
+if [ -z "${PKGS_TO_UPLOAD}" ]; then
+    rapids-echo-stderr "Couldn't find any packages to upload in: ${RAPIDS_LOCAL_CONDA_CHANNEL}"
+    ls -l "${RAPIDS_LOCAL_CONDA_CHANNEL}/"*
+    continue
+fi
+
+rapids-echo-stderr "Uploading packages to Anaconda.org (channel='legate', label='${conda_label}'): ${PKGS_TO_UPLOAD}"
+
+# export RAPIDS_RETRY_SLEEP=180
+# # shellcheck disable=SC2086
+# rapids-retry anaconda \
+#     -t "${CONDA_LEGATE_TOKEN}" \
+#     upload \
+#     --label "${RAPIDS_CONDA_UPLOAD_LABEL:-main}" \
+#     --skip-existing \
+#     --no-progress \
+#     ${PKGS_TO_UPLOAD}