v23.09.00

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.github/copy-pr-bot.yaml

@@ -0,0 +1,4 @@
+# Configuration file for `copy-pr-bot` GitHub App
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/
+
+enabled: true

.github/workflows/ci-gh.yml

@@ -1,7 +1,7 @@
-name: Build cunumeric on GH
+name: Build and test cunumeric on GH
 
 concurrency:
-  group: ci-gpu-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  group: ci-build-and-test-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -11,99 +11,15 @@ on:
       - "branch-*"
 
 jobs:
-  build:
-    permissions:
-      id-token: write # This is required for configure-aws-credentials
-      contents: read  # This is required for actions/checkout
-
-    # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4`
-    runs-on: ${{ github.repository == 'nv-legate/cunumeric' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
-      volumes:
-        - ${{ github.workspace }}/out:/tmp/out
-      env:
-        DEFAULT_CONDA_ENV: legate
-        PYTHONDONTWRITEBYTECODE: 1
-        SCCACHE_REGION: us-east-2
-        SCCACHE_BUCKET: rapids-sccache-east
-        SCCACHE_S3_KEY_PREFIX: legate-cunumeric-dev
-        GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}"
-        GITHUB_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}"
-        VAULT_HOST: "${{ secrets.PERSONAL_ACCESS_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
-        VAULT_S3_TTL: "28800s" # 8 hours
-
-    steps:
-      - name: Checkout legate.core
-        uses: actions/checkout@v3
-        with:
-          repository: nv-legate/legate.core
-          fetch-depth: 0
-          path: legate
-
-      - name: Checkout cunumeric (= this repo)
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          path: cunumeric
-
-      - name: Setup
-        shell: bash -eo pipefail {0}
-        run: |
-          export LEGATE_SHA=$(cat cunumeric/cmake/versions.json | jq -r '.packages.legate_core.git_tag')
-          echo "Checking out LEGATE_SHA: ${LEGATE_SHA}"
-          git -C legate checkout $LEGATE_SHA
-
-          cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/;
-          cp -ar legate/continuous_integration/home/coder/.local /home/coder/;
-          mv legate /home/coder/legate
-
-          cp -ar cunumeric/continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/;
-          mv cunumeric /home/coder/cunumeric;
-
-          chmod a+x /home/coder/.local/bin/*;
-          chown -R coder:coder /home/coder/;
-          chown -R coder:coder /tmp/out;
-
-      - if: github.repository == 'nv-legate/cunumeric'
-        name: Get AWS credentials for sccache bucket
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          aws-region: us-east-2
-          role-duration-seconds: 28800 # 8 hours
-          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
-
-      - name: Create conda env
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint make-conda-env;
-
-      - name: Build legate.core C++ library
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-legate-cpp;
-
-      - name: Build legate.core Python Wheel
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-legate-wheel;
-
-      - name: Build legate.core Conda Package
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-legate-conda;
-
-      - name: Build cunumeric C++ library
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-cunumeric-cpp;
-
-      - name: Build cunumeric Python Wheel
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-cunumeric-wheel;
-
-      - name: Build cunumeric Conda Package
-        shell: su coder {0}
-        run: cd ~/; exec entrypoint build-cunumeric-conda;
-
-      - name: Upload build output
-        uses: actions/upload-artifact@v3
-        with:
-          name: "cunumeric-${{ github.sha }}"
-          path: ./out/*
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {build-target: cpu}
+          - {build-target: gpu}
+    uses:
+      ./.github/workflows/gh-build-and-test.yml
+    with:
+      build-target: ${{ matrix.build-target }}
+      sha: ${{ github.sha }}

.github/workflows/gh-build-and-test.yml

@@ -0,0 +1,32 @@
+on:
+  workflow_call:
+    inputs:
+      build-target:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: "Build cunumeric (with ${{ inputs.build-target }} legate) on GH"
+    uses:
+      ./.github/workflows/gh-build.yml
+    with:
+      build-target: ${{ inputs.build-target }}
+      # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4`
+      runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
+      sha: ${{ inputs.sha }}
+
+  cleanup:
+    needs:
+      - build
+
+    # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled.
+    if: always()
+    uses:
+      ./.github/workflows/gh-cleanup.yml
+    with:
+      build-target: ${{ inputs.build-target }}
+      sha: ${{ inputs.sha }}

.github/workflows/gh-build.yml

@@ -0,0 +1,123 @@
+name: Build cunumeric on GH
+
+on:
+  workflow_call:
+    inputs:
+      build-target:
+        required: true
+        type: string
+      runs-on:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
+  IMAGE_NAME_LEGATE: legate.core-${{ inputs.build-target }}
+  IMAGE_NAME_CUNUMERIC: cunumeric-${{ inputs.build-target }}
+  USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }}
+
+jobs:
+  build:
+    name: build-${{ inputs.build-target }}-sub-workflow
+
+    permissions:
+      id-token: write # This is required for configure-aws-credentials
+      contents: read  # This is required for actions/checkout
+      packages: write # This is required to push docker image to ghcr.io
+
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout legate.core
+        uses: actions/checkout@v3
+        with:
+          repository: nv-legate/legate.core
+          fetch-depth: 0
+          path: legate
+
+      - name: Checkout cunumeric (= this repo)
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: cunumeric
+
+      - if: github.repository_owner == 'nv-legate'
+        name: Get AWS credentials for sccache bucket
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-east-2
+          role-duration-seconds: 28800 # 8 hours
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
+
+      - name: Docker system prune
+        run: |
+          docker version
+          docker system prune --all --force
+
+      - name: Build legate.core using docker build
+        run: |
+          echo BUILD_TARGET: ${{ inputs.build-target }}
+          echo USE_CUDA: ${{ env.USE_CUDA }}
+
+          export LEGATE_SHA=$(cat cunumeric/cmake/versions.json | jq -r '.packages.legate_core.git_tag')
+          echo "Checking out LEGATE_SHA: ${LEGATE_SHA}"
+          git -C legate checkout $LEGATE_SHA
+
+          IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }}
+
+          chmod +x legate/continuous_integration/build-docker-image
+          legate/continuous_integration/build-docker-image \
+            --base-image "$BASE_IMAGE" \
+            --image-tag "$IMAGE_TAG_LEGATE" \
+            --source-dir legate
+
+      - name: Build cunumeric using docker build
+        run: |
+          IMAGE_TAG_CUNUMERIC=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
+          IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }}
+
+          legate/continuous_integration/build-docker-image \
+            --base-image "$IMAGE_TAG_LEGATE" \
+            --image-tag "$IMAGE_TAG_CUNUMERIC" \
+            --source-dir cunumeric
+
+      - name: Dump docker history of image before upload
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
+          docker history $IMAGE_TAG
+
+      - name: Log in to container image registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push cunumeric image
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
+
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          IMAGE_ID=$IMAGE_ID/$IMAGE_TAG
+
+          docker tag $IMAGE_TAG $IMAGE_ID
+          docker push $IMAGE_ID
+
+      - name: Copy artifacts back to the host
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
+          mkdir -p artifacts
+          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts
+
+      - name: Display structure of workdir
+        run: ls -R
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: "cunumeric-${{ inputs.build-target }}-${{ inputs.sha }}"
+          path: artifacts

.github/workflows/gh-cleanup.yml

@@ -0,0 +1,43 @@
+name: Clean up
+
+on:
+  workflow_call:
+    inputs:
+      build-target:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+env:
+  IMAGE_NAME: cunumeric-${{ inputs.build-target }}
+
+jobs:
+  cleanup:
+    permissions:
+      packages: write
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Delete docker image
+        run: |
+          set -xeuo pipefail
+
+          PACKAGE_NAME=${{ env.IMAGE_NAME }}
+          PACKAGE_VERSION_ID=$(
+            curl -L \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ github.token }}"\
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions |
+            jq '.[] | select(.metadata.container.tags[] == "${{ inputs.sha }}") | .id' -
+          )
+
+          curl -L \
+            -X DELETE \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ github.token }}"\
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions/$PACKAGE_VERSION_ID

.gitignore

@@ -41,3 +41,4 @@ cunumeric/install_info.py
 /_skbuild
 /_cmake_test_compile
 .ipynb_checkpoints
+.legate-test-last-failed

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: 'v1.4.1'
+      rev: 'v1.5.1'
       hooks:
         - id: mypy
           language: system

CMakeLists.txt

@@ -55,7 +55,7 @@ include(rapids-cuda)
 include(rapids-export)
 include(rapids-find)
 
-set(cunumeric_version 23.07.00)
+set(cunumeric_version 23.09.00)
 
 # For now we want the optimization flags to match on both normal make and cmake
 # builds so we override the cmake defaults here for release, this changes

cmake/versions.json

@@ -1,11 +1,11 @@
 {
   "packages" : {
     "legate_core" : {
-      "version": "23.07.00",
+      "version": "23.09.00",
       "git_url" : "https://github.com/nv-legate/legate.core.git",
       "git_shallow": false,
       "always_download": false,
-      "git_tag" : "ac75ac05a9056f49729797415ee489b45686a528"
+      "git_tag" : "14cca04834095553e4d88f503dc4cd35e4072212"
     }
   }
 }

conda/conda-build/meta.yaml

@@ -135,6 +135,7 @@ requirements:
 {% else %}
     - legate-core ={{ core_version }}
     - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }}
+    - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }}
     - cutensor >=1.3 =*_*
     - libcublas
     - libcusolver =11.4.1.48-0