From 31aa3369d4056813ddda532161e717e571c44ce9 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <128171450+sandeepd-nv@users.noreply.github.com>
Date: Tue, 26 Sep 2023 18:24:09 +0530
Subject: [PATCH] GH artifacts based CI (#1043)

---
 .github/actions/download-artifacts/action.yml |  48 ++++++
 .github/workflows/ci-gh.yml                   |  11 +-
 .github/workflows/gh-build-and-test.yml       |  88 +++++++++--
 .github/workflows/gh-build.yml                | 148 ++++++++----------
 .github/workflows/gh-cleanup.yml              |  43 -----
 .github/workflows/gh-test.yml                 |  91 +++++++++++
 cmake/versions.json                           |   2 +-
 conda/conda-build/meta.yaml                   |   4 +-
 continuous_integration/Dockerfile             |  44 ------
 continuous_integration/dot-gitconfig          |   3 +
 .../home/coder/.local/bin/build-cunumeric-all |  17 --
 .../scripts/build-cunumeric-all               |  44 ++++++
 .../bin => scripts}/build-cunumeric-conda     |  19 ++-
 .../bin => scripts}/build-cunumeric-cpp       |   1 -
 .../bin => scripts}/build-cunumeric-wheel     |   0
 continuous_integration/scripts/entrypoint     |  46 ++++++
 continuous_integration/scripts/test-cunumeric |  61 ++++++++
 17 files changed, 458 insertions(+), 212 deletions(-)
 create mode 100644 .github/actions/download-artifacts/action.yml
 delete mode 100644 .github/workflows/gh-cleanup.yml
 create mode 100644 .github/workflows/gh-test.yml
 delete mode 100644 continuous_integration/Dockerfile
 create mode 100644 continuous_integration/dot-gitconfig
 delete mode 100644 continuous_integration/home/coder/.local/bin/build-cunumeric-all
 create mode 100755 continuous_integration/scripts/build-cunumeric-all
 rename continuous_integration/{home/coder/.local/bin => scripts}/build-cunumeric-conda (86%)
 rename continuous_integration/{home/coder/.local/bin => scripts}/build-cunumeric-cpp (93%)
 rename continuous_integration/{home/coder/.local/bin => scripts}/build-cunumeric-wheel (100%)
 create mode 100755 continuous_integration/scripts/entrypoint
 create mode 100755 continuous_integration/scripts/test-cunumeric

diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml
new file mode 100644
index 000000000..e8019b1b1
--- /dev/null
+++ b/.github/actions/download-artifacts/action.yml
@@ -0,0 +1,48 @@
+name: setup-legate-conda
+
+description: Download dependencies (artifacts)
+
+inputs:
+  device: {type: string, required: true}
+  git_sha: {type: string, required: true}
+
+runs:
+  using: composite
+  steps:
+
+    - id: cache
+      name: Cache conda artifacts
+      uses: actions/cache@v3
+      with:
+        key: "nv-legate/legate.core@${{ inputs.git_sha }}-${{ inputs.device }}"
+        path: .artifacts
+
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Download conda artifacts
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        path: .artifacts-dl
+        repo: nv-legate/legate.core
+        commit: ${{ inputs.git_sha }}
+        workflow_conclusion: success
+        workflow: "ci-gh-${{ inputs.device }}-build-and-test.yml"
+        name: "legate.core-${{ inputs.device }}-[0-9a-z]{40}"
+        name_is_regexp: true
+
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Move conda artifacts into cached dir
+      shell: bash --noprofile --norc -xeo pipefail {0}
+      run: |
+        mkdir -p .artifacts;
+        find .artifacts-dl/legate.core-${{ inputs.device }}-*/ \
+          -maxdepth 2 -type d -name legate_core -exec mv {} .artifacts/ \;
+        find .artifacts-dl/legate.core-${{ inputs.device }}-*/ \
+          -maxdepth 2 -type f -name "environment*.yaml" -exec mv {} .artifacts/ \;
+
+    - name: Copy and change cache dir ownership
+      shell: bash --noprofile --norc -xeo pipefail {0}
+      run: |
+        # Copy and change directory ownership
+        cp -ar .artifacts /home/coder/.artifacts;
+        chown -R coder:coder /home/coder/.artifacts;
+        ls -R /home/coder/.artifacts
diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml
index f1aafdd59..ffb77c10e 100644
--- a/.github/workflows/ci-gh.yml
+++ b/.github/workflows/ci-gh.yml
@@ -16,10 +16,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {build-target: cpu}
-          - {build-target: gpu}
+          - device: "gpu"
+            image: "rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04"
+
+          - device: "cpu"
+            image: "rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04"
     uses:
       ./.github/workflows/gh-build-and-test.yml
     with:
-      build-target: ${{ matrix.build-target }}
-      sha: ${{ github.sha }}
+      device: ${{ matrix.device }}
+      image: ${{ matrix.image }}
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index f297b9761..e0e87899d 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -1,32 +1,92 @@
 on:
   workflow_call:
     inputs:
-      build-target:
-        required: true
+      image:
         type: string
-      sha:
         required: true
+      device:
         type: string
+        required: true
+
 
 jobs:
   build:
-    name: "Build cunumeric (with ${{ inputs.build-target }} legate) on GH"
+    name: "Build cunumeric (with ${{ inputs.device }} legate) on GH"
     uses:
       ./.github/workflows/gh-build.yml
     with:
-      build-target: ${{ inputs.build-target }}
-      # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4`
-      runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
-      sha: ${{ inputs.sha }}
+      device: ${{ inputs.device }}
+      image: ${{ inputs.image }}
+      runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-32cpu' || 'ubuntu-latest' }}
 
-  cleanup:
+  test:
     needs:
       - build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: 1 CPU test
+            options: test --cpus 1 --unit --debug
+            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu4' }}
+            has-gpu: false
+            enabled: true
+
+          - name: 2 CPUs test
+            options: test --cpus 2 --debug
+            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu8' }}
+            has-gpu: false
+            enabled: true
+
+          - name: GPU test
+            options: test --use cuda --gpus 1 --debug
+            runner: linux-amd64-gpu-v100-latest-1
+            has-gpu: true
+            enabled: ${{ inputs.device == 'gpu' }}
+
+          - name: 2 GPUs test
+            options: test --use cuda --gpus 2 --debug
+            runner: linux-amd64-2gpu
+            has-gpu: true
+            enabled: ${{ inputs.device == 'gpu' }}
+
+          - name: OpenMP test
+            options: test --use openmp --omps 1 --ompthreads 2 --debug
+            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-32cpu' }}
+            has-gpu: ${{ inputs.device == 'gpu' }}
+            enabled: false
+
+          - name: 2 NUMA OpenMPs test
+            options: test --use openmp --omps 2 --ompthreads 2 --numamem 2048 --debug
+            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-32cpu' }}
+            has-gpu: ${{ inputs.device == 'gpu' }}
+            enabled: false
+
+          - name: Eager execution test
+            options: test --use eager --debug
+            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu4' }}
+            has-gpu: ${{ inputs.device == 'gpu' }}
+            enabled: true
+
+          - name: mypy
+            options: mypy
+            runner: linux-amd64-cpu4
+            has-gpu: false
+            enabled: true
+
+          - name: documentation
+            options: docs
+            runner: linux-amd64-32cpu
+            has-gpu: false
+            enabled: ${{ inputs.device == 'gpu' }}
 
-    # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled.
-    if: always()
     uses:
-      ./.github/workflows/gh-cleanup.yml
+      ./.github/workflows/gh-test.yml
     with:
-      build-target: ${{ inputs.build-target }}
-      sha: ${{ inputs.sha }}
+      name: ${{ matrix.name }}
+      device: ${{ inputs.device }}
+      image: ${{ inputs.image }}
+      runs-on: ${{ matrix.runner }}
+      has-gpu: ${{ matrix.has-gpu }}
+      test-options: ${{ matrix.options }}
+      enabled: ${{ matrix.enabled }}
diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index c84ac0b9a..308b5f78c 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -1,123 +1,101 @@
-name: Build cunumeric on GH
+name: Build
 
 on:
   workflow_call:
     inputs:
-      build-target:
-        required: true
+      image:
         type: string
-      runs-on:
+        required: true
+      device:
         required: true
         type: string
-      sha:
+      runs-on:
         required: true
         type: string
 
-env:
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
-  IMAGE_NAME_LEGATE: legate.core-${{ inputs.build-target }}
-  IMAGE_NAME_CUNUMERIC: cunumeric-${{ inputs.build-target }}
-  USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }}
-
 jobs:
   build:
-    name: build-${{ inputs.build-target }}-sub-workflow
+    name: build-${{ inputs.device }}-sub-workflow
 
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
-      packages: write # This is required to push docker image to ghcr.io
-
+      
     runs-on: ${{ inputs.runs-on }}
 
-    steps:
-      - name: Checkout legate.core
-        uses: actions/checkout@v3
-        with:
-          repository: nv-legate/legate.core
-          fetch-depth: 0
-          path: legate
+    container:
+      options: -u root
+      image: "${{ inputs.image }}"
+      env:
+        CUDA_VERSION: "12.0"
+        CUDA_VERSION_MAJOR: "12"
+        CUDA_VERSION_MINOR: "0"
+        SCCACHE_REGION: "us-east-2"
+        SCCACHE_BUCKET: "rapids-sccache-devs"
+        SCCACHE_S3_KEY_PREFIX: "legate-cunumeric-dev"
+        USE_CUDA: "${{ inputs.device == 'gpu' && 'ON' || 'OFF' }}"
+        GH_TOKEN: "${{ env.GH_TOKEN }}"
+        GITHUB_TOKEN: "${{ env.GITHUB_TOKEN }}"
+        VAULT_HOST: "${{ github.repository_owner != 'nv-legate' && 'https://vault.ops.k8s.rapids.ai' || '' }}"
+    defaults:
+      run:
+        shell: su coder {0}
+        working-directory: /home/coder
 
+    steps:
       - name: Checkout cunumeric (= this repo)
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
           path: cunumeric
+          persist-credentials: false
 
-      - if: github.repository_owner == 'nv-legate'
-        name: Get AWS credentials for sccache bucket
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          aws-region: us-east-2
-          role-duration-seconds: 28800 # 8 hours
-          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
-
-      - name: Docker system prune
+      - name: Dump environment
         run: |
-          docker version
-          docker system prune --all --force
+          env
 
-      - name: Build legate.core using docker build
+      - name: Copy source folder
         run: |
-          echo BUILD_TARGET: ${{ inputs.build-target }}
-          echo USE_CUDA: ${{ env.USE_CUDA }}
-
-          export LEGATE_SHA=$(cat cunumeric/cmake/versions.json | jq -r '.packages.legate_core.git_tag')
-          echo "Checking out LEGATE_SHA: ${LEGATE_SHA}"
-          git -C legate checkout $LEGATE_SHA
-
-          IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }}
-
-          chmod +x legate/continuous_integration/build-docker-image
-          legate/continuous_integration/build-docker-image \
-            --base-image "$BASE_IMAGE" \
-            --image-tag "$IMAGE_TAG_LEGATE" \
-            --source-dir legate
-
-      - name: Build cunumeric using docker build
-        run: |
-          IMAGE_TAG_CUNUMERIC=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
-          IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }}
-
-          legate/continuous_integration/build-docker-image \
-            --base-image "$IMAGE_TAG_LEGATE" \
-            --image-tag "$IMAGE_TAG_CUNUMERIC" \
-            --source-dir cunumeric
-
-      - name: Dump docker history of image before upload
+          set -x
+          pwd
+          cp -r $GITHUB_WORKSPACE/cunumeric .
+          chown -R coder:coder cunumeric;
+          ls -R
+
+      - name: Copy .gitconfig
+        run: cp ~/cunumeric/continuous_integration/dot-gitconfig ~/.gitconfig
+
+      - id: legate_core_info
+        name: Read legate.core SHA
+        shell: bash --noprofile --norc -xeo pipefail {0}
         run: |
-          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
-          docker history $IMAGE_TAG
-
-      - name: Log in to container image registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
-
-      - name: Push cunumeric image
-        run: |
-          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
-
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}
+          git_tag="$(jq -r '.packages.legate_core.git_tag' cunumeric/cmake/versions.json)";
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          echo "git_tag=$git_tag" | tee -a "${GITHUB_OUTPUT}";
 
-          IMAGE_ID=$IMAGE_ID/$IMAGE_TAG
+      - name: Download dependencies (artifacts)
+        uses: ./cunumeric/.github/actions/download-artifacts
+        with:
+          device: "${{ inputs.device }}"
+          git_sha: "${{ steps.legate_core_info.outputs.git_tag }}"
 
-          docker tag $IMAGE_TAG $IMAGE_ID
-          docker push $IMAGE_ID
+      - if: github.repository_owner == 'nv-legate'
+        name: Get AWS credentials for sccache bucket
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-east-2
+          role-duration-seconds: 28800 # 8 hours
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
 
-      - name: Copy artifacts back to the host
+      - name: Build cunumeric
         run: |
-          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
-          mkdir -p artifacts
-          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts
-
-      - name: Display structure of workdir
-        run: ls -R
+          export PATH="/home/coder/cunumeric/continuous_integration/scripts:$PATH"
+          build-cunumeric-all
 
       - name: Upload build artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: "cunumeric-${{ inputs.build-target }}-${{ inputs.sha }}"
-          path: artifacts
+          name: "cunumeric-${{ inputs.device }}-${{ github.sha }}"
+          path: |
+            /tmp/out
+            /tmp/conda-build
diff --git a/.github/workflows/gh-cleanup.yml b/.github/workflows/gh-cleanup.yml
deleted file mode 100644
index 6451c401c..000000000
--- a/.github/workflows/gh-cleanup.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-name: Clean up
-
-on:
-  workflow_call:
-    inputs:
-      build-target:
-        required: true
-        type: string
-      sha:
-        required: true
-        type: string
-
-env:
-  IMAGE_NAME: cunumeric-${{ inputs.build-target }}
-
-jobs:
-  cleanup:
-    permissions:
-      packages: write
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Delete docker image
-        run: |
-          set -xeuo pipefail
-
-          PACKAGE_NAME=${{ env.IMAGE_NAME }}
-          PACKAGE_VERSION_ID=$(
-            curl -L \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ github.token }}"\
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions |
-            jq '.[] | select(.metadata.container.tags[] == "${{ inputs.sha }}") | .id' -
-          )
-
-          curl -L \
-            -X DELETE \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ github.token }}"\
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions/$PACKAGE_VERSION_ID
diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml
new file mode 100644
index 000000000..675f27e9b
--- /dev/null
+++ b/.github/workflows/gh-test.yml
@@ -0,0 +1,91 @@
+name: Test cunumeric on GH
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        required: true
+        type: string
+      image:
+        type: string
+        required: true
+      device:
+        required: true
+        type: string
+      runs-on:
+        required: true
+        type: string
+      has-gpu:
+        required: true
+        type: boolean
+        description: "The runner has GPU(s)."
+      test-options:
+        required: true
+        type: string
+      enabled:
+        required: true
+        type: boolean
+
+env:
+  build_artifact_name: "cunumeric-${{ inputs.device }}-${{ github.sha }}"
+
+jobs:
+  test:
+    name: ${{ inputs.name }}
+    if: inputs.enabled && github.repository_owner == 'nv-legate'
+    runs-on: ${{ inputs.runs-on }}
+
+    container:
+      options: -u root
+      image: "${{ inputs.image }}"
+      env:
+        # CUDA_VERSION: "${{ inputs.CUDA }}"
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+
+    defaults:
+      run:
+        shell: su coder {0}
+        working-directory: /home/coder
+
+    steps:
+      - if: inputs.has-gpu
+        name: Run nvidia-smi to make sure GPU is working
+        run: nvidia-smi
+
+      - name: Install numactl
+        run: |
+          export DEBIAN_FRONTEND=noninteractive && \
+            sudo apt-get update && \
+            sudo apt-get install -y numactl
+
+      - name: Checkout cunumeric
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: cunumeric
+          persist-credentials: false
+
+      - name: Copy source folder
+        run: |
+          set -x
+          pwd
+          cp -r $GITHUB_WORKSPACE/cunumeric .
+          chown -R coder:coder cunumeric;
+          ls -R
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.build_artifact_name }}
+          path: /home/coder/.artifacts
+
+      - name: Run cunumeric test / analysis
+        shell: su coder {0}
+        run: |
+          set -x
+          sudo chown -R coder:coder /home/coder/.artifacts
+
+          export PATH="/home/coder/cunumeric/continuous_integration/scripts:$PATH"
+
+          set -eo pipefail
+          test-cunumeric ${{ inputs.test-options }}
diff --git a/cmake/versions.json b/cmake/versions.json
index d1ae134ab..7a98894dd 100644
--- a/cmake/versions.json
+++ b/cmake/versions.json
@@ -5,7 +5,7 @@
       "git_url" : "https://github.com/nv-legate/legate.core.git",
       "git_shallow": false,
       "always_download": false,
-      "git_tag" : "14cca04834095553e4d88f503dc4cd35e4072212"
+      "git_tag" : "06b0e4d7fded0b4207fd8b4ba34c330333ee3543"
     }
   }
 }
diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml
index 81569e3a2..c652d931b 100644
--- a/conda/conda-build/meta.yaml
+++ b/conda/conda-build/meta.yaml
@@ -10,7 +10,7 @@
 ## The placeholder version is strictly for making two-pass conda build process.
 ## It should not be used for any other purpose, and this is not a default version.
 {% set placeholder_version = '0.0.0.dev' %}
-{% set default_cuda_version = '11.8' %}
+{% set default_cuda_version = '12.0' %}
 {% set cuda_version='.'.join(environ.get('CUDA', default_cuda_version).split('.')[:2]) %}
 {% set cuda_major=cuda_version.split('.')[0]|int %}
 {% set py_version=environ.get('CONDA_PY', '') %}
@@ -138,7 +138,7 @@ requirements:
     - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }}
     - cutensor >=1.3 =*_*
     - libcublas
-    - libcusolver =11.4.1.48-0
+    - libcusolver >=11.4.1.48-0
     - libcufft
 {% endif %}
     - opt_einsum >=3.3
diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
deleted file mode 100644
index 4e6478d14..000000000
--- a/continuous_integration/Dockerfile
+++ /dev/null
@@ -1,44 +0,0 @@
-ARG BASE_IMAGE
-FROM ${BASE_IMAGE} as stage0
-
-COPY --chown=coder:coder continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/
-COPY --chown=coder:coder . /home/coder/cunumeric
-
-RUN chmod a+x /home/coder/.local/bin/*
-
-#---------------------------------------------------
-FROM stage0 as setup
-
-USER coder
-WORKDIR /home/coder
-
-RUN set -x && . conda-utils && \
-    get_yaml_and_make_conda_env && \
-    install_legate_core_with_war
-
-#---------------------------------------------------
-FROM setup as build
-USER coder
-WORKDIR /home/coder
-
-ARG GITHUB_TOKEN
-ENV GITHUB_TOKEN=${GITHUB_TOKEN}
-ARG AWS_SESSION_TOKEN
-ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
-ARG AWS_ACCESS_KEY_ID
-ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-ARG AWS_SECRET_ACCESS_KEY
-ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-
-COPY --chown=coder:coder .creds /run/secrets
-
-RUN entrypoint build-cunumeric-all
-
-#---------------------------------------------------
-FROM stage0 as final
-USER coder
-WORKDIR /home/coder
-
-COPY --from=build --chown=coder:coder /tmp/out /tmp/out
-COPY --from=build --chown=coder:coder /tmp/conda-build /tmp/conda-build
-COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml
diff --git a/continuous_integration/dot-gitconfig b/continuous_integration/dot-gitconfig
new file mode 100644
index 000000000..91ac79c70
--- /dev/null
+++ b/continuous_integration/dot-gitconfig
@@ -0,0 +1,3 @@
+[user]
+	email = users.noreply.github.com
+	name = anon
\ No newline at end of file
diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-all b/continuous_integration/home/coder/.local/bin/build-cunumeric-all
deleted file mode 100644
index 62c6da0d7..000000000
--- a/continuous_integration/home/coder/.local/bin/build-cunumeric-all
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-
-build_cunumeric_all() {
-    set -x
-    cd ~/;
-
-    conda info
-
-    set -euo pipefail;
-
-    build-cunumeric-cpp;
-    build-cunumeric-wheel;
-    build-cunumeric-conda;
-}
-
-(build_cunumeric_all "$@");
diff --git a/continuous_integration/scripts/build-cunumeric-all b/continuous_integration/scripts/build-cunumeric-all
new file mode 100755
index 000000000..bcdbf62ec
--- /dev/null
+++ b/continuous_integration/scripts/build-cunumeric-all
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+setup_env() {
+    yaml_file=$(find ~/.artifacts -name "environment*.yaml" | head -n 1)
+
+    [ "${USE_CUDA:-}" = "ON" ] &&
+        echo "  - libcublas-dev" >> "${yaml_file}" &&
+        echo "  - libcufft-dev" >> "${yaml_file}" &&
+        echo "  - libcurand-dev" >> "${yaml_file}" &&
+        echo "  - libcusolver-dev" >> "${yaml_file}";
+
+    echo "YAML file..."
+    cat "${yaml_file}"
+
+    mkdir -p /tmp/out;
+
+    cp "${yaml_file}" /tmp/out
+
+    mamba env create -n legate -f "$yaml_file"
+
+    mamba uninstall -yn legate numpy
+
+    mamba install -yn legate -c ~/.artifacts/legate_core -c conda-forge -c nvidia legate-core
+
+    mamba activate legate
+}
+
+build_cunumeric_all() {
+    set -xeo pipefail
+
+    setup_env;
+    cd ~/cunumeric;
+    conda info;
+
+    set -xeuo pipefail;
+    printf "\n\n\n\n********* BUILDING CUNUMERIC CPP *********\n"
+    build-cunumeric-cpp;
+    printf "\n\n\n\n********* BUILDING CUNUMERIC WHEEL *********\n"
+    build-cunumeric-wheel;
+    printf "\n\n\n\n********* BUILDING CUNUMERIC CONDA *********\n"
+    build-cunumeric-conda;
+}
+
+(build_cunumeric_all "$@");
diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-conda b/continuous_integration/scripts/build-cunumeric-conda
similarity index 86%
rename from continuous_integration/home/coder/.local/bin/build-cunumeric-conda
rename to continuous_integration/scripts/build-cunumeric-conda
index 0be424252..ee4efefcb 100755
--- a/continuous_integration/home/coder/.local/bin/build-cunumeric-conda
+++ b/continuous_integration/scripts/build-cunumeric-conda
@@ -9,6 +9,9 @@ build_cunumeric_conda_package() {
         python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)";
     fi
 
+    mkdir -p /tmp/conda-build /tmp/out
+    cp -r ~/.artifacts/legate_core /tmp/conda-build/
+
     local conda_build_args=();
     conda_build_args+=(--override-channels);
     conda_build_args+=(-c conda-forge);
@@ -34,7 +37,21 @@ build_cunumeric_conda_package() {
 
     # Synthesize new cunumeric conda-build build.sh script
 
-    cat <<EOF >> ~/cunumeric/conda/conda-build/conda_build_config.yaml
+    cat <<EOF > ~/cunumeric/conda/conda-build/conda_build_config.yaml
+gpu_enabled:
+  - "${GPU_ENABLED}"
+
+python:
+  - "${python_version}"
+
+numpy_version:
+  - ">=1.22"
+
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
+use_local_path:
+  - "true"
 
 numpy:
   - 1.22
diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-cpp b/continuous_integration/scripts/build-cunumeric-cpp
similarity index 93%
rename from continuous_integration/home/coder/.local/bin/build-cunumeric-cpp
rename to continuous_integration/scripts/build-cunumeric-cpp
index 83f6dcd8c..fd08ceac2 100755
--- a/continuous_integration/home/coder/.local/bin/build-cunumeric-cpp
+++ b/continuous_integration/scripts/build-cunumeric-cpp
@@ -9,7 +9,6 @@ build_cunumeric_cpp() {
     cmake_args+=(-DBUILD_MARCH=${BUILD_MARCH:-haswell});
     cmake_args+=(-DCMAKE_BUILD_TYPE=Release);
     cmake_args+=(-DCMAKE_CUDA_ARCHITECTURES=RAPIDS);
-    cmake_args+=(-Dlegate_core_ROOT=$HOME/legate/build);
     cmake_args+=(-DCMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)});
     cmake_args+=(${@});
 
diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-wheel b/continuous_integration/scripts/build-cunumeric-wheel
similarity index 100%
rename from continuous_integration/home/coder/.local/bin/build-cunumeric-wheel
rename to continuous_integration/scripts/build-cunumeric-wheel
diff --git a/continuous_integration/scripts/entrypoint b/continuous_integration/scripts/entrypoint
new file mode 100755
index 000000000..298fc1c7a
--- /dev/null
+++ b/continuous_integration/scripts/entrypoint
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+sccache_stop_server_and_show_stats() {
+    sccache --stop-server || true && sccache --show-stats;
+}
+
+init_devcontainer() {
+    # disable xtrace and history
+    local xtrace_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'xtrace'; echo $?);
+    local history_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'history'; echo $?);
+    { set +xo history; } 2>/dev/null;
+    eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)";
+    if [ "${history_enabled}" -eq "0" ]; then { set -o history; } 2>/dev/null; fi;
+    if [ "${xtrace_enabled}" -eq "0" ]; then { set -o xtrace; } 2>/dev/null; fi;
+
+    set -xeo pipefail
+
+    . devcontainer-utils-post-attach-command;
+
+    sleep 10;
+    . devcontainer-utils-vault-s3-test;
+    . devcontainer-utils-vault-s3-export 0;
+}
+
+entrypoint() {
+    set -x
+
+    mkdir -p /home/coder/.cache;
+
+    local secrets_dir=/run/secrets
+
+    if [ -d "$secrets_dir" ] && [ "$(ls -A $secrets_dir)" ]; then
+        init_devcontainer
+    else
+        sccache_stop_server_and_show_stats
+    fi
+
+    echo AWS_REGION=${AWS_REGION:-}
+    echo AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
+    echo AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+    echo AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+
+    exec "$@";
+}
+
+entrypoint "$@";
diff --git a/continuous_integration/scripts/test-cunumeric b/continuous_integration/scripts/test-cunumeric
new file mode 100755
index 000000000..ca57b42e9
--- /dev/null
+++ b/continuous_integration/scripts/test-cunumeric
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+setup_env() {
+    mamba create -yn legate -c ~/.artifacts/conda-build/legate_core -c ~/.artifacts/conda-build/cunumeric -c conda-forge -c "nvidia/label/cuda-12.0.0" legate-core cunumeric
+}
+
+setup_test_env() {
+    mamba install -y "clang-tools>=8" "clang>=8" colorama coverage mock pre-commit pytest-cov pytest-lazy-fixture pytest-mock pytest types-docutils pynvml
+
+    pip install tifffile
+}
+
+setup_docs_env() {
+    mamba install -y pandoc doxygen
+    pip install ipython jinja2 "markdown<3.4.0" "pydata-sphinx-theme>=0.13"  myst-parser  nbsphinx  sphinx-copybutton  "sphinx>=4.4.0"
+
+}
+
+setup_mypy_env() {
+    mamba install -y "mypy>=0.961" jinja2 nbsphinx sphinx-copybutton  "sphinx>=4.4.0" types-docutils
+}
+
+test-cunumeric() {
+    set -xeo pipefail
+
+    setup_env;
+
+    set +u
+    mamba activate legate;
+    conda info;
+
+    cd ~/cunumeric;
+
+    case "$1" in
+        "test")
+            echo "Executing tests..."
+            shift;
+            setup_test_env;
+            ./test.py --verbose "$@"
+            ;;
+        "mypy")
+            echo "Installing and executing mypy..."
+            shift;
+            setup_mypy_env;
+            mypy cunumeric
+            ;;
+        "docs")
+            echo "Building docs..."
+            shift;
+            setup_docs_env;
+            cd docs/cunumeric
+            make clean html
+            ;;
+        *)
+            echo "Invalid command: $1"
+            return 1
+            ;;
+    esac
+}
+
+(test-cunumeric "$@");
\ No newline at end of file