diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 000000000..3e4e48b0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index a81802f9d..f1aafdd59 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -1,7 +1,7 @@ -name: Build cunumeric on GH +name: Build and test cunumeric on GH concurrency: - group: ci-gpu-on-${{ github.event_name }}-from-${{ github.ref_name }} + group: ci-build-and-test-on-${{ github.event_name }}-from-${{ github.ref_name }} cancel-in-progress: true on: @@ -11,99 +11,15 @@ on: - "branch-*" jobs: - build: - permissions: - id-token: write # This is required for configure-aws-credentials - contents: read # This is required for actions/checkout - - # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` - runs-on: ${{ github.repository == 'nv-legate/cunumeric' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} - container: - options: -u root - image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 - volumes: - - ${{ github.workspace }}/out:/tmp/out - env: - DEFAULT_CONDA_ENV: legate - PYTHONDONTWRITEBYTECODE: 1 - SCCACHE_REGION: us-east-2 - SCCACHE_BUCKET: rapids-sccache-east - SCCACHE_S3_KEY_PREFIX: legate-cunumeric-dev - GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" - GITHUB_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" - VAULT_HOST: "${{ secrets.PERSONAL_ACCESS_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}" - VAULT_S3_TTL: "28800s" # 8 hours - - steps: - - name: Checkout legate.core - uses: actions/checkout@v3 - with: - repository: nv-legate/legate.core - fetch-depth: 0 - path: legate - - - name: Checkout cunumeric (= this repo) - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: cunumeric - - - name: Setup - shell: bash -eo pipefail {0} - run: | - export LEGATE_SHA=$(cat cunumeric/cmake/versions.json | jq -r '.packages.legate_core.git_tag') - echo "Checking out LEGATE_SHA: ${LEGATE_SHA}" - git -C legate checkout $LEGATE_SHA - - cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/; - cp -ar legate/continuous_integration/home/coder/.local /home/coder/; - mv legate /home/coder/legate - - cp -ar cunumeric/continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/; - mv cunumeric /home/coder/cunumeric; - - chmod a+x /home/coder/.local/bin/*; - chown -R coder:coder /home/coder/; - chown -R coder:coder /tmp/out; - - - if: github.repository == 'nv-legate/cunumeric' - name: Get AWS credentials for sccache bucket - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: us-east-2 - role-duration-seconds: 28800 # 8 hours - role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate - - - name: Create conda env - shell: su coder {0} - run: cd ~/; exec entrypoint get-yaml-and-make-conda-env; - - - name: Build legate.core C++ library - shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-cpp; - - - name: Build legate.core Python Wheel - shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-wheel; - - - name: Build legate.core Conda Package - shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-conda; - - - name: Build cunumeric C++ library - shell: su coder {0} - run: cd ~/; exec entrypoint build-cunumeric-cpp; - - - name: Build cunumeric Python Wheel - shell: su coder {0} - run: cd ~/; exec entrypoint build-cunumeric-wheel; - - - name: Build cunumeric Conda Package - shell: su coder {0} - run: cd ~/; exec entrypoint build-cunumeric-conda; - - - name: Upload build output - uses: actions/upload-artifact@v3 - with: - name: "cunumeric-${{ github.sha }}" - path: ./out/* + build-and-test: + strategy: + fail-fast: false + matrix: + include: + - {build-target: cpu} + - {build-target: gpu} + uses: + ./.github/workflows/gh-build-and-test.yml + with: + build-target: ${{ matrix.build-target }} + sha: ${{ github.sha }} diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml new file mode 100644 index 000000000..f297b9761 --- /dev/null +++ b/.github/workflows/gh-build-and-test.yml @@ -0,0 +1,32 @@ +on: + workflow_call: + inputs: + build-target: + required: true + type: string + sha: + required: true + type: string + +jobs: + build: + name: "Build cunumeric (with ${{ inputs.build-target }} legate) on GH" + uses: + ./.github/workflows/gh-build.yml + with: + build-target: ${{ inputs.build-target }} + # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` + runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + sha: ${{ inputs.sha }} + + cleanup: + needs: + - build + + # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled. + if: always() + uses: + ./.github/workflows/gh-cleanup.yml + with: + build-target: ${{ inputs.build-target }} + sha: ${{ inputs.sha }} diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml new file mode 100644 index 000000000..c84ac0b9a --- /dev/null +++ b/.github/workflows/gh-build.yml @@ -0,0 +1,123 @@ +name: Build cunumeric on GH + +on: + workflow_call: + inputs: + build-target: + required: true + type: string + runs-on: + required: true + type: string + sha: + required: true + type: string + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 + IMAGE_NAME_LEGATE: legate.core-${{ inputs.build-target }} + IMAGE_NAME_CUNUMERIC: cunumeric-${{ inputs.build-target }} + USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} + +jobs: + build: + name: build-${{ inputs.build-target }}-sub-workflow + + permissions: + id-token: write # This is required for configure-aws-credentials + contents: read # This is required for actions/checkout + packages: write # This is required to push docker image to ghcr.io + + runs-on: ${{ inputs.runs-on }} + + steps: + - name: Checkout legate.core + uses: actions/checkout@v3 + with: + repository: nv-legate/legate.core + fetch-depth: 0 + path: legate + + - name: Checkout cunumeric (= this repo) + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: cunumeric + + - if: github.repository_owner == 'nv-legate' + name: Get AWS credentials for sccache bucket + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: us-east-2 + role-duration-seconds: 28800 # 8 hours + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate + + - name: Docker system prune + run: | + docker version + docker system prune --all --force + + - name: Build legate.core using docker build + run: | + echo BUILD_TARGET: ${{ inputs.build-target }} + echo USE_CUDA: ${{ env.USE_CUDA }} + + export LEGATE_SHA=$(cat cunumeric/cmake/versions.json | jq -r '.packages.legate_core.git_tag') + echo "Checking out LEGATE_SHA: ${LEGATE_SHA}" + git -C legate checkout $LEGATE_SHA + + IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }} + + chmod +x legate/continuous_integration/build-docker-image + legate/continuous_integration/build-docker-image \ + --base-image "$BASE_IMAGE" \ + --image-tag "$IMAGE_TAG_LEGATE" \ + --source-dir legate + + - name: Build cunumeric using docker build + run: | + IMAGE_TAG_CUNUMERIC=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} + IMAGE_TAG_LEGATE=${{ env.IMAGE_NAME_LEGATE }}:${{ inputs.sha }} + + legate/continuous_integration/build-docker-image \ + --base-image "$IMAGE_TAG_LEGATE" \ + --image-tag "$IMAGE_TAG_CUNUMERIC" \ + --source-dir cunumeric + + - name: Dump docker history of image before upload + run: | + IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} + docker history $IMAGE_TAG + + - name: Log in to container image registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + + - name: Push cunumeric image + run: | + IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} + + IMAGE_ID=ghcr.io/${{ github.repository_owner }} + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + IMAGE_ID=$IMAGE_ID/$IMAGE_TAG + + docker tag $IMAGE_TAG $IMAGE_ID + docker push $IMAGE_ID + + - name: Copy artifacts back to the host + run: | + IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} + mkdir -p artifacts + docker run -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts + + - name: Display structure of workdir + run: ls -R + + - name: Upload build artifacts + uses: actions/upload-artifact@v3 + with: + name: "cunumeric-${{ inputs.build-target }}-${{ inputs.sha }}" + path: artifacts diff --git a/.github/workflows/gh-cleanup.yml b/.github/workflows/gh-cleanup.yml new file mode 100644 index 000000000..6451c401c --- /dev/null +++ b/.github/workflows/gh-cleanup.yml @@ -0,0 +1,43 @@ +name: Clean up + +on: + workflow_call: + inputs: + build-target: + required: true + type: string + sha: + required: true + type: string + +env: + IMAGE_NAME: cunumeric-${{ inputs.build-target }} + +jobs: + cleanup: + permissions: + packages: write + + runs-on: ubuntu-latest + + steps: + - name: Delete docker image + run: | + set -xeuo pipefail + + PACKAGE_NAME=${{ env.IMAGE_NAME }} + PACKAGE_VERSION_ID=$( + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ github.token }}"\ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions | + jq '.[] | select(.metadata.container.tags[] == "${{ inputs.sha }}") | .id' - + ) + + curl -L \ + -X DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ github.token }}"\ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions/$PACKAGE_VERSION_ID diff --git a/cmake/versions.json b/cmake/versions.json index 98dbd8c3a..d7e1d8133 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -5,7 +5,7 @@ "git_url" : "https://github.com/nv-legate/legate.core.git", "git_shallow": false, "always_download": false, - "git_tag" : "a405f595603238c8557cb5fefd3981d190a2fb1d" + "git_tag" : "4b79075eb5d7035d501c334c87a87939af79abc2" } } } diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile new file mode 100644 index 000000000..4e6478d14 --- /dev/null +++ b/continuous_integration/Dockerfile @@ -0,0 +1,44 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as stage0 + +COPY --chown=coder:coder continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/ +COPY --chown=coder:coder . /home/coder/cunumeric + +RUN chmod a+x /home/coder/.local/bin/* + +#--------------------------------------------------- +FROM stage0 as setup + +USER coder +WORKDIR /home/coder + +RUN set -x && . conda-utils && \ + get_yaml_and_make_conda_env && \ + install_legate_core_with_war + +#--------------------------------------------------- +FROM setup as build +USER coder +WORKDIR /home/coder + +ARG GITHUB_TOKEN +ENV GITHUB_TOKEN=${GITHUB_TOKEN} +ARG AWS_SESSION_TOKEN +ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} +ARG AWS_ACCESS_KEY_ID +ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} +ARG AWS_SECRET_ACCESS_KEY +ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + +COPY --chown=coder:coder .creds /run/secrets + +RUN entrypoint build-cunumeric-all + +#--------------------------------------------------- +FROM stage0 as final +USER coder +WORKDIR /home/coder + +COPY --from=build --chown=coder:coder /tmp/out /tmp/out +COPY --from=build --chown=coder:coder /tmp/conda-build /tmp/conda-build +COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-all b/continuous_integration/home/coder/.local/bin/build-cunumeric-all new file mode 100644 index 000000000..62c6da0d7 --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/build-cunumeric-all @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + + +build_cunumeric_all() { + set -x + cd ~/; + + conda info + + set -euo pipefail; + + build-cunumeric-cpp; + build-cunumeric-wheel; + build-cunumeric-conda; +} + +(build_cunumeric_all "$@"); diff --git a/continuous_integration/home/coder/.local/bin/build-cunumeric-conda b/continuous_integration/home/coder/.local/bin/build-cunumeric-conda index b957e13af..0be424252 100755 --- a/continuous_integration/home/coder/.local/bin/build-cunumeric-conda +++ b/continuous_integration/home/coder/.local/bin/build-cunumeric-conda @@ -23,7 +23,11 @@ build_cunumeric_conda_package() { conda_build_args+=("--build-id-pat=''"); conda_build_args+=(--no-include-recipe); conda_build_args+=(--no-anaconda-upload); - conda_build_args+=(--variants "{gpu_enabled:true,python:${python_version}}"); + + GPU_ENABLED=true + [ "${USE_CUDA:-}" = "OFF" ] && GPU_ENABLED=false + + conda_build_args+=(--variants "{gpu_enabled:${GPU_ENABLED},python:${python_version}}"); rm -rf /tmp/conda-build/cunumeric; mkdir -p /tmp/conda-build/cunumeric;