Merge pull request #84 from sandeepd-nv/add_ci1

Implement Github CI
NVIDIA · Aug 26, 2024 · e1e3325 · e1e3325
2 parents e83073a + 01389ef
commit e1e3325
Show file tree

Hide file tree

Showing 15 changed files with 718 additions and 0 deletions.
diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2
@@ -0,0 +1,90 @@
+name: build
+
+description: Build specified project
+
+inputs:
+  build-type:
+    required: true
+    type: string
+    description: One of ci / release
+  target-device:
+    required: true
+    type: string
+  host-platform:
+    required: true
+    type: string
+  use-container:
+    required: true
+    type: boolean
+  docker-image:
+    type: string
+    required: true
+  upload-enabled:
+    required: true
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+
+<% for package_id, package_info in packages.items() %>
+    - name: Download <<package_info.repo>> (artifacts)
+      uses: ./.github/actions/download-artifacts
+      with:
+        artifact-repo: "<<package_info.repo>>"
+        artifact-name: "<<package_info.artifact_name | replace_placeholder('repo', package_info.repo) | replace_placeholder('git_tag', package_info.git_tag) >>"
+        target-device: "${{ inputs.target-device }}"
+        git_sha: "<<package_info.git_tag>>"
+        host-platform: ${{ inputs.host-platform }}
+        dest-dir: ${{ env.ARTIFACTS_DIR }}
+        dependencies-workflow: <<package_info.artifact_workflow>>
+<% endfor %>
+
+<% if packages %>
+
+    - name: Display structure of downloaded artifacts
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        pwd
+        ls -lahR ${{ env.ARTIFACTS_DIR }}
+<% endif %>
+
+    - if: ${{ inputs.use-container }}
+      name: Build (in container)
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+
+        docker run \
+          -e AWS_REGION \
+          -e AWS_SESSION_TOKEN \
+          -e AWS_ACCESS_KEY_ID \
+          -e AWS_SECRET_ACCESS_KEY \
+          -e GITHUB_TOKEN \
+          -e ARTIFACTS_DIR="$ARTIFACTS_DIR" \
+          -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
+          -e USE_CUDA="$USE_CUDA" \
+          -e REPO_DIR="$REPO_DIR" \
+          -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
+          -e PYTHON_VERSION="$PYTHON_VERSION" \
+          -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
+          -v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \
+          --rm "${{ inputs.docker-image }}" \
+          /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
+
+    - if: ${{ !inputs.use-container }}
+      name: Build (without container)
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
+
+    - name: Display structure of the artifacts folder (post build)
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }}
+        ls -lahR ${{ env.ARTIFACTS_DIR }}
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+        path: ${{ env.ARTIFACTS_DIR }}
diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml
@@ -0,0 +1,59 @@
+name: download-artifacts
+
+description: Download dependencies (artifacts)
+
+inputs:
+  artifact-repo:
+    type: string
+    require: true
+  artifact-name:
+    type: string
+    require: true
+  target-device:
+    type: string
+    required: true
+  git_sha:
+    type: string
+    required: true
+  host-platform:
+    type: string
+    required: true
+  dest-dir:
+    type: string
+    required: true
+  dependencies-workflow:
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+
+    - id: cache
+      name: Cache conda artifacts
+      uses: actions/cache@v4
+      with:
+        key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.host-platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}"
+        path: ${{ inputs.dest-dir }}
+
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Download ${{ inputs.artifact-repo }} artifacts
+      uses: dawidd6/action-download-artifact@v3
+      with:
+        path: ${{ inputs.dest-dir }}
+        repo: nvidia/${{ inputs.artifact-repo }}
+        check_artifacts: true
+        commit: ${{ inputs.git_sha }}
+        workflow_conclusion: ""
+        workflow: ${{ inputs.dependencies-workflow }}
+        name: ${{ inputs.artifact-name }}
+        skip_unpack: true
+        if_no_artifact_found: fail
+        allow_forks: false
+
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Unpack artifact
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        cd ${{ inputs.dest-dir }}
+        unzip *.zip
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -0,0 +1,68 @@
+name: Common setup
+
+inputs:
+  client-repo:
+    required: true
+    type: string
+  build-type:
+    required: true
+    type: string
+  target-device:
+    required: true
+    type: string
+  host-platform:
+    required: true
+    type: string
+  build-mode:
+    required: true
+    type: string
+  upload-enabled:
+    required: true
+    type: boolean
+  python-version:
+    required: false
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Set REPO_DIR and Dump environment
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV
+        env
+
+    - name: Set environment variables
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+
+        WITH_TESTS_STR=''
+        if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then
+          WITH_TESTS_STR='-with_tests'
+        fi
+
+        TARGET_PLATFORM='linux-64'
+        if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
+          TARGET_PLATFORM='linux-aarch64'
+        fi
+
+        BUILD_MODE="${{ inputs.build-mode }}"
+        BUILD_MODE_STR=""
+        [ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}"
+
+        if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
+          # We upload release versions in the default folder.
+          PKG_DIR="${TARGET_PLATFORM}"
+        else
+          PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
+        fi
+
+        echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV
+        echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
+        echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled  == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
+        echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
+        echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+        echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
+        echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
+        echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml
@@ -0,0 +1,35 @@
+name: Build and test
+
+concurrency:
+  group: ${{ startsWith(github.ref_name, 'main') && format('unique-{0}', github.run_id) || format('ci-build-and-test-on-{0}-from-{1}', github.event_name, github.ref_name) }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+      - "main"
+
+jobs:
+  build-and-test:
+    name: Build and test (${{ matrix.host-platform }}, ${{ matrix.target-device }}, ${{ matrix.build-mode }})
+    strategy:
+      fail-fast: false
+      matrix:
+        host-platform:
+          - linux-x64
+        target-device:
+          - gpu
+        build-mode:
+          - release
+        upload-enabled:
+          - false
+    uses:
+      ./.github/workflows/gh-build-and-test.yml
+    with:
+      host-platform: ${{ matrix.host-platform }}
+      target-device: ${{ matrix.target-device }}
+      build-mode: ${{ matrix.build-mode }}
+      build-type: ci
+      upload-enabled: ${{ matrix.upload-enabled }}
+    secrets: inherit
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
@@ -0,0 +1,34 @@
+on:
+  workflow_call:
+    inputs:
+      host-platform:
+        type: string
+        required: true
+      target-device:
+        type: string
+        required: true
+      build-mode:
+        type: string
+        required: true
+      build-type:
+        type: string
+        required: true
+      upload-enabled:
+        type: boolean
+        required: true
+jobs:
+  build:
+    if: ${{ github.repository_owner == 'nvidia' }}
+    uses:
+      ./.github/workflows/gh-build.yml
+    with:
+      client-repo: ${{ github.event.repository.name }}
+      target-device: ${{ inputs.target-device }}
+      runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu16') || (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu16') || (inputs.host-platform == 'mac' && 'macos-latest') }}
+      build-type: ${{ inputs.build-type }}
+      use-container: ${{ inputs.host-platform == 'linux-x64' || inputs.host-platform == 'linux-aarch64'}}
+      host-platform: ${{ inputs.host-platform }}
+      dependencies-file: ""
+      build-mode: ${{ inputs.build-mode }}
+      upload-enabled: ${{ inputs.upload-enabled }}
+    secrets: inherit
diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
@@ -0,0 +1,95 @@
+name: Build
+
+on:
+  workflow_call:
+    inputs:
+      client-repo:
+        required: true
+        type: string
+      target-device:
+        required: true
+        type: string
+      runs-on:
+        required: true
+        type: string
+      build-type:
+        required: true
+        type: string
+        description: One of ci / release
+      use-container:
+        required: true
+        type: boolean
+      host-platform:
+        required: true
+        type: string
+      dependencies-file:
+        required: true
+        type: string
+        description: path to versions.json relative to the target repo dir
+      build-mode:
+        required: true
+        type: string
+      upload-enabled:
+        required: true
+        type: boolean
+      python-version:
+        required: false
+        type: string
+
+jobs:
+  build:
+    name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, CMake build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}", Use container=${{ inputs.use-container }} )
+
+    permissions:
+      id-token: write # This is required for configure-aws-credentials
+      contents: read  # This is required for actions/checkout
+
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout ${{ inputs.client-repo }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          client-repo: ${{ inputs.client-repo }}
+          build-type: ${{ inputs.build-type }}
+          target-device: "${{ inputs.target-device }}"
+          host-platform: ${{ inputs.host-platform }}
+          build-mode: ${{ inputs.build-mode }}
+          upload-enabled: ${{ inputs.upload-enabled }}
+          python-version: ${{ inputs.python-version }}
+
+      - name: Render templates
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          pip -q install jinja2
+
+          DEPENDENCIES_FILE=""
+
+          if [ -z "${{ inputs.dependencies-file }}" ]; then
+            DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json"
+          else
+            DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}"
+          fi
+
+          ${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml
+
+      - name: Dump templates
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          echo ${REPO_DIR}/.github/actions/build/action.yml
+          cat ${REPO_DIR}/.github/actions/build/action.yml
+
+      - name: Call build action
+        uses: ./.github/actions/build
+        with:
+          build-type: ${{ inputs.build-type }}
+          target-device: "${{ inputs.target-device }}"
+          host-platform: ${{ inputs.host-platform }}
+          use-container: ${{ inputs.use-container }}
+          docker-image: "condaforge/miniforge3:latest"
+          upload-enabled: ${{ inputs.upload-enabled }}
diff --git a/.gitignore b/.gitignore
@@ -168,3 +168,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# Dont ignore
+!.github/actions/build/