Bump setuptools from 75.4.0 to 75.6.0 in /shared_aws/py_package #2283
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: PR validation | |
on: | |
pull_request: | |
types: [synchronize, opened, reopened, edited, labeled, unlabeled] | |
branches: | |
- main | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
cancel-in-progress: true | |
jobs: | |
container-build-context: | |
name: define container image build context variables | |
runs-on: ubuntu-22.04 | |
outputs: | |
tagname: ${{ steps.store_tagname.outputs.tagname }} | |
steps: | |
- name: Store image tag in env | |
id: store_tagname | |
shell: bash | |
run: | | |
echo "tagname=PR-${{ github.event.pull_request.number }}-validation" >> $GITHUB_OUTPUT | |
pipeline-seq-retrieval-update-dependency-lock-files: | |
name: pipeline/seq_retrieval update and upload dependency lock file | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./pipeline/seq_retrieval/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/seq_retrieval/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Upload updated (main) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_deps_lock | |
path: pipeline/seq_retrieval/requirements.txt | |
if-no-files-found: error | |
- name: Upload updated (test) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_tests_deps_lock | |
path: pipeline/seq_retrieval/tests/requirements.txt | |
if-no-files-found: error | |
api-update-dependency-lock-files: | |
name: api update and upload dependency lock file | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./api/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
api/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Upload updated (main) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api/requirements.txt | |
if-no-files-found: error | |
- name: Upload updated (test) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests/requirements.txt | |
if-no-files-found: error | |
webui-update-dependency-lock-files: | |
name: webui update and upload dependency lock file | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./webui/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
webui/ | |
- uses: actions/setup-node@v4 | |
with: | |
node-version-file: "webui/.nvmrc" | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-lock | |
- name: Upload updated lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: webui_deps_lock | |
path: webui/package-lock.json | |
if-no-files-found: error | |
shared-aws-py-package-update-dependency-lock-files: | |
name: shared_aws py-package update and upload dependency lock file | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./shared_aws/py_package/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
shared_aws/py_package/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Upload updated (main) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: shared_aws_py_package_deps_lock | |
path: shared_aws/py_package/requirements.txt | |
if-no-files-found: error | |
- name: Upload updated (test) lock file as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: shared_aws_py_package_test_deps_lock | |
path: shared_aws/py_package/tests/requirements.txt | |
if-no-files-found: error | |
shared-aws-py-package-code-checks: | |
name: shared_aws py-package code checks | |
needs: | |
- shared-aws-py-package-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./shared_aws/py_package/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
shared_aws/py_package/ | |
- name: Download updated shared_aws py-package prod dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_deps_lock | |
path: shared_aws/py_package | |
- name: Download updated shared_aws py-package test dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_test_deps_lock | |
path: shared_aws/py_package/tests | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Unit tests | |
run: make run-unit-tests | |
shared-aws-py-package-build: | |
name: shared_aws python package build | |
needs: | |
- shared-aws-py-package-code-checks | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./shared_aws/py_package/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
shared_aws/py_package/ | |
- name: Download updated shared_aws py-package prod dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_deps_lock | |
path: shared_aws/py_package | |
- name: Download updated shared_aws py-package test dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_test_deps_lock | |
path: shared_aws/py_package/tests | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Build package | |
run: | | |
make clean build | |
- name: Upload package as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: shared_aws/py_package/dist/pavi_shared_aws-0.0.0-py3-none-any.whl | |
shared-aws-infra-update-dependency-lock-files: | |
name: shared_aws aws_infra update and upload dependency lock file | |
needs: | |
- shared-aws-py-package-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./shared_aws/aws_infra/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
shared_aws/aws_infra/ | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Bundle all deps lock files | |
run: | | |
tar -czv -f shared-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json | |
- name: Upload deps-lock-files bundle as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: shared_aws_infra_deps_locks_bundle | |
path: shared_aws/aws_infra/shared-aws-infra-deps-locks.tar.gz | |
if-no-files-found: error | |
shared-aws-infra-code-checks: | |
name: shared_aws infra code checks | |
needs: | |
- shared-aws-infra-update-dependency-lock-files | |
- shared-aws-py-package-build | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./shared_aws/aws_infra/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
shared_aws/aws_infra/ | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Download updated shared_aws/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_infra_deps_locks_bundle | |
path: shared_aws/aws_infra | |
- name: Unpack the bundle | |
run: | | |
tar -xzv -f shared-aws-infra-deps-locks.tar.gz | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Unit tests | |
run: make run-unit-tests | |
- name: Setup node.js (CDK CLI requirement) | |
uses: actions/setup-node@v4 | |
with: | |
node-version-file: "shared_aws/aws_infra/.nvmrc" | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test | |
aws-region: us-east-1 | |
- name: Validate production CDK stack code | |
run: make validate-stack | |
pipeline-aws-infra-update-dependency-lock-files: | |
name: pipeline/aws_infra update and upload dependency lock file | |
needs: | |
- shared-aws-py-package-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./pipeline/aws_infra/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/aws_infra/ | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Bundle all deps lock files | |
run: | | |
tar -czv -f pipeline-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json | |
- name: Upload deps-lock-files bundle as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: pipeline_aws_infra_deps_locks_bundle | |
path: pipeline/aws_infra/pipeline-aws-infra-deps-locks.tar.gz | |
if-no-files-found: error | |
api-aws-infra-update-dependency-lock-files: | |
name: api/aws_infra update and upload dependency lock file | |
needs: | |
- shared-aws-py-package-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./api/aws_infra/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
api/aws_infra/ | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Bundle all deps lock files | |
run: | | |
tar -czv -f api-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json | |
- name: Upload deps-lock-files bundle as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: api_aws_infra_deps_locks_bundle | |
path: api/aws_infra/api-aws-infra-deps-locks.tar.gz | |
if-no-files-found: error | |
webui-aws-infra-update-dependency-lock-files: | |
name: webui/aws_infra update and upload dependency lock file | |
needs: | |
- shared-aws-py-package-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./webui/aws_infra/ | |
steps: | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
webui/aws_infra/ | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Update lock files | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
shell: bash | |
run: | | |
make update-deps-locks-all | |
- name: Bundle all deps lock files | |
run: | | |
tar -czv -f webui-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json | |
- name: Upload deps-lock-files bundle as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: webui_aws_infra_deps_locks_bundle | |
path: webui/aws_infra/webui-aws-infra-deps-locks.tar.gz | |
if-no-files-found: error | |
pipeline-seq-retrieval-container-image-build: | |
name: pipeline/seq_retrieval container-image build | |
needs: | |
- container-build-context | |
- pipeline-seq-retrieval-update-dependency-lock-files | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
pipeline/seq_retrieval/ | |
- name: Download updated seq_retrieval (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_deps_lock | |
path: pipeline/seq_retrieval | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
# This step will configure environment variables to be used by all steps | |
# involving AWS interaction further down | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-seq-retrieval-image-upload | |
aws-region: us-east-1 | |
- name: Amazon ECR login | |
id: login-ecr | |
uses: aws-actions/amazon-ecr-login@v2 | |
- name: Build container image | |
uses: docker/build-push-action@v6 | |
env: | |
DOCKER_BUILD_SUMMARY: false | |
with: | |
context: ./pipeline/seq_retrieval/ | |
push: false | |
tags: agr_pavi/pipeline_seq_retrieval:latest | |
outputs: type=docker,dest=/tmp/pavi_seq_retrieval_docker_image.tar | |
- name: Upload image as artifact (share between jobs) | |
uses: actions/upload-artifact@v4 | |
with: | |
name: seq_retrieval_image | |
path: /tmp/pavi_seq_retrieval_docker_image.tar | |
- name: Load, tag and push image to registry | |
run: | | |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar | |
docker tag agr_pavi/pipeline_seq_retrieval:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}} | |
docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}} | |
pipeline-seq-retrieval-code-checks: | |
name: pipeline/seq_retrieval code checks | |
needs: | |
- pipeline-seq-retrieval-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./pipeline/seq_retrieval/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/seq_retrieval/ | |
- name: Download updated seq_retrieval (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_deps_lock | |
path: pipeline/seq_retrieval | |
- name: Download updated seq_retrieval (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_tests_deps_lock | |
path: pipeline/seq_retrieval/tests | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Run unit tests | |
run: | | |
make run-unit-tests | |
pipeline-alignment-container-image-build: | |
name: pipeline/alignment container-image build | |
needs: | |
- container-build-context | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
pipeline/alignment/ | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
# This step will configure environment variables to be used by all steps | |
# involving AWS interaction further down | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-alignment-image-upload | |
aws-region: us-east-1 | |
- name: Amazon ECR login | |
id: login-ecr | |
uses: aws-actions/amazon-ecr-login@v2 | |
- name: Build container image | |
uses: docker/build-push-action@v6 | |
env: | |
DOCKER_BUILD_SUMMARY: false | |
with: | |
context: ./pipeline/alignment/ | |
push: false | |
tags: agr_pavi/pipeline_alignment:latest | |
outputs: type=docker,dest=/tmp/pavi_alignment_docker_image.tar | |
- name: Upload image as artifact (share between jobs) | |
uses: actions/upload-artifact@v4 | |
with: | |
name: alignment_image | |
path: /tmp/pavi_alignment_docker_image.tar | |
- name: Load, tag and push image to registry | |
run: | | |
docker load --input /tmp/pavi_alignment_docker_image.tar | |
docker tag agr_pavi/pipeline_alignment:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}} | |
docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}} | |
pipeline-alignment-unit-testing: | |
name: pipeline/alignment unit testing | |
needs: | |
- pipeline-alignment-container-image-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./pipeline/alignment/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/alignment/ | |
- name: Download alignment image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: alignment_image | |
path: /tmp | |
- name: Load alignment Docker image | |
run: | | |
docker load --input /tmp/pavi_alignment_docker_image.tar | |
- name: Run unit test | |
run: | | |
make run-unit-tests | |
pipeline-workflow-integration-testing: | |
name: pipeline/workflow integration testing | |
needs: | |
- container-build-context | |
- pipeline-seq-retrieval-container-image-build | |
- pipeline-alignment-container-image-build | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./pipeline/workflow/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/workflow/ | |
tests/resources/ | |
- name: Download seq_retrieval image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: seq_retrieval_image | |
path: /tmp | |
- name: Download alignment image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: alignment_image | |
path: /tmp | |
- name: Load seq_retrieval Docker image | |
run: | | |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar | |
- name: Load alignment Docker image | |
run: | | |
docker load --input /tmp/pavi_alignment_docker_image.tar | |
- name: Run local integration test | |
run: | | |
make run-integration-test-local | |
# This step will configure environment variables to be used by all steps | |
# involving AWS interaction further down | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-workflow-aws-test | |
aws-region: us-east-1 | |
- name: Run AWS integration test | |
run: | | |
make run-integration-test-aws TAG_NAME=${{needs.container-build-context.outputs.tagname}} | |
pipeline-aws-infra-code-checks: | |
name: pipeline/aws_infra code checks | |
needs: | |
- pipeline-aws-infra-update-dependency-lock-files | |
- shared-aws-py-package-build | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./pipeline/aws_infra | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
pipeline/aws_infra/ | |
- name: Download updated pipeline/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_aws_infra_deps_locks_bundle | |
path: pipeline/aws_infra | |
- name: Unpack the bundle | |
working-directory: pipeline/aws_infra/ | |
run: | | |
tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Unit tests | |
run: make run-unit-tests | |
- name: Setup node.js (CDK CLI requirement) | |
uses: actions/setup-node@v4 | |
with: | |
node-version-file: "pipeline/aws_infra/.nvmrc" | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test | |
aws-region: us-east-1 | |
- name: Validate production CDK stack code | |
run: make validate-stack | |
api-code-checks: | |
name: API code checks | |
needs: | |
- api-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./api/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
api/ | |
- name: Download updated api (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api | |
- name: Download updated api (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
api-container-image-build: | |
name: API container-image build | |
needs: | |
- api-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Download updated api (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api | |
- name: Download updated api (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Build container image | |
uses: docker/build-push-action@v6 | |
env: | |
DOCKER_BUILD_SUMMARY: false | |
with: | |
context: ./ | |
file: api/Dockerfile | |
push: false | |
tags: agr_pavi/api:latest | |
outputs: type=docker,dest=/tmp/pavi_api_docker_image.tar | |
- name: Upload image as artifact (share between jobs) | |
uses: actions/upload-artifact@v4 | |
with: | |
name: api_image | |
path: /tmp/pavi_api_docker_image.tar | |
api-unit-integration-testing: | |
name: API unit and integration testing | |
needs: | |
- api-update-dependency-lock-files | |
- pipeline-seq-retrieval-container-image-build | |
- pipeline-alignment-container-image-build | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./api/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Download updated api (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api | |
- name: Download updated api (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests | |
- name: Download seq_retrieval image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: seq_retrieval_image | |
path: /tmp | |
- name: Download alignment image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: alignment_image | |
path: /tmp | |
- name: Load seq_retrieval Docker image | |
run: | | |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar | |
- name: Load alignment Docker image | |
run: | | |
docker load --input /tmp/pavi_alignment_docker_image.tar | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
- name: Run unit and integration tests | |
run: | | |
make run-tests | |
api-container-integration-testing: | |
name: API container integration testing (AWS batch execution) | |
needs: | |
- container-build-context | |
- pipeline-seq-retrieval-container-image-build | |
- pipeline-alignment-container-image-build | |
- api-container-image-build | |
- api-update-dependency-lock-files | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./api/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
api/ | |
tests/resources/ | |
- name: Download updated api (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api | |
- name: Download updated api (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests | |
- name: Download API image artifact (from previous job) | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_image | |
path: /tmp | |
- name: Load API Docker image | |
run: | | |
docker load --input /tmp/pavi_api_docker_image.tar | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.12' | |
# This step will configure environment variables to be used by all steps | |
# involving AWS interaction further down | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test | |
aws-region: us-east-1 | |
- name: Run container integration test (AWS execution) | |
run: | | |
make run-integration-test-container TAG_NAME=${{needs.container-build-context.outputs.tagname}} | |
api-aws-infra-code-checks: | |
name: api/aws_infra code checks | |
needs: | |
- api-aws-infra-update-dependency-lock-files | |
- shared-aws-py-package-build | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./api/aws_infra | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
api/aws_infra/ | |
- name: Download updated api/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_aws_infra_deps_locks_bundle | |
path: api/aws_infra | |
- name: Unpack the bundle | |
working-directory: api/aws_infra/ | |
run: | | |
tar -xzv -f api-aws-infra-deps-locks.tar.gz | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Setup node.js (CDK CLI requirement) | |
uses: actions/setup-node@v4 | |
with: | |
node-version-file: "api/aws_infra/.nvmrc" | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test | |
aws-region: us-east-1 | |
- name: Validate production CDK stack code | |
run: make validate-all | |
webui-aws-infra-code-checks: | |
name: webui/aws_infra code checks | |
needs: | |
- webui-aws-infra-update-dependency-lock-files | |
- shared-aws-py-package-build | |
permissions: | |
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
working-directory: ./webui/aws_infra | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
webui/aws_infra/ | |
- name: Download updated webui/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_aws_infra_deps_locks_bundle | |
path: webui/aws_infra | |
- name: Unpack the bundle | |
working-directory: webui/aws_infra/ | |
run: | | |
tar -xzv -f webui-aws-infra-deps-locks.tar.gz | |
- name: Download shared AWS package | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package | |
path: /tmp/ | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Setup node.js (CDK CLI requirement) | |
uses: actions/setup-node@v4 | |
with: | |
node-version-file: "webui/aws_infra/.nvmrc" | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test | |
aws-region: us-east-1 | |
- name: Validate production CDK stack code | |
run: make validate-all | |
webui-code-checks: | |
name: webUI code checks | |
needs: | |
- webui-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
defaults: | |
run: | |
shell: bash | |
working-directory: ./webui/ | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
sparse-checkout: | | |
Makefile | |
webui/ | |
- name: Download updated webui dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_deps_lock | |
path: webui | |
- uses: actions/setup-node@v4 | |
with: | |
node-version-file: "webui/.nvmrc" | |
- name: Code typing test | |
run: | | |
make run-type-checks | |
- name: Code style test | |
run: | | |
make run-style-checks | |
- name: Unit tests | |
run: make run-unit-tests | |
webui-container-image-build: | |
name: webUI container-image build | |
needs: | |
- webui-update-dependency-lock-files | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Download updated webui dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_deps_lock | |
path: webui | |
- name: Report node version stored in .nvmrc | |
id: nvmrc-node-version | |
run: | | |
{ | |
echo 'content<<EOF' | |
cat webui/.nvmrc | |
echo EOF | |
} >> "$GITHUB_OUTPUT" | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Build container image | |
uses: docker/build-push-action@v6 | |
env: | |
DOCKER_BUILD_SUMMARY: false | |
with: | |
context: ./webui/ | |
file: webui/Dockerfile | |
push: false | |
build-args: NODE_VERSION=${{steps.nvmrc-node-version.outputs.content}} | |
tags: agr_pavi/webui:latest | |
outputs: type=docker,dest=/tmp/pavi_webui_docker_image.tar | |
- name: Upload image as artifact (share between jobs) | |
uses: actions/upload-artifact@v4 | |
with: | |
name: webui_image | |
path: /tmp/pavi_webui_docker_image.tar | |
e2e-testing: | |
name: end-to-end testing | |
needs: | |
- container-build-context | |
- webui-update-dependency-lock-files | |
- webui-container-image-build | |
- api-container-image-build | |
- pipeline-alignment-container-image-build | |
- pipeline-seq-retrieval-container-image-build | |
permissions: | |
id-token: write # Required for OIDC authentication to AWS (to perform AWS actions / access AGR AWS resources) | |
runs-on: ubuntu-22.04 | |
steps: | |
# Configure environment variables used for AWS interaction, | |
# required for API container interactions with pipeline resources | |
- name: AWS credentials configuration | |
uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }} | |
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-e2e-testing | |
aws-region: us-east-1 | |
- name: Check out repository code | |
uses: actions/checkout@v4 | |
- name: Download API image artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_image | |
path: /tmp | |
- name: Load API Docker image | |
run: | | |
docker load --input /tmp/pavi_api_docker_image.tar | |
- name: Run local API container to run E2E tests on | |
working-directory: api/ | |
run: | | |
API_PIPELINE_IMAGE_TAG=${{needs.container-build-context.outputs.tagname}} make run-container-dev | |
- name: Download webUI image artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_image | |
path: /tmp | |
- name: Load webUI Docker image | |
run: | | |
docker load --input /tmp/pavi_webui_docker_image.tar | |
- name: Run local webUI container to run E2E tests on | |
working-directory: webui/ | |
run: | | |
PAVI_API_PORT=8080 make run-container-dev | |
- name: Download updated webui dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_deps_lock | |
path: webui | |
- name: setup webui-compatible node.js version | |
uses: actions/setup-node@v4 | |
with: | |
node-version-file: "webui/.nvmrc" | |
- name: Run cypress E2E tests | |
uses: cypress-io/github-action@v6 | |
with: | |
working-directory: webui/ | |
wait-on: 'http://localhost:8080/api/health, http://localhost:3000/health' | |
env: 'API_BASE_URL=http://localhost:8080' | |
- name: Report API container logs on E2E test failure | |
if: failure() | |
run: | | |
docker logs agr.pavi.dev-local.api.server | |
- name: Report webUI container logs on E2E test failure | |
if: failure() | |
run: | | |
docker logs agr.pavi.dev-local.webui.server | |
- name: Cleanup webUI server (running container) | |
if: success() || failure() | |
working-directory: webui/ | |
run: | | |
make stop-container-dev | |
- name: Cleanup API server (running container) | |
if: success() || failure() | |
working-directory: api/ | |
run: | | |
make stop-container-dev | |
stage-deps-lock-updates: | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
runs-on: ubuntu-22.04 | |
permissions: | |
pull-requests: write | |
# Always stage updated lock files, even on failure, | |
# to highlight dependency changes which could be the cause of the failure. | |
needs: | |
- shared-aws-py-package-update-dependency-lock-files | |
- shared-aws-infra-update-dependency-lock-files | |
- api-aws-infra-update-dependency-lock-files | |
- pipeline-aws-infra-update-dependency-lock-files | |
- webui-aws-infra-update-dependency-lock-files | |
- api-update-dependency-lock-files | |
- pipeline-seq-retrieval-update-dependency-lock-files | |
- webui-update-dependency-lock-files | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: ${{ github.head_ref }} | |
fetch-depth: 0 | |
- name: Download updated seq_retrieval (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_deps_lock | |
path: pipeline/seq_retrieval | |
- name: Download updated seq_retrieval (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_seq_retrieval_tests_deps_lock | |
path: pipeline/seq_retrieval/tests | |
- name: Download updated api (main) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_deps_lock | |
path: api | |
- name: Download updated api (test) dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_tests_deps_lock | |
path: api/tests | |
- name: Download updated pipeline/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: pipeline_aws_infra_deps_locks_bundle | |
path: pipeline/aws_infra | |
- name: Unpack the bundle | |
working-directory: pipeline/aws_infra/ | |
run: | | |
tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz | |
- name: Download updated shared_aws py-package prod dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_deps_lock | |
path: shared_aws/py_package | |
- name: Download updated shared_aws py-package test dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_py_package_test_deps_lock | |
path: shared_aws/py_package/tests | |
- name: Download updated shared_aws/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: shared_aws_infra_deps_locks_bundle | |
path: shared_aws/aws_infra | |
- name: Unpack the bundle | |
working-directory: shared_aws/aws_infra/ | |
run: | | |
tar -xzv -f shared-aws-infra-deps-locks.tar.gz | |
- name: Download updated api/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: api_aws_infra_deps_locks_bundle | |
path: api/aws_infra | |
- name: Unpack the bundle | |
working-directory: api/aws_infra/ | |
run: | | |
tar -xzv -f api-aws-infra-deps-locks.tar.gz | |
- name: Download updated webui/aws_infra dependencies lock files bundle | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_aws_infra_deps_locks_bundle | |
path: webui/aws_infra | |
- name: Unpack the bundle | |
working-directory: webui/aws_infra/ | |
run: | | |
tar -xzv -f webui-aws-infra-deps-locks.tar.gz | |
- name: Download updated webui dependencies lock file | |
uses: actions/download-artifact@v4 | |
with: | |
name: webui_deps_lock | |
path: webui | |
- name: Bundle all deps lock files | |
run: | | |
find . -regex '.*/\(requirements\.txt\|package-lock\.json\)' -print | \ | |
tar -czv -f deps-lock-files.tar.gz --files-from - | |
# Upload the deps-lock-files bundle as workflow artifact (to be downloaded on PR merge) | |
- name: Upload deps-lock-files bundle as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: deps_lock_files_bundle | |
path: deps-lock-files.tar.gz | |
# Share diff of changes in PR | |
- name: Generate list of files that were updated | |
id: diff-files-list | |
run: | | |
git diff --name-only -- > diff-files.txt | |
{ | |
echo 'content<<EOF' | |
cat diff-files.txt | |
echo EOF | |
} >> "$GITHUB_OUTPUT" | |
- name: Calculate diff for each individual file | |
id: diff-files-content | |
if: ${{ steps.diff-files-list.outputs.content }} | |
run: | | |
mkdir diff-files/ | |
cat diff-files.txt | xargs -I {} bash ci-scripts/create-git-diff-markdown.sh {} diff-files | |
{ | |
echo 'total_char_count<<EOF' | |
find ./diff-files/ -type f -exec cat {} \; | wc -m | |
echo EOF | |
} >> "$GITHUB_OUTPUT" | |
- name: Generate and store change message summary | |
if: ${{ steps.diff-files-list.outputs.content }} | |
run: | | |
{ | |
echo 'The validation workflow automatically applied the below dependency lock file updates.' | |
echo 'A bundle with all dependency lock files is available [here](${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}) as artifact called `deps_lock_files_bundle`.' | |
} > git-diff-message.md | |
# gh pr addComment API has a max 65536 character limit | |
- name: Report all diffs in a single comment | |
if: ${{ steps.diff-files-content.outputs.total_char_count < 60000 }} | |
run: | | |
{ | |
echo '' | |
cat diff-files.txt | xargs -I {} cat diff-files/{}.md | |
} >> git-diff-message.md | |
- name: Generate and store no-change message | |
if: ${{ !steps.diff-files-list.outputs.content }} | |
run: | | |
echo 'Automatic dependency lock file updates enabled, but no updates found. No (additional) dependency updates will be made when merging this PR.' \ | |
> git-diff-message.md | |
- name: Post PR comment with change message | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file git-diff-message.md | |
- name: Report each file diff in a separate comment | |
if: ${{ steps.diff-files-content.outputs.total_char_count >= 60000 }} | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
cat diff-files.txt | xargs -I {} gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file diff-files/{}.md | |
report-no-deps-lock-updates-label: | |
if: ${{ contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} | |
runs-on: ubuntu-22.04 | |
permissions: | |
pull-requests: write | |
steps: | |
- name: Post PR comment with notification no updates will be made | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
gh pr comment ${{ github.event.pull_request._links.html.href }} --body '`no-deps-lock-updates` label detected, so automatic dependency lock file updates are disabled. No (additional) dependency updates will be made when merging this PR.' |