Skip to content
PR validation

Bump mypy from 1.11.2 to 1.13.0 in /webui/aws_infra #2131

Sign in to view logs

Bump mypy from 1.11.2 to 1.13.0 in /webui/aws_infra

Bump mypy from 1.11.2 to 1.13.0 in /webui/aws_infra #2131

Workflow file for this run

.github/workflows/PR-validation.yml at e8c361e
name: PR validation
on:
pull_request:
types: [synchronize, opened, reopened, edited, labeled, unlabeled]
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
container-build-context:
name: define container image build context variables
runs-on: ubuntu-22.04
outputs:
tagname: ${{ steps.store_tagname.outputs.tagname }}
steps:
- name: Store image tag in env
id: store_tagname
shell: bash
run: |
echo "tagname=PR-${{ github.event.pull_request.number }}-validation" >> $GITHUB_OUTPUT
pipeline-seq-retrieval-update-dependency-lock-files:
name: pipeline/seq_retrieval update and upload dependency lock file
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./pipeline/seq_retrieval/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/seq_retrieval/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Upload updated (main) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: pipeline_seq_retrieval_deps_lock
path: pipeline/seq_retrieval/requirements.txt
if-no-files-found: error
- name: Upload updated (test) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: pipeline_seq_retrieval_tests_deps_lock
path: pipeline/seq_retrieval/tests/requirements.txt
if-no-files-found: error
api-update-dependency-lock-files:
name: api update and upload dependency lock file
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./api/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
api/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Upload updated (main) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: api_deps_lock
path: api/requirements.txt
if-no-files-found: error
- name: Upload updated (test) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests/requirements.txt
if-no-files-found: error
webui-update-dependency-lock-files:
name: webui update and upload dependency lock file
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./webui/
steps:
- uses: actions/setup-node@v4
with:
node-version: "20"
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
webui/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-lock
- name: Upload updated lock file as artifact
uses: actions/upload-artifact@v4
with:
name: webui_deps_lock
path: webui/package-lock.json
if-no-files-found: error
shared-aws-py-package-update-dependency-lock-files:
name: shared_aws py-package update and upload dependency lock file
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./shared_aws/py_package/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
shared_aws/py_package/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Upload updated (main) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_py_package_deps_lock
path: shared_aws/py_package/requirements.txt
if-no-files-found: error
- name: Upload updated (test) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_py_package_test_deps_lock
path: shared_aws/py_package/tests/requirements.txt
if-no-files-found: error
shared-aws-py-package-code-checks:
name: shared_aws py-package code checks
needs:
- shared-aws-py-package-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws/py_package/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
shared_aws/py_package/
- name: Download updated shared_aws py-package prod dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_deps_lock
path: shared_aws/py_package
- name: Download updated shared_aws py-package test dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_test_deps_lock
path: shared_aws/py_package/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
shared-aws-py-package-build:
name: shared_aws python package build
needs:
- shared-aws-py-package-code-checks
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws/py_package/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
shared_aws/py_package/
- name: Download updated shared_aws py-package prod dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_deps_lock
path: shared_aws/py_package
- name: Download updated shared_aws py-package test dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_test_deps_lock
path: shared_aws/py_package/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build package
run: |
make clean build
- name: Upload package as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_py_package
path: shared_aws/py_package/dist/pavi_shared_aws-0.0.0-py3-none-any.whl
shared-aws-infra-update-dependency-lock-files:
name: shared_aws aws_infra update and upload dependency lock file
needs:
- shared-aws-py-package-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./shared_aws/aws_infra/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
shared_aws/aws_infra/
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Bundle all deps lock files
run: |
tar -czv -f shared-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
- name: Upload deps-lock-files bundle as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_deps_locks_bundle
path: shared_aws/aws_infra/shared-aws-infra-deps-locks.tar.gz
if-no-files-found: error
shared-aws-infra-code-checks:
name: shared_aws infra code checks
needs:
- shared-aws-infra-update-dependency-lock-files
- shared-aws-py-package-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws/aws_infra/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
shared_aws/aws_infra/
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Download updated shared_aws/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_locks_bundle
path: shared_aws/aws_infra
- name: Unpack the bundle
run: |
tar -xzv -f shared-aws-infra-deps-locks.tar.gz
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
- name: Setup node.js (CDK CLI requirement)
uses: actions/setup-node@v4
with:
node-version: "20"
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-stack
pipeline-aws-infra-update-dependency-lock-files:
name: pipeline/aws_infra update and upload dependency lock file
needs:
- shared-aws-py-package-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./pipeline/aws_infra/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/aws_infra/
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Bundle all deps lock files
run: |
tar -czv -f pipeline-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
- name: Upload deps-lock-files bundle as artifact
uses: actions/upload-artifact@v4
with:
name: pipeline_aws_infra_deps_locks_bundle
path: pipeline/aws_infra/pipeline-aws-infra-deps-locks.tar.gz
if-no-files-found: error
api-aws-infra-update-dependency-lock-files:
name: api/aws_infra update and upload dependency lock file
needs:
- shared-aws-py-package-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./api/aws_infra/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
api/aws_infra/
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Bundle all deps lock files
run: |
tar -czv -f api-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
- name: Upload deps-lock-files bundle as artifact
uses: actions/upload-artifact@v4
with:
name: api_aws_infra_deps_locks_bundle
path: api/aws_infra/api-aws-infra-deps-locks.tar.gz
if-no-files-found: error
webui-aws-infra-update-dependency-lock-files:
name: webui/aws_infra update and upload dependency lock file
needs:
- shared-aws-py-package-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./webui/aws_infra/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
webui/aws_infra/
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make update-deps-locks-all
- name: Bundle all deps lock files
run: |
tar -czv -f webui-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
- name: Upload deps-lock-files bundle as artifact
uses: actions/upload-artifact@v4
with:
name: webui_aws_infra_deps_locks_bundle
path: webui/aws_infra/webui-aws-infra-deps-locks.tar.gz
if-no-files-found: error
pipeline-seq-retrieval-container-image-build:
name: pipeline/seq_retrieval container-image build
needs:
- container-build-context
- pipeline-seq-retrieval-update-dependency-lock-files
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
pipeline/seq_retrieval/
- name: Download updated seq_retrieval (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: pipeline_seq_retrieval_deps_lock
path: pipeline/seq_retrieval
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# This step will configure environment variables to be used by all steps
# involving AWS interaction further down
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-seq-retrieval-image-upload
aws-region: us-east-1
- name: Amazon ECR login
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Build container image
uses: docker/build-push-action@v6
env:
DOCKER_BUILD_SUMMARY: false
with:
context: ./pipeline/seq_retrieval/
push: false
tags: agr_pavi/pipeline_seq_retrieval:latest
outputs: type=docker,dest=/tmp/pavi_seq_retrieval_docker_image.tar
- name: Upload image as artifact (share between jobs)
uses: actions/upload-artifact@v4
with:
name: seq_retrieval_image
path: /tmp/pavi_seq_retrieval_docker_image.tar
- name: Load, tag and push image to registry
run: |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
docker tag agr_pavi/pipeline_seq_retrieval:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
pipeline-seq-retrieval-code-checks:
name: pipeline/seq_retrieval code checks
needs:
- pipeline-seq-retrieval-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./pipeline/seq_retrieval/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/seq_retrieval/
- name: Download updated seq_retrieval (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: pipeline_seq_retrieval_deps_lock
path: pipeline/seq_retrieval
- name: Download updated seq_retrieval (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: pipeline_seq_retrieval_tests_deps_lock
path: pipeline/seq_retrieval/tests
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Run unit tests
run: |
make run-unit-tests
pipeline-alignment-container-image-build:
name: pipeline/alignment container-image build
needs:
- container-build-context
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
pipeline/alignment/
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# This step will configure environment variables to be used by all steps
# involving AWS interaction further down
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-alignment-image-upload
aws-region: us-east-1
- name: Amazon ECR login
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Build container image
uses: docker/build-push-action@v6
env:
DOCKER_BUILD_SUMMARY: false
with:
context: ./pipeline/alignment/
push: false
tags: agr_pavi/pipeline_alignment:latest
outputs: type=docker,dest=/tmp/pavi_alignment_docker_image.tar
- name: Upload image as artifact (share between jobs)
uses: actions/upload-artifact@v4
with:
name: alignment_image
path: /tmp/pavi_alignment_docker_image.tar
- name: Load, tag and push image to registry
run: |
docker load --input /tmp/pavi_alignment_docker_image.tar
docker tag agr_pavi/pipeline_alignment:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
pipeline-alignment-unit-testing:
name: pipeline/alignment unit testing
needs:
- pipeline-alignment-container-image-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./pipeline/alignment/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/alignment/
- name: Download alignment image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: alignment_image
path: /tmp
- name: Load alignment Docker image
run: |
docker load --input /tmp/pavi_alignment_docker_image.tar
- name: Run unit test
run: |
make run-unit-tests
pipeline-workflow-integration-testing:
name: pipeline/workflow integration testing
needs:
- container-build-context
- pipeline-seq-retrieval-container-image-build
- pipeline-alignment-container-image-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./pipeline/workflow/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/workflow/
tests/resources/
- name: Download seq_retrieval image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: seq_retrieval_image
path: /tmp
- name: Download alignment image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: alignment_image
path: /tmp
- name: Load seq_retrieval Docker image
run: |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
- name: Load alignment Docker image
run: |
docker load --input /tmp/pavi_alignment_docker_image.tar
- name: Run local integration test
run: |
make run-integration-test-local
# This step will configure environment variables to be used by all steps
# involving AWS interaction further down
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-workflow-aws-test
aws-region: us-east-1
- name: Run AWS integration test
run: |
make run-integration-test-aws TAG_NAME=${{needs.container-build-context.outputs.tagname}}
pipeline-aws-infra-code-checks:
name: pipeline/aws_infra code checks
needs:
- pipeline-aws-infra-update-dependency-lock-files
- shared-aws-py-package-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./pipeline/aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
pipeline/aws_infra/
- name: Download updated pipeline/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: pipeline_aws_infra_deps_locks_bundle
path: pipeline/aws_infra
- name: Unpack the bundle
working-directory: pipeline/aws_infra/
run: |
tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
- name: Setup node.js (CDK CLI requirement)
uses: actions/setup-node@v4
with:
node-version: "20"
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-stack
api-code-checks:
name: API code checks
needs:
- api-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./api/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
api/
- name: Download updated api (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_deps_lock
path: api
- name: Download updated api (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
api-container-image-build:
name: API container-image build
needs:
- api-update-dependency-lock-files
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download updated api (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_deps_lock
path: api
- name: Download updated api (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build container image
uses: docker/build-push-action@v6
env:
DOCKER_BUILD_SUMMARY: false
with:
context: ./
file: api/Dockerfile
push: false
tags: agr_pavi/api:latest
outputs: type=docker,dest=/tmp/pavi_api_docker_image.tar
- name: Upload image as artifact (share between jobs)
uses: actions/upload-artifact@v4
with:
name: api_image
path: /tmp/pavi_api_docker_image.tar
api-unit-integration-testing:
name: API unit and integration testing
needs:
- api-update-dependency-lock-files
- pipeline-seq-retrieval-container-image-build
- pipeline-alignment-container-image-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./api/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download updated api (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_deps_lock
path: api
- name: Download updated api (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests
- name: Download seq_retrieval image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: seq_retrieval_image
path: /tmp
- name: Download alignment image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: alignment_image
path: /tmp
- name: Load seq_retrieval Docker image
run: |
docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
- name: Load alignment Docker image
run: |
docker load --input /tmp/pavi_alignment_docker_image.tar
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Run unit and integration tests
run: |
make run-tests
api-container-integration-testing:
name: API container integration testing (AWS batch execution)
needs:
- container-build-context
- pipeline-seq-retrieval-container-image-build
- pipeline-alignment-container-image-build
- api-container-image-build
- api-update-dependency-lock-files
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./api/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
api/
tests/resources/
- name: Download updated api (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_deps_lock
path: api
- name: Download updated api (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests
- name: Download API image artifact (from previous job)
uses: actions/download-artifact@v4
with:
name: api_image
path: /tmp
- name: Load API Docker image
run: |
docker load --input /tmp/pavi_api_docker_image.tar
- uses: actions/setup-python@v5
with:
python-version: '3.12'
# This step will configure environment variables to be used by all steps
# involving AWS interaction further down
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test
aws-region: us-east-1
- name: Run container integration test (AWS execution)
run: |
make run-integration-test-container TAG_NAME=${{needs.container-build-context.outputs.tagname}}
api-aws-infra-code-checks:
name: api/aws_infra code checks
needs:
- api-aws-infra-update-dependency-lock-files
- shared-aws-py-package-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./api/aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
api/aws_infra/
- name: Download updated api/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: api_aws_infra_deps_locks_bundle
path: api/aws_infra
- name: Unpack the bundle
working-directory: api/aws_infra/
run: |
tar -xzv -f api-aws-infra-deps-locks.tar.gz
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Setup node.js (CDK CLI requirement)
uses: actions/setup-node@v4
with:
node-version: "20"
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-all
webui-aws-infra-code-checks:
name: webui/aws_infra code checks
needs:
- webui-aws-infra-update-dependency-lock-files
- shared-aws-py-package-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./webui/aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
webui/aws_infra/
- name: Download updated webui/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_deps_locks_bundle
path: webui/aws_infra
- name: Unpack the bundle
working-directory: webui/aws_infra/
run: |
tar -xzv -f webui-aws-infra-deps-locks.tar.gz
- name: Download shared AWS package
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package
path: /tmp/
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Setup node.js (CDK CLI requirement)
uses: actions/setup-node@v4
with:
node-version: "20"
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-all
webui-code-checks:
name: webUI code checks
needs:
- webui-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./webui/
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
Makefile
webui/
- name: Download updated webui dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_deps_lock
path: webui
- uses: actions/setup-node@v4
with:
node-version: "20"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
webui-container-image-build:
name: webUI container-image build
needs:
- webui-update-dependency-lock-files
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download updated webui dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_deps_lock
path: webui
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build container image
uses: docker/build-push-action@v6
env:
DOCKER_BUILD_SUMMARY: false
with:
context: ./webui/
file: webui/Dockerfile
push: false
tags: agr_pavi/webui:latest
outputs: type=docker,dest=/tmp/pavi_webui_docker_image.tar
- name: Upload image as artifact (share between jobs)
uses: actions/upload-artifact@v4
with:
name: webui_image
path: /tmp/pavi_webui_docker_image.tar
e2e-testing:
name: end-to-end testing
needs:
- container-build-context
- webui-update-dependency-lock-files
- webui-container-image-build
- api-container-image-build
- pipeline-alignment-container-image-build
- pipeline-seq-retrieval-container-image-build
permissions:
id-token: write # Required for OIDC authentication to AWS (to perform AWS actions / access AGR AWS resources)
runs-on: ubuntu-22.04
steps:
# Configure environment variables used for AWS interaction,
# required for API container interactions with pipeline resources
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-e2e-testing
aws-region: us-east-1
- name: Check out repository code
uses: actions/checkout@v4
- name: Download API image artifact
uses: actions/download-artifact@v4
with:
name: api_image
path: /tmp
- name: Load API Docker image
run: |
docker load --input /tmp/pavi_api_docker_image.tar
- name: Run local API container to run E2E tests on
working-directory: api/
run: |
API_PIPELINE_IMAGE_TAG=${{needs.container-build-context.outputs.tagname}} make run-container-dev
- name: Download webUI image artifact
uses: actions/download-artifact@v4
with:
name: webui_image
path: /tmp
- name: Load webUI Docker image
run: |
docker load --input /tmp/pavi_webui_docker_image.tar
- name: Run local webUI container to run E2E tests on
working-directory: webui/
run: |
PAVI_API_PORT=8080 make run-container-dev
- name: Download updated webui dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_deps_lock
path: webui
- name: setup webui-compatible node.js version
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Run cypress E2E tests
uses: cypress-io/github-action@v6
with:
working-directory: webui/
wait-on: 'http://localhost:8080/api/health, http://localhost:3000/health'
env: 'API_BASE_URL=http://localhost:8080'
- name: Report API container logs on E2E test failure
if: failure()
run: |
docker logs agr.pavi.dev-local.api.server
- name: Report webUI container logs on E2E test failure
if: failure()
run: |
docker logs agr.pavi.dev-local.webui.server
- name: Cleanup webUI server (running container)
if: success() || failure()
working-directory: webui/
run: |
make stop-container-dev
- name: Cleanup API server (running container)
if: success() || failure()
working-directory: api/
run: |
make stop-container-dev
stage-deps-lock-updates:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
runs-on: ubuntu-22.04
permissions:
pull-requests: write
# Always stage updated lock files, even on failure,
# to highlight dependency changes which could be the cause of the failure.
needs:
- shared-aws-py-package-update-dependency-lock-files
- shared-aws-infra-update-dependency-lock-files
- api-aws-infra-update-dependency-lock-files
- pipeline-aws-infra-update-dependency-lock-files
- webui-aws-infra-update-dependency-lock-files
- api-update-dependency-lock-files
- pipeline-seq-retrieval-update-dependency-lock-files
- webui-update-dependency-lock-files
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
fetch-depth: 0
- name: Download updated seq_retrieval (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: pipeline_seq_retrieval_deps_lock
path: pipeline/seq_retrieval
- name: Download updated seq_retrieval (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: pipeline_seq_retrieval_tests_deps_lock
path: pipeline/seq_retrieval/tests
- name: Download updated api (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_deps_lock
path: api
- name: Download updated api (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: api_tests_deps_lock
path: api/tests
- name: Download updated pipeline/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: pipeline_aws_infra_deps_locks_bundle
path: pipeline/aws_infra
- name: Unpack the bundle
working-directory: pipeline/aws_infra/
run: |
tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
- name: Download updated shared_aws py-package prod dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_deps_lock
path: shared_aws/py_package
- name: Download updated shared_aws py-package test dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_py_package_test_deps_lock
path: shared_aws/py_package/tests
- name: Download updated shared_aws/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_locks_bundle
path: shared_aws/aws_infra
- name: Unpack the bundle
working-directory: shared_aws/aws_infra/
run: |
tar -xzv -f shared-aws-infra-deps-locks.tar.gz
- name: Download updated api/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: api_aws_infra_deps_locks_bundle
path: api/aws_infra
- name: Unpack the bundle
working-directory: api/aws_infra/
run: |
tar -xzv -f api-aws-infra-deps-locks.tar.gz
- name: Download updated webui/aws_infra dependencies lock files bundle
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_deps_locks_bundle
path: webui/aws_infra
- name: Unpack the bundle
working-directory: webui/aws_infra/
run: |
tar -xzv -f webui-aws-infra-deps-locks.tar.gz
- name: Download updated webui dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_deps_lock
path: webui
- name: Bundle all deps lock files
run: |
find . -regex '.*/\(requirements\.txt\|package-lock\.json\)' -print | \
tar -czv -f deps-lock-files.tar.gz --files-from -
# Upload the deps-lock-files bundle as workflow artifact (to be downloaded on PR merge)
- name: Upload deps-lock-files bundle as artifact
uses: actions/upload-artifact@v4
with:
name: deps_lock_files_bundle
path: deps-lock-files.tar.gz
# Share diff of changes in PR
- name: Generate list of files that were updated
id: diff-files-list
run: |
git diff --name-only -- > diff-files.txt
{
echo 'content<<EOF'
cat diff-files.txt
echo EOF
} >> "$GITHUB_OUTPUT"
- name: Calculate diff for each individual file
id: diff-files-content
if: ${{ steps.diff-files-list.outputs.content }}
run: |
mkdir diff-files/
cat diff-files.txt | xargs -I {} bash ci-scripts/create-git-diff-markdown.sh {} diff-files
{
echo 'total_char_count<<EOF'
find ./diff-files/ -type f -exec cat {} \; | wc -m
echo EOF
} >> "$GITHUB_OUTPUT"
- name: Generate and store change message summary
if: ${{ steps.diff-files-list.outputs.content }}
run: |
{
echo 'The validation workflow automatically applied the below dependency lock file updates.'
echo 'A bundle with all dependency lock files is available [here](${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}) as artifact called `deps_lock_files_bundle`.'
} > git-diff-message.md
# gh pr addComment API has a max 65536 character limit
- name: Report all diffs in a single comment
if: ${{ steps.diff-files-content.outputs.total_char_count < 60000 }}
run: |
{
echo ''
cat diff-files.txt | xargs -I {} cat diff-files/{}.md
} >> git-diff-message.md
- name: Generate and store no-change message
if: ${{ !steps.diff-files-list.outputs.content }}
run: |
echo 'Automatic dependency lock file updates enabled, but no updates found. No (additional) dependency updates will be made when merging this PR.' \
> git-diff-message.md
- name: Post PR comment with change message
env:
GH_TOKEN: ${{ github.token }}
run: |
gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file git-diff-message.md
- name: Report each file diff in a separate comment
if: ${{ steps.diff-files-content.outputs.total_char_count >= 60000 }}
env:
GH_TOKEN: ${{ github.token }}
run: |
cat diff-files.txt | xargs -I {} gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file diff-files/{}.md
report-no-deps-lock-updates-label:
if: ${{ contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
runs-on: ubuntu-22.04
permissions:
pull-requests: write
steps:
- name: Post PR comment with notification no updates will be made
env:
GH_TOKEN: ${{ github.token }}
run: |
gh pr comment ${{ github.event.pull_request._links.html.href }} --body '`no-deps-lock-updates` label detected, so automatic dependency lock file updates are disabled. No (additional) dependency updates will be made when merging this PR.'