Merge pull request #88 from alliance-genome/webUI-deployment-KANBAN-583

Refactored shared_aws_infra as built python package
alliance-genome · Jun 20, 2024 · 79eeb52 · 79eeb52
2 parents 1d1de92 + 38004ab
commit 79eeb52
Show file tree

Hide file tree

Showing 35 changed files with 313 additions and 161 deletions.
diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml
@@ -40,7 +40,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make pip-tools deps-lock-update test-deps-lock-update
+          make pip-tools update-deps-lock update-test-deps-lock
       - name: Upload updated (main) lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -74,7 +74,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make pip-tools deps-lock-update test-deps-lock-update
+          make pip-tools update-deps-lock update-test-deps-lock
       - name: Upload updated (main) lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -108,7 +108,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make deps-lock-update
+          make update-deps-lock
       - name: Upload updated lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -117,6 +117,8 @@ jobs:
           if-no-files-found: error
   pipeline-aws-infra-update-dependency-lock-files:
     name: pipeline/aws_infra update and upload dependency lock file
+    needs:
+      - shared-aws-infra-package-build
     runs-on: ubuntu-22.04
     defaults:
       run:
@@ -132,11 +134,16 @@ jobs:
           fetch-depth: 0
           sparse-checkout: |
             pipeline/aws_infra/
+      - name: Download shared AWS infra package
+        uses: actions/download-artifact@v4
+        with:
+          name: shared_aws_infra_package
+          path: /tmp/
       - name: Update lock files
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make pip-tools deps-lock-update test-deps-lock-update
+          make pip-tools update-deps-lock update-test-deps-lock
       - name: Upload updated (main) lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -151,6 +158,8 @@ jobs:
           if-no-files-found: error
   api-aws-infra-update-dependency-lock-files:
     name: api/aws_infra update and upload dependency lock file
+    needs:
+      - shared-aws-infra-package-build
     runs-on: ubuntu-22.04
     defaults:
       run:
@@ -166,11 +175,16 @@ jobs:
           fetch-depth: 0
           sparse-checkout: |
             api/aws_infra/
+      - name: Download shared AWS infra package
+        uses: actions/download-artifact@v4
+        with:
+          name: shared_aws_infra_package
+          path: /tmp/
       - name: Update lock files
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make pip-tools deps-lock-update test-deps-lock-update
+          make pip-tools update-deps-lock update-test-deps-lock
       - name: Upload updated (main) lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -204,7 +218,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
         shell: bash
         run: |
-          make pip-tools deps-lock-update test-deps-lock-update
+          make pip-tools update-deps-lock update-test-deps-lock
       - name: Upload updated (main) lock file as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -254,6 +268,43 @@ jobs:
         make run-style-checks
     - name: Unit tests
       run: make run-unit-tests
+  shared-aws-infra-package-build:
+    name: shared_aws_infra python package build
+    needs:
+      - shared-aws-infra-code-checks
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./shared_aws_infra
+    steps:
+    - name: Check out repository code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        sparse-checkout: |
+          shared_aws_infra/
+    - name: Download updated shared_aws_infra (main) dependencies lock file
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_deps_lock
+        path: shared_aws_infra
+    - name: Download updated shared_aws_infra (test) dependencies lock file
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_test_deps_lock
+        path: shared_aws_infra/tests
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Build package
+      run: |
+        make build
+    - name: Upload package as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: shared_aws_infra/dist/pavi_shared_aws_infra-0.0.0-py3-none-any.whl
   pipeline-seq-retrieval-container-image-build:
     name: pipeline/seq_retrieval container-image build
     needs:
@@ -468,7 +519,7 @@ jobs:
     name: pipeline/aws_infra code checks
     needs:
       - pipeline-aws-infra-update-dependency-lock-files
-      - shared-aws-infra-update-dependency-lock-files
+      - shared-aws-infra-package-build
     permissions:
       id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
@@ -482,7 +533,6 @@ jobs:
         fetch-depth: 0
         sparse-checkout: |
           pipeline/aws_infra/
-          shared_aws_infra/
     - name: Download updated pipeline/aws_infra (main) dependencies lock file
       uses: actions/download-artifact@v4
       with:
@@ -493,6 +543,11 @@ jobs:
       with:
         name: pipeline_aws_infra_tests_deps_lock
         path: pipeline/aws_infra/tests
+    - name: Download shared AWS infra package
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: /tmp/
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
@@ -696,6 +751,7 @@ jobs:
     name: api/aws_infra code checks
     needs:
       - api-aws-infra-update-dependency-lock-files
+      - shared-aws-infra-package-build
     permissions:
       id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
@@ -709,7 +765,6 @@ jobs:
         fetch-depth: 0
         sparse-checkout: |
           api/aws_infra/
-          shared_aws_infra/
     - name: Download updated api/aws_infra (main) dependencies lock file
       uses: actions/download-artifact@v4
       with:
@@ -720,6 +775,11 @@ jobs:
       with:
         name: api_aws_infra_tests_deps_lock
         path: api/aws_infra/tests
+    - name: Download shared AWS infra package
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: /tmp/
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
@@ -813,7 +873,7 @@ jobs:
       - pipeline-seq-retrieval-update-dependency-lock-files
       - pipeline-seq-retrieval-code-checks
       - shared-aws-infra-update-dependency-lock-files
-      - shared-aws-infra-code-checks
+      - shared-aws-infra-package-build
       - pipeline-aws-infra-update-dependency-lock-files
       - pipeline-aws-infra-code-checks
       - pipeline-workflow-integration-testing

diff --git a/.github/workflows/main-build-and-deploy.yml b/.github/workflows/main-build-and-deploy.yml
@@ -14,9 +14,36 @@ jobs:
       shell: bash
       run: |
         echo "PR merge detected and deployment wanted."
+  shared-aws-infra-package-build:
+    name: shared_aws_infra python package build
+    needs:
+      - on-merge-and-deploy
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./shared_aws_infra
+    steps:
+    - name: Check out repository code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        sparse-checkout: |
+          shared_aws_infra/
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Build package
+      run: |
+        make build
+    - name: Upload package as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: shared_aws_infra/dist/pavi_shared_aws_infra-0.0.0-py3-none-any.whl
   pipeline-deploy-aws-infra:
     name: Deploy/update AWS infrastructure for pipeline
-    needs: [on-merge-and-deploy]
+    needs: [shared-aws-infra-package-build]
     permissions:
       id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
@@ -36,6 +63,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: "3.12"
+    - name: Download shared AWS infra package
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: /tmp/
     - name: Install CDK stack dependencies
       run: pip install -r requirements.txt
     - name: AWS credentials configuration
@@ -50,7 +82,7 @@ jobs:
       run: make deploy ADD_CDK_ARGS="--require-approval never"
   api-deploy-image-repo:
     name: Deploy/update container image repository stack for API
-    needs: [on-merge-and-deploy]
+    needs: [shared-aws-infra-package-build]
     permissions:
       id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
@@ -70,6 +102,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: "3.12"
+    - name: Download shared AWS infra package
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: /tmp/
     - name: Install CDK stack dependencies
       run: pip install -r requirements.txt
     - name: AWS credentials configuration
@@ -193,6 +230,7 @@ jobs:
     name: Deploy application (version) for API
     needs:
       - on-merge-and-deploy
+      - shared-aws-infra-package-build
       - api-build-and-push-docker-image
       - pipeline-alignment-build-and-push-docker-image
       - pipeline-seq-retrieval-build-and-push-docker-image
@@ -221,6 +259,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: "3.12"
+    - name: Download shared AWS infra package
+      uses: actions/download-artifact@v4
+      with:
+        name: shared_aws_infra_package
+        path: /tmp/
     - name: Install CDK stack dependencies
       run: pip install -r requirements.txt
     - name: AWS credentials configuration

diff --git a/api/Makefile b/api/Makefile
@@ -31,7 +31,7 @@ protein-msa.nf:
 nextflow.config:
 	ln -s ../pipeline/workflow/nextflow.config nextflow.config
 
-run-server-dev: deps nextflow.sh protein-msa.nf nextflow.config
+run-server-dev: install-deps nextflow.sh protein-msa.nf nextflow.config
 	@export API_RESULTS_PATH_PREFIX="`pwd`/" && \
 	 export API_EXECUTION_ENV="local" && \
 	 fastapi dev src/main.py
@@ -58,31 +58,31 @@ requirements.txt:
 tests/requirements.txt:
 	pip-compile --generate-hashes --no-strip-extras --extra=test -o tests/requirements.txt
 
-deps-lock-update:
+update-deps-lock:
 	pip-compile --generate-hashes --no-strip-extras --upgrade -o requirements.txt
 
-test-deps-lock-update:
+update-test-deps-lock:
 	pip-compile --generate-hashes --no-strip-extras --extra=test --upgrade -o tests/requirements.txt
 
-deps: requirements.txt
+install-deps: requirements.txt
 	pip install -r requirements.txt
 
-test-deps: tests/requirements.txt
+install-test-deps: tests/requirements.txt
 	pip install -r tests/requirements.txt
 
-run-type-checks: test-deps
+run-type-checks: install-test-deps
 	mypy --install-types --non-interactive --warn-unused-config ./
 
-run-style-checks: test-deps
+run-style-checks: install-test-deps
 	flake8 ./
 
-run-tests: test-deps nextflow.sh protein-msa.nf nextflow.config
+run-tests: install-test-deps nextflow.sh protein-msa.nf nextflow.config
 	python -m pytest --cov --cov-fail-under=80
 
-run-tests-dev: test-deps nextflow.sh protein-msa.nf nextflow.config
+run-tests-dev: install-test-deps nextflow.sh protein-msa.nf nextflow.config
 	python -m pytest --cov --cov-fail-under=80 --cov-report html -v
 
-run-integration-test-container: test-deps
+run-integration-test-container: install-test-deps
 	export API_PIPELINE_IMAGE_TAG=${TAG_NAME} && \
 	  docker-compose -f docker-compose-dev.yml --env-file dev.env up -d agr.pavi.dev-local.api
 	sleep 30 # Allow container some startup time before attempting to connect

diff --git a/api/aws_infra/Makefile b/api/aws_infra/Makefile
@@ -43,28 +43,28 @@ requirements.txt:
 tests/requirements.txt:
 	pip-compile --generate-hashes --no-strip-extras --extra=test -o tests/requirements.txt
 
-deps-lock-update:
+update-deps-lock:
 	pip-compile --generate-hashes --no-strip-extras --upgrade -o requirements.txt
 
-test-deps-lock-update:
+update-test-deps-lock:
 	pip-compile --generate-hashes --no-strip-extras --extra=test --upgrade -o tests/requirements.txt
 
-deps: requirements.txt
+install-deps: requirements.txt
 	pip install -r requirements.txt
 
-test-deps: tests/requirements.txt
+install-test-deps: tests/requirements.txt
 	pip install -r tests/requirements.txt
 
-run-unit-tests: check-node test-deps
+run-unit-tests: check-node install-test-deps
 	python -m pytest
 
 run-unit-tests-dev: check-venv-active run-unit-tests
 	@:
 
-run-type-checks: test-deps
+run-type-checks: install-test-deps
 	mypy --install-types --non-interactive --warn-unused-config ./
 
-run-style-checks: test-deps
+run-style-checks: install-test-deps
 	flake8 ./
 
 validate-image-stack: check-node run-unit-tests