From e386afd71334ca64b718f3e42c14b85233356cd3 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Tue, 30 Apr 2024 14:35:16 +0100 Subject: [PATCH 1/2] Added Managed_by and Product AWS tags on PAVI-managed AWS resources. --- pipeline/aws_infra/cdk_classes/aws_batch.py | 15 ++++++++++++++- pipeline/aws_infra/cdk_classes/cdk_infra_stack.py | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pipeline/aws_infra/cdk_classes/aws_batch.py b/pipeline/aws_infra/cdk_classes/aws_batch.py index 62328373..9f04e0de 100644 --- a/pipeline/aws_infra/cdk_classes/aws_batch.py +++ b/pipeline/aws_infra/cdk_classes/aws_batch.py @@ -5,7 +5,8 @@ aws_iam as iam, RemovalPolicy, Stack, - aws_s3 as s3 + aws_s3 as s3, + Tags as cdk_tags ) from typing import Optional @@ -45,6 +46,8 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option removal_policy=RemovalPolicy.RETAIN, versioned=False ) + cdk_tags.of(self.nf_workdir_bucket).add("Product", "PAVI") + cdk_tags.of(self.nf_workdir_bucket).add("Managed_by", "PAVI") else: self.nf_workdir_bucket = s3.Bucket.from_bucket_name( scope=scope, id='pavi-pipeline-nf-workdir-bucket', @@ -67,6 +70,8 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option ) ] ) + cdk_tags.of(s3_workdir_bucket_policy_doc).add("Product", "PAVI") + cdk_tags.of(s3_workdir_bucket_policy_doc).add("Managed_by", "PAVI") instance_role = iam.Role(scope, 'pavi-pipeline-compute-environment-instance-role', description='Role granting permissions for Nextflow ECS execution', @@ -78,6 +83,8 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option inline_policies={ 's3-workdir-policy': s3_workdir_bucket_policy_doc }) + cdk_tags.of(instance_role).add("Product", "PAVI") + cdk_tags.of(instance_role).add("Managed_by", "PAVI") ce_name = 'pavi_pipeline_ecs' if env_suffix: @@ -93,6 +100,9 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option self.compute_environment.apply_removal_policy(RemovalPolicy.DESTROY) + cdk_tags.of(self.compute_environment).add("Product", "PAVI") + cdk_tags.of(self.compute_environment).add("Managed_by", "PAVI") + # Create the job queue jq_name = 'pavi_pipeline' if env_suffix: @@ -103,3 +113,6 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option ) self.job_queue.add_compute_environment(self.compute_environment, order=1) # type: ignore + + cdk_tags.of(self.job_queue).add("Product", "PAVI") + cdk_tags.of(self.job_queue).add("Managed_by", "PAVI") diff --git a/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py b/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py index 955e17da..77bcad28 100644 --- a/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py +++ b/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py @@ -1,6 +1,7 @@ from aws_cdk import ( Stack, - aws_ecr as ecr + aws_ecr as ecr, + Tags as cdk_tags ) from constructs import Construct @@ -39,6 +40,8 @@ def __init__(self, scope: Construct, construct_id: str, env_suffix: str = "", env_suffix=env_suffix) else: self.seq_retrieval_ecr_repo = ecr.Repository.from_repository_name(self, id='PAVI-pipeline-seq-retrieval-repo', repository_name=shared_seq_retrieval_image_repo) + cdk_tags.of(self.seq_retrieval_ecr_repo).add("Product", "PAVI") + cdk_tags.of(self.seq_retrieval_ecr_repo).add("Managed_by", "PAVI") # Import or create shared_alignment_image_repo if not shared_alignment_image_repo: @@ -46,5 +49,7 @@ def __init__(self, scope: Construct, construct_id: str, env_suffix: str = "", env_suffix=env_suffix) else: self.alignment_ecr_repo = ecr.Repository.from_repository_name(self, id='PAVI-pipeline-alignment-repo', repository_name=shared_alignment_image_repo) + cdk_tags.of(self.alignment_ecr_repo).add("Product", "PAVI") + cdk_tags.of(self.alignment_ecr_repo).add("Managed_by", "PAVI") self.execution_environment = PaviExecutionEnvironment(self, env_suffix=env_suffix, shared_work_dir_bucket=shared_work_dir_bucket) From 8f34eae45f0681d6c875bb49a693f7e9bdc494ab Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Tue, 30 Apr 2024 14:53:31 +0100 Subject: [PATCH 2/2] Name launched EC2 execution workers --- pipeline/aws_infra/cdk_classes/aws_batch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipeline/aws_infra/cdk_classes/aws_batch.py b/pipeline/aws_infra/cdk_classes/aws_batch.py index 9f04e0de..30555adb 100644 --- a/pipeline/aws_infra/cdk_classes/aws_batch.py +++ b/pipeline/aws_infra/cdk_classes/aws_batch.py @@ -103,6 +103,8 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option cdk_tags.of(self.compute_environment).add("Product", "PAVI") cdk_tags.of(self.compute_environment).add("Managed_by", "PAVI") + self.compute_environment.tags.set_tag('Name', 'PAVI pipeline execution worker', priority=None, apply_to_launched_instances=True) + # Create the job queue jq_name = 'pavi_pipeline' if env_suffix: