From 6877ed4aae1b87a485eebc63047fc8a2a7081497 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Mon, 28 Nov 2022 19:51:29 +0000 Subject: [PATCH 01/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From b941f8cdb37ab3e2e2c2c52059b41e2f013911ef Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:34:59 +0000 Subject: [PATCH 02/26] feat(Dependencies): Update peter-evans/create-pull-request action to v4.2.3 | datasource | package | from | to | | ----------- | ------------------------------- | ------ | ------ | | github-tags | peter-evans/create-pull-request | v4.2.0 | v4.2.3 | --- .github/workflows/blacked.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/blacked.yml b/.github/workflows/blacked.yml index 0890319b..21355075 100644 --- a/.github/workflows/blacked.yml +++ b/.github/workflows/blacked.yml @@ -17,7 +17,7 @@ jobs: - name: Create Pull Request if: steps.git-check.outputs.modified == 'true' id: cpr - uses: peter-evans/create-pull-request@v4.2.0 + uses: peter-evans/create-pull-request@v4.2.3 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: fix(Linting):blacked code From d52da0aeb91b7172a7c57fd1a5f4216625507f11 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:35:06 +0000 Subject: [PATCH 03/26] feat(Dependencies): Update dependency openstacksdk to v0.103.0 | datasource | package | from | to | | ---------- | ------------ | ------- | ------- | | pypi | openstacksdk | 0.102.0 | 0.103.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16488de..8cfb06a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ setuptools==65.5.1 thrift==0.16.0 python-keystoneclient==5.0.1 -openstacksdk ==0.102.0 +openstacksdk ==0.103.0 deprecated == 1.2.13 ansible==6.6.0 Click==8.1.3 From 3dd10da925ffcb0436822011f1182d16ff228627 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:35:11 +0000 Subject: [PATCH 04/26] feat(Dependencies): Update dependency setuptools to v65.6.3 | datasource | package | from | to | | ---------- | ---------- | ------ | ------ | | pypi | setuptools | 65.5.1 | 65.6.3 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16488de..ddd58ae9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==65.5.1 +setuptools==65.6.3 thrift==0.16.0 python-keystoneclient==5.0.1 openstacksdk ==0.102.0 From 9e6e34c50f527411efd63bca4b34a96131856180 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:35:23 +0000 Subject: [PATCH 05/26] feat(Dependencies): Update dependency flake8 to v6 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | flake8 | 5.0.4 | 6.0.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16488de..ade41758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ openstacksdk ==0.102.0 deprecated == 1.2.13 ansible==6.6.0 Click==8.1.3 -flake8==5.0.4 +flake8==6.0.0 paramiko==2.12.0 ruamel.yaml==0.17.21 pyvim==3.0.3 From 4c0c513dbd2769606980318916289a9205012695 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 28 Nov 2022 22:35:39 +0000 Subject: [PATCH 06/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 831243ac0e3b7eaf0d726b293a19ca5114532090 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 28 Nov 2022 22:35:40 +0000 Subject: [PATCH 07/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From a42c2cafdd65599f36c7df60ba98b866b6f1fd40 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 28 Nov 2022 22:35:44 +0000 Subject: [PATCH 08/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From b70793e815808f9c3100a52288709745dea7d708 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 28 Nov 2022 22:35:58 +0000 Subject: [PATCH 09/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 2713462cd0affa6ae33fc318192414e08cce02df Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 4 Dec 2022 13:27:49 +0000 Subject: [PATCH 10/26] feat(Dependencies): Update dependency redis to v4.4.0 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | redis | 4.3.4 | 4.4.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16488de..f9ce5c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ flake8==5.0.4 paramiko==2.12.0 ruamel.yaml==0.17.21 pyvim==3.0.3 -redis==4.3.4 +redis==4.4.0 requests==2.28.1 pyyaml==6.0 From 69cd7efd322c8153903c8a3d238cb783aba762b2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Sun, 4 Dec 2022 13:28:29 +0000 Subject: [PATCH 11/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 8728f2cc54077827bb5a1a26c2cb8b9e274ff931 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 16:50:52 +0000 Subject: [PATCH 12/26] feat(Dependencies): Update dependency oefenweb.rstudio_server to v2.7.2 | datasource | package | from | to | | ---------- | ----------------------- | ------ | ------ | | galaxy | oefenweb.rstudio_server | v2.7.1 | v2.7.2 | --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index f8c8c556..32558bb2 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,6 +1,6 @@ roles: - name: oefenweb.rstudio_server - version: v2.7.1 + version: v2.7.2 - name: oefenweb.latest_r version: v3.1.1 From f3bff504346ef8fa997433f3ebc561a0066e1b62 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 5 Dec 2022 16:51:38 +0000 Subject: [PATCH 13/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 288fd15d16d3f75917903b6338bca4d6f31a7f6c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 22:55:04 +0000 Subject: [PATCH 14/26] feat(Dependencies): Update dependency ansible to v7 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | ansible | 6.6.0 | 7.1.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16488de..ab467da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ thrift==0.16.0 python-keystoneclient==5.0.1 openstacksdk ==0.102.0 deprecated == 1.2.13 -ansible==6.6.0 +ansible==7.1.0 Click==8.1.3 flake8==5.0.4 paramiko==2.12.0 From e3af7b8d1192582a18ff093b1426dabd3efafe45 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Wed, 7 Dec 2022 22:55:56 +0000 Subject: [PATCH 15/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 06a3d4970bebc42cee46d326a911bcaa7fd41b9a Mon Sep 17 00:00:00 2001 From: dweinholz Date: Thu, 8 Dec 2022 10:27:36 +0100 Subject: [PATCH 16/26] feat(Image):pre-build resenvs does not need any ansible runs --- VirtualMachineService/VirtualMachineHandler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..c7c074c4 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -1058,17 +1058,22 @@ def start_server_without_playbook( volume_ids_path_new=volume_ids_path_new, volume_ids_path_attach=volume_ids_path_attach, ) + unlock_ubuntu_user_script = "\npasswd -u ubuntu\n" + unlock_ubuntu_user_script=encodeutils.safe_encode(unlock_ubuntu_user_script.encode("utf-8")) if additional_keys: if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( add_key_script + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + init_script + ) else: - init_script = self.create_add_keys_script(keys=additional_keys) + init_script = self.create_add_keys_script(keys=additional_keys) + encodeutils.safe_encode( + "\n".encode("utf-8")) + unlock_ubuntu_user_script server = self.conn.create_server( name=servername, From 3464f872a9f181257792800216fef7b92fd9e73f Mon Sep 17 00:00:00 2001 From: dweinholz Date: Thu, 8 Dec 2022 09:28:36 +0000 Subject: [PATCH 17/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 276 +++++++++--------- 1 file changed, 144 insertions(+), 132 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index c7c074c4..a40accac 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1059,21 +1059,25 @@ def start_server_without_playbook( volume_ids_path_attach=volume_ids_path_attach, ) unlock_ubuntu_user_script = "\npasswd -u ubuntu\n" - unlock_ubuntu_user_script=encodeutils.safe_encode(unlock_ubuntu_user_script.encode("utf-8")) + unlock_ubuntu_user_script = encodeutils.safe_encode( + unlock_ubuntu_user_script.encode("utf-8") + ) if additional_keys: if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script - + init_script - + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + + init_script ) else: - init_script = self.create_add_keys_script(keys=additional_keys) + encodeutils.safe_encode( - "\n".encode("utf-8")) + unlock_ubuntu_user_script + init_script = ( + self.create_add_keys_script(keys=additional_keys) + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + ) server = self.conn.create_server( name=servername, @@ -1102,17 +1106,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1166,7 +1170,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1212,16 +1216,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1297,7 +1301,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1348,8 +1352,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2107,19 +2111,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2132,9 +2136,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2426,9 +2428,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2437,7 +2439,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2588,13 +2590,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2609,15 +2619,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2667,7 +2677,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2800,7 +2812,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2844,7 +2856,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2890,17 +2902,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From 16cba6e9650a043f24e1a8971ac53e0a7fb74842 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:51:24 +0000 Subject: [PATCH 18/26] feat(Dependencies): Update docker.elastic.co/beats/filebeat Docker tag to v8.5.3 | datasource | package | from | to | | ---------- | -------------------------------- | ----- | ----- | | docker | docker.elastic.co/beats/filebeat | 8.5.1 | 8.5.3 | | docker | docker.elastic.co/beats/filebeat | 8.5.1 | 8.5.3 | --- docker-compose.bibigrid.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index f80dba17..ae19b433 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -40,7 +40,7 @@ services: # filebeat filebeat: - image: docker.elastic.co/beats/filebeat:8.5.1 + image: docker.elastic.co/beats/filebeat:8.5.3 env_file: - .env volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 27aa7279..05bd84e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: # filebeat filebeat: container_name: client_filebeat - image: docker.elastic.co/beats/filebeat:8.5.1 + image: docker.elastic.co/beats/filebeat:8.5.3 env_file: - .env volumes: From fc8020e06c9a0ac5ffdeab3d3802a7741b9ca8ae Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Thu, 8 Dec 2022 17:52:24 +0000 Subject: [PATCH 19/26] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 262 +++++++++--------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 2458618a..e0bde431 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -253,9 +253,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -398,10 +398,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") @@ -412,7 +412,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -453,7 +453,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -491,11 +491,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -517,11 +517,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -579,10 +579,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -768,7 +768,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -913,17 +913,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -964,11 +964,11 @@ def volume_ids( return {} def prepare_security_groups_new_server( - self, - resenv: List[str], - servername: str, - http: bool = False, - https: bool = False, + self, + resenv: List[str], + servername: str, + http: bool = False, + https: bool = False, ): custom_security_groups = [] @@ -1002,18 +1002,18 @@ def prepare_security_groups_new_server( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1062,9 +1062,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1097,17 +1097,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1161,7 +1161,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1207,16 +1207,16 @@ def create_resenv_security_group(self, resenv_template: str): return None def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1292,7 +1292,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1343,8 +1343,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2102,19 +2102,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2127,9 +2127,7 @@ def add_cluster_machine( LOG.info(f"Checking if [{version}] in [{image}]") if version in image: - LOG.info( - f"Version {version} in {image}!\Checking for image ..." - ) + LOG.info(f"Version {version} in {image}!\Checking for image ...") openstack_image = self.get_active_image_by_os_version( os_version=version, os_distro="ubuntu" ) @@ -2421,9 +2419,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2432,7 +2430,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec["name"] + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: @@ -2583,13 +2581,21 @@ def resume_server(self, openstack_id): return False def validate_gateway_security_group(self): - LOG.info(f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}") - gateway_security_id = self.conn.get_security_group(self.GATEWAY_SECURITY_GROUP_ID) + LOG.info( + f"Check if gateway security group exists {self.GATEWAY_SECURITY_GROUP_ID}" + ) + gateway_security_id = self.conn.get_security_group( + self.GATEWAY_SECURITY_GROUP_ID + ) if not gateway_security_id: - LOG.error(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!") + LOG.error( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} does not exist!" + ) sys.exit(1) else: - LOG.info(f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found") + LOG.info( + f"Gateway Security Group ID {self.GATEWAY_SECURITY_GROUP_ID} found" + ) def create_or_get_default_ssh_security_group(self): LOG.info("Get default SimpleVM SSH Security Group") @@ -2604,15 +2610,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2662,7 +2668,9 @@ def create_security_group( if udp: LOG.info( "Add udp rule port {} to security group {} ({})".format( - udp_port, name, new_security_group["id"], + udp_port, + name, + new_security_group["id"], ) ) @@ -2795,7 +2803,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2839,7 +2847,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2885,17 +2893,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port From b19bca0b8fcd1d1d6524f4ef892ba1bf1e500212 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 11:16:59 +0000 Subject: [PATCH 20/26] feat(Dependencies): Update dependency oefenweb.rstudio_server to v3 | datasource | package | from | to | | ---------- | ----------------------- | ------ | ------ | | galaxy | oefenweb.rstudio_server | v2.7.2 | v3.0.2 | --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 32558bb2..766924c7 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,6 +1,6 @@ roles: - name: oefenweb.rstudio_server - version: v2.7.2 + version: v3.0.2 - name: oefenweb.latest_r version: v3.1.1 From c744bf14747973f325c94cbb00a6b5a693fe0b63 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 11:21:37 +0000 Subject: [PATCH 21/26] feat(Dependencies): Update python Docker tag to v3.10.9 | datasource | package | from | to | | ---------- | ------- | ------ | ------ | | docker | python | 3.10.8 | 3.10.9 | --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6ce48ef..7bc18443 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.8-buster +FROM python:3.10.9-buster RUN apt-get update -y RUN apt-get install -y build-essential WORKDIR /code From 494313cf9d7299d820db7c007ee93344253dfafb Mon Sep 17 00:00:00 2001 From: dweinholz Date: Fri, 9 Dec 2022 17:23:20 +0100 Subject: [PATCH 22/26] feat/thrift_dockerfile --- thrift.dockerfile | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 thrift.dockerfile diff --git a/thrift.dockerfile b/thrift.dockerfile new file mode 100644 index 00000000..7e260a38 --- /dev/null +++ b/thrift.dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install -y \ + wget \ + build-essential \ + automake \ + libtool \ + flex \ + bison \ + pkg-config \ + libboost-all-dev \ + libevent-dev \ + libdouble-conversion-dev \ + libgoogle-glog-dev \ + libgflags-dev \ + liblz4-dev \ + liblzma-dev \ + libsnappy-dev \ + zlib1g-dev \ + binutils-dev \ + libjemalloc-dev \ + libssl-dev + +RUN wget http://archive.apache.org/dist/thrift/0.11.0/thrift-0.11.0.tar.gz && \ + tar xzf thrift-0.11.0.tar.gz && \ + rm thrift-0.11.0.tar.gz && \ + cd thrift-0.11.0 && \ + ./configure --without-python && \ + make && \ + make install + +COPY . /app + +WORKDIR /app + +# Set the default target language to Python +ENV TARGET_LANG python + +CMD thrift -r --gen $TARGET_LANG /app/$THRIFT_FILE From 9601dc00f66a74ce6c544aa0aefceebab12e9207 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Fri, 9 Dec 2022 17:34:13 +0100 Subject: [PATCH 23/26] Update thrift.dockerfile --- thrift.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thrift.dockerfile b/thrift.dockerfile index 7e260a38..ed40921b 100644 --- a/thrift.dockerfile +++ b/thrift.dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 - +ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y \ wget \ build-essential \ From 5a22a4b809cfb16124fbfe1a6138297d65ae94dc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 17:35:32 +0000 Subject: [PATCH 24/26] feat(Dependencies): Update python Docker tag to v3.11.1 | datasource | package | from | to | | ---------- | ------- | ------ | ------ | | docker | python | 3.10.9 | 3.11.1 | --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7bc18443..36b4bc5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.9-buster +FROM python:3.11.1-buster RUN apt-get update -y RUN apt-get install -y build-essential WORKDIR /code From cf4470010613f74801f2401b3af33436b99bbe64 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 20:20:16 +0000 Subject: [PATCH 25/26] feat(Dependencies): Update redis Docker tag to v7.0.6 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | docker | redis | 7.0.5 | 7.0.6 | | docker | redis | 7.0.5 | 7.0.6 | | docker | redis | 7.0.5 | 7.0.6 | | docker | redis | 7.0.5 | 7.0.6 | --- docker-compose.bibigrid.yml | 2 +- docker-compose.dev.bibigrid.yml | 2 +- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index ae19b433..14504cc0 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -17,7 +17,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.5 + image: redis:7.0.6 expose: - "6379" networks: diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index f60ee435..d9ac8b52 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.5 + image: redis:7.0.6 expose: - "6379" networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5a07526c..2722c90b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.5 + image: redis:7.0.6 expose: - "6379" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 05bd84e4..d1f8b3c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.5 + image: redis:7.0.6 expose: - "6379" networks: From 820d71a1e6972c714524f563c32c1b0b56387e68 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 23:31:06 +0000 Subject: [PATCH 26/26] feat(Dependencies): Update actions/checkout action to v3.2.0 | datasource | package | from | to | | ----------- | ---------------- | ------ | ------ | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | | github-tags | actions/checkout | v3.1.0 | v3.2.0 | --- .github/workflows/blacked.yml | 2 +- .github/workflows/build_image.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/master-protection.yml | 2 +- .github/workflows/publish_docker.yml | 2 +- .github/workflows/release_image.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/blacked.yml b/.github/workflows/blacked.yml index 21355075..cac46eb9 100644 --- a/.github/workflows/blacked.yml +++ b/.github/workflows/blacked.yml @@ -4,7 +4,7 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.2.0 - name: Checking out Branch run: git checkout "${GITHUB_REF:11}" - name: black diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 91ed87cf..d4f3d830 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -8,5 +8,5 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.2.0 - run: docker build --no-cache . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3600bc19..ccee15bf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/master-protection.yml b/.github/workflows/master-protection.yml index 323ed8dd..8eaa57f6 100644 --- a/.github/workflows/master-protection.yml +++ b/.github/workflows/master-protection.yml @@ -14,7 +14,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.2.0 - name: Check Tag run: | diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 2ca2e2a9..9a815886 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -15,7 +15,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.2.0 - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml index cfd83a19..1c599d09 100644 --- a/.github/workflows/release_image.yml +++ b/.github/workflows/release_image.yml @@ -11,7 +11,7 @@ jobs: - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.2.0 - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master with: