From 23f7d37bbfc7c935badd3ef90e53fe80433fb210 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:57:45 +0000 Subject: [PATCH 1/2] feat(Dependencies): Update dependency Click to v8.1.6 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | Click | 8.1.5 | 8.1.6 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 521c8f52..edaf123e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ thrift==0.16.0 python-keystoneclient==5.1.0 openstacksdk ==1.3.1 deprecated ==1.2.14 -Click==8.1.5 +Click==8.1.6 ansible==8.2.0 flake8==6.0.0 paramiko==3.2.0 From b7764eb7931d6c5ccae209ef4820c0b794a4f0f1 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Fri, 28 Jul 2023 10:44:49 +0200 Subject: [PATCH 2/2] fix no not found if any error --- .../VirtualMachineHandler.py | 316 +++++++++--------- 1 file changed, 166 insertions(+), 150 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 34cfe210..67361321 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -106,6 +106,7 @@ class VirtualMachineHandler(Iface): ERROR = "ERROR" SHUTOFF = "SHUTOFF" NOT_FOUND = "NOT_FOUND" + CHECKING_STATUS = "CHECKING_STATUS" PREPARE_PLAYBOOK_BUILD = "PREPARE_PLAYBOOK_BUILD" BUILD_PLAYBOOK = "BUILD_PLAYBOOK" PLAYBOOK_FAILED = "PLAYBOOK_FAILED" @@ -273,8 +274,8 @@ def __init__(self, config): self.RE_BACKEND_URL = cfg["forc"]["forc_url"] backend_url_host = self.RE_BACKEND_URL.split(":") self.FORC_URL = ( - cfg["forc"].get("openresty_url", None) - or f"https:{backend_url_host[1]}/" + cfg["forc"].get("openresty_url", None) + or f"https:{backend_url_host[1]}/" ) self.FORC_API_KEY = os.environ.get("FORC_API_KEY", None) @@ -283,9 +284,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError self.LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -385,7 +386,7 @@ def get_Flavors(self): return flavors except Exception as e: self.LOG.exception(f"Get Flavors Error: {e}") - return () + return [] @deprecated( version="1.0.0", @@ -430,10 +431,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") if not properties: @@ -443,7 +444,7 @@ def get_Images(self): tags = img.get("tags", []) self.LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): self.LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -472,7 +473,7 @@ def get_Images(self): return images except Exception as e: self.LOG.exception(f"Get Images Error: {e}") - return () + return [] def prepare_image(self, img): try: @@ -484,7 +485,7 @@ def prepare_image(self, img): tags = img.get("tags", []) self.LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): self.LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -524,11 +525,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -538,7 +539,7 @@ def get_public_Images(self): return images except Exception as e: self.LOG.exception(f"Get Images Error: {e}") - return () + return [] def get_private_Images(self): """ @@ -550,11 +551,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -564,7 +565,7 @@ def get_private_Images(self): return images except Exception as e: self.LOG.exception(f"Get Images Error: {e}") - return () + return [] def get_Image_with_Tag(self, id): """ @@ -612,10 +613,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -653,7 +654,7 @@ def get_Images_by_filter(self, filter_list): return images except Exception as e: self.LOG.exception(f"Get Images Error: {e}") - return () + return [] def delete_keypair(self, key_name): key_pair = self.conn.compute.find_keypair(key_name) @@ -730,10 +731,15 @@ def get_server(self, openstack_id: str) -> VM: self.LOG.info(f"Get Server {openstack_id}") try: server: Server = self.conn.get_server_by_id(openstack_id) + if not server: + self.LOG.exception(f"No Server found {openstack_id}") + return VM(status=self.NOT_FOUND) return self.openstack_server_to_thrift_server(server=server) - except Exception as e: - self.LOG.exception(f"No Server found {openstack_id} | Error {e}") - return VM(status=self.NOT_FOUND) + + + except Exception: + self.LOG.exception(f"Could get server {openstack_id}") + return VM(status=self.CHECKING_STATUS) def get_servers_by_ids(self, ids): servers = [] @@ -741,7 +747,11 @@ def get_servers_by_ids(self, ids): self.LOG.info(f"Get server {id}") try: server = self.conn.get_server_by_id(id) - servers.append(server) + if server: + servers.append(server) + else: + self.LOG.exception(f"No Server found {openstack_id}") + except Exception as e: self.LOG.exception(f"Requested VM {id} not found!\n {e}") server_list = [] @@ -801,7 +811,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): self.LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -946,17 +956,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -1040,7 +1050,7 @@ def get_or_create_project_security_group(self, project_name, project_id): return new_security_group["id"] def get_research_environment_security_groups( - self, research_environment_names: list[str] + self, research_environment_names: list[str] ): custom_security_groups = [] @@ -1061,18 +1071,18 @@ def get_research_environment_security_groups( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1133,17 +1143,17 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + + init_script ) else: init_script = ( - self.create_add_keys_script(keys=additional_keys) - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script + self.create_add_keys_script(keys=additional_keys) + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script ) server = self.conn.create_server( @@ -1169,17 +1179,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1239,7 +1249,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): self.LOG.info( f"Create {resenv_template} Security Group for Instance: {server_id}" @@ -1269,16 +1279,16 @@ def create_resenv_security_group_and_attach_to_server( ) def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ Start a new Server. @@ -1359,7 +1369,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks self.LOG.info( @@ -1414,8 +1424,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1775,28 +1785,34 @@ def get_volume(self, volume_id): self.LOG.info(f"Get Volume {volume_id}") try: os_volume = self.conn.get_volume_by_id(id=volume_id) - self.LOG.info(os_volume) - if os_volume.attachments: - device = os_volume.attachments[0]["device"] - server_id = os_volume.attachments[0]["server_id"] + if os_volume: + self.LOG.info(os_volume) + if os_volume.attachments: + device = os_volume.attachments[0]["device"] + server_id = os_volume.attachments[0]["server_id"] + else: + device = None + server_id = None + + thrift_volume = Volume( + status=os_volume.status, + id=os_volume.id, + name=os_volume.name, + description=os_volume.description, + created_at=os_volume.created_at, + device=device, + server_id=server_id, + size=os_volume.size, + ) + + return thrift_volume else: - device = None - server_id = None - - thrift_volume = Volume( - status=os_volume.status, - id=os_volume.id, - name=os_volume.name, - description=os_volume.description, - created_at=os_volume.created_at, - device=device, - server_id=server_id, - size=os_volume.size, - ) - return thrift_volume + self.LOG.exception(f"Could not find volume {id}") + return Volume(status=self.NOT_FOUND) + except Exception: self.LOG.exception(f"Could not find volume {id}") - return Volume(status=self.NOT_FOUND) + return Volume(status=self.CHECKING_STATUS) def attach_volume_to_server(self, openstack_id, volume_id): """ @@ -1846,13 +1862,13 @@ def check_server_status(self, openstack_id: str) -> VM: self.LOG.info(f"Check Status VM {openstack_id}") try: server = self.conn.compute.get_server(openstack_id) + if not server: + self.LOG.exception(f"No Server with id {openstack_id} ") + return VM(status=self.NOT_FOUND) except Exception: - self.LOG.exception(f"No Server with id {openstack_id} ") - return VM(status=self.NOT_FOUND) + self.LOG.exception(f"Could not get server {openstack_id} ") + return VM(status=self.CHECKING_STATUS) - if server is None: - self.LOG.exception(f"No Server with id {openstack_id} ") - return VM(status=self.NOT_FOUND) serv = server.to_dict() @@ -2178,7 +2194,7 @@ def get_active_image_by_os_version(self, os_version, os_distro): return None def get_active_image_by_os_version_and_slurm_version( - self, os_version, os_distro, slurm_version + self, os_version, os_distro, slurm_version ): # 18.04 deprecated if os_version == "18.04": @@ -2210,20 +2226,20 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, - slurm_version, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, + slurm_version, ): self.LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2575,9 +2591,9 @@ def delete_server(self, openstack_id): return True task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2588,11 +2604,11 @@ def delete_server(self, openstack_id): ) if ( - sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sg["name"] - and not self.is_security_group_in_use( - security_group_id=sg["id"] - ) + sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and "bibigrid" not in sg["name"] + and not self.is_security_group_in_use( + security_group_id=sg["id"] + ) ): self.LOG.info(f"Delete security group {sg['name']}") @@ -2762,15 +2778,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): self.LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2957,7 +2973,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] self.LOG.info(self.ALL_TEMPLATES) @@ -3009,7 +3025,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -3030,7 +3046,7 @@ def load_resenv_metadata(self): return templates_metada def get_or_create_research_environment_security_group( - self, resenv_metadata: ResearchEnvironmentMetadata + self, resenv_metadata: ResearchEnvironmentMetadata ): if not resenv_metadata.needs_forc_support: return None