From ab6cc509c4625da0f33c1a22c723598e3267b714 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Mon, 31 Jul 2023 06:23:58 +0000 Subject: [PATCH 1/8] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 234 +++++++++--------- 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 0650a184..ade94482 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -274,8 +274,8 @@ def __init__(self, config): self.RE_BACKEND_URL = cfg["forc"]["forc_url"] backend_url_host = self.RE_BACKEND_URL.split(":") self.FORC_URL = ( - cfg["forc"].get("openresty_url", None) - or f"https:{backend_url_host[1]}/" + cfg["forc"].get("openresty_url", None) + or f"https:{backend_url_host[1]}/" ) self.FORC_API_KEY = os.environ.get("FORC_API_KEY", None) @@ -284,9 +284,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError self.LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -431,10 +431,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") if not properties: @@ -444,7 +444,7 @@ def get_Images(self): tags = img.get("tags", []) self.LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): self.LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -485,7 +485,7 @@ def prepare_image(self, img): tags = img.get("tags", []) self.LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): self.LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -525,11 +525,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -551,11 +551,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -613,10 +613,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -813,7 +813,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): self.LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -958,17 +958,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -1052,7 +1052,7 @@ def get_or_create_project_security_group(self, project_name, project_id): return new_security_group["id"] def get_research_environment_security_groups( - self, research_environment_names: list[str] + self, research_environment_names: list[str] ): custom_security_groups = [] @@ -1073,18 +1073,18 @@ def get_research_environment_security_groups( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1145,17 +1145,17 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + + init_script ) else: init_script = ( - self.create_add_keys_script(keys=additional_keys) - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script + self.create_add_keys_script(keys=additional_keys) + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script ) server = self.conn.create_server( @@ -1181,17 +1181,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1251,7 +1251,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): self.LOG.info( f"Create {resenv_template} Security Group for Instance: {server_id}" @@ -1281,16 +1281,16 @@ def create_resenv_security_group_and_attach_to_server( ) def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ Start a new Server. @@ -1371,7 +1371,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks self.LOG.info( @@ -1426,8 +1426,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2201,7 +2201,7 @@ def get_active_image_by_os_version(self, os_version, os_distro): return None def get_active_image_by_os_version_and_slurm_version( - self, os_version, os_distro, slurm_version + self, os_version, os_distro, slurm_version ): # 18.04 deprecated if os_version == "18.04": @@ -2233,20 +2233,20 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, - slurm_version, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, + slurm_version, ): self.LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2598,9 +2598,9 @@ def delete_server(self, openstack_id): return True task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2611,11 +2611,11 @@ def delete_server(self, openstack_id): ) if ( - sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sg["name"] - and not self.is_security_group_in_use( - security_group_id=sg["id"] - ) + sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and "bibigrid" not in sg["name"] + and not self.is_security_group_in_use( + security_group_id=sg["id"] + ) ): self.LOG.info(f"Delete security group {sg['name']}") @@ -2785,15 +2785,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): self.LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2980,7 +2980,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] self.LOG.info(self.ALL_TEMPLATES) @@ -3032,7 +3032,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -3053,7 +3053,7 @@ def load_resenv_metadata(self): return templates_metada def get_or_create_research_environment_security_group( - self, resenv_metadata: ResearchEnvironmentMetadata + self, resenv_metadata: ResearchEnvironmentMetadata ): if not resenv_metadata.needs_forc_support: return None From 2cbbf33a484f0a9bc44e42e8af6c333d99da8170 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 14:19:29 +0000 Subject: [PATCH 2/8] feat(Dependencies): Update dependency openstacksdk to v1.4.0 | datasource | package | from | to | | ---------- | ------------ | ----- | ----- | | pypi | openstacksdk | 1.3.1 | 1.4.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d7832e09..769d4445 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ setuptools==68.0.0 thrift==0.16.0 python-keystoneclient==5.1.0 -openstacksdk ==1.3.1 +openstacksdk ==1.4.0 deprecated ==1.2.14 Click==8.1.6 ansible==8.2.0 From 82fb4f342cd9afaa6c5f9bdadd6c7bb188d2eb9b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 12:47:13 +0000 Subject: [PATCH 3/8] feat(Dependencies): Update pre-commit hook sondrelg/pep585-upgrade to v1.0.1 | datasource | package | from | to | | ----------- | ----------------------- | ---- | ------ | | github-tags | sondrelg/pep585-upgrade | v1 | v1.0.1 | --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 914206bd..024df725 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/sondrelg/pep585-upgrade - rev: 'v1' # Use the sha / tag you want to point at + rev: 'v1.0.1' # Use the sha / tag you want to point at hooks: - id: upgrade-type-hints From 1c594197314697f24cbb4b33dee7a6fe4e374f24 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 12:47:18 +0000 Subject: [PATCH 4/8] feat(Dependencies): Update all non-major dependencies | datasource | package | from | to | | ----------- | --------------------------- | ------ | ------- | | github-tags | PyCQA/isort | 5.10.1 | 5.12.0 | | github-tags | myint/autoflake | v1.4 | v1.7.8 | | github-tags | pre-commit/pre-commit-hooks | v4.1.0 | v4.4.0 | | github-tags | psf/black | 22.1.0 | 22.12.0 | --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 914206bd..78a6fbf1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ default_stages: [ commit ] repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.1.0 + rev: 22.12.0 hooks: - id: black - repo: https://github.com/sondrelg/pep585-upgrade @@ -18,7 +18,7 @@ repos: - id: upgrade-type-hints - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/neutrinoceros/flynt/ @@ -27,7 +27,7 @@ repos: - id: flynt - repo: https://github.com/myint/autoflake - rev: v1.4 + rev: v1.7.8 hooks: - id: autoflake args: From ac012cac3637898753d73d15d8f8d82eae7d7325 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:13:36 +0000 Subject: [PATCH 5/8] feat(Dependencies): Update pre-commit hook PyCQA/flake8 to v6 | datasource | package | from | to | | ----------- | ------------ | ----- | ----- | | github-tags | PyCQA/flake8 | 4.0.1 | 6.1.0 | --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 78a6fbf1..f620d056 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - --remove-all-unused-imports - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 6.1.0 hooks: - id: flake8 args: [ "--config=setup.cfg" ] From 0c7152895b2808f78da205088e67d8e2a41c5906 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:13:42 +0000 Subject: [PATCH 6/8] feat(Dependencies): Update pre-commit hook myint/autoflake to v2 | datasource | package | from | to | | ----------- | --------------- | ------ | ------ | | github-tags | myint/autoflake | v1.7.8 | v2.2.0 | --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 78a6fbf1..ee1275c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: flynt - repo: https://github.com/myint/autoflake - rev: v1.7.8 + rev: v2.2.0 hooks: - id: autoflake args: From 3c028367e9783fa534c85096b18d7be8ea226956 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 8 Aug 2023 09:35:10 +0200 Subject: [PATCH 7/8] Update VirtualMachineHandler.py --- VirtualMachineService/VirtualMachineHandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 16e9dbb2..5c50e34f 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -2665,7 +2665,7 @@ def delete_volume(self, volume_id): try: self.LOG.info(f"Delete Volume {volume_id}") - self.conn.block_storage.delete_volume(volume=volume_id, force=True) + self.conn.block_storage.delete_volume(volume=volume_id) return True except ConflictException: self.LOG.exception(f"Delete volume {volume_id} error") From f6192a445add3307f641d589e8badde6f4d4bf6f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 07:35:36 +0000 Subject: [PATCH 8/8] feat(Dependencies): Update pre-commit hook psf/black to v23 | datasource | package | from | to | | ----------- | --------- | ------- | ------ | | github-tags | psf/black | 22.12.0 | 23.7.0 | --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb72a16a..1e776638 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: check-yaml - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/sondrelg/pep585-upgrade