From 30a315d18d7ca91546292e6a87329d6b3715baa5 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Mon, 16 May 2022 17:07:31 +0200 Subject: [PATCH 01/12] refactor(Playbooks):refacted resenv structure --- .../VirtualMachineHandler.py | 309 +++++++++--------- VirtualMachineService/ancon/Playbook.py | 68 ++-- .../ancon/playbooks/roles/guacamole/LICENSE | 201 ------------ .../ancon/playbooks/roles/guacamole/README.md | 66 ---- .../roles/guacamole/defaults/main.yml | 2 - .../guacamole/files/guacamole.properties | 1 - .../roles/guacamole/handlers/main.yml | 21 -- .../playbooks/roles/guacamole/meta/main.yml | 18 - .../roles/guacamole/tasks/005-desktop.yml | 35 -- .../roles/guacamole/tasks/010-guacamole.yml | 116 ------- .../playbooks/roles/guacamole/tasks/main.yml | 8 - .../guacamole/templates/user-mapping.xml.j2 | 18 - .../playbooks/roles/guacamole/tests/inventory | 1 - .../playbooks/roles/guacamole/tests/test.yml | 5 - .../playbooks/roles/guacamole/vars/main.yml | 9 - .../theia/templates/theia-ide.service.j2 | 14 - .../roles/theia/templates/theia-ide.sh.j2 | 5 - 17 files changed, 183 insertions(+), 714 deletions(-) delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/LICENSE delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/README.md delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/defaults/main.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/files/guacamole.properties delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/handlers/main.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/meta/main.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/005-desktop.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/010-guacamole.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/main.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/templates/user-mapping.xml.j2 delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/tests/inventory delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/tests/test.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/guacamole/vars/main.yml delete mode 100644 VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.service.j2 delete mode 100644 VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.sh.j2 diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index cc73f3c8..e7bdce00 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -6,6 +6,9 @@ import math import sys from uuid import uuid4 +from pathlib import Path +import zipfile +import shutil try: from ancon.Playbook import ALL_TEMPLATES, Playbook @@ -55,7 +58,8 @@ import urllib from contextlib import closing from distutils.version import LooseVersion - +import glob +import shutil import redis import requests as req import yaml @@ -239,9 +243,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -382,10 +386,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): metadata = img["metadata"] @@ -393,7 +397,7 @@ def get_Images(self): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -429,7 +433,7 @@ def prepare_image(self, img): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -465,11 +469,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -491,11 +495,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -548,10 +552,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags") if "resenv" in filter_list: @@ -734,7 +738,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -879,17 +883,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -962,18 +966,18 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https): return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1022,9 +1026,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1057,17 +1061,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1121,16 +1125,16 @@ def start_server( return {} def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1206,7 +1210,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info(msg=f"Starting Playbook for (openstack_id): {openstack_id}") @@ -1255,8 +1259,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1525,38 +1529,11 @@ def get_template_version_for(self, template): def get_templates(self): return [] - # Todo test this method def get_allowed_templates(self): - templates_metada = [] - # Todo load Metadata from multiple folders - for file in os.listdir(PLAYBOOKS_DIR): - if "_metadata.yml" in file: - with open(PLAYBOOKS_DIR + file) as template_metadata: - try: - loaded_metadata = yaml.load( - template_metadata, Loader=yaml.FullLoader - ) - template_name = loaded_metadata[TEMPLATE_NAME] - if loaded_metadata["needs_forc_support"]: - if template_name in list(self.FORC_ALLOWED.keys()): - templates_metada.append(json.dumps(loaded_metadata)) - if template_name not in self.ALL_TEMPLATES: - ALL_TEMPLATES.append(template_name) - else: - LOG.info( - "Failed to find supporting FORC file for " - + str(template_name) - ) - else: - templates_metada.append(json.dumps(loaded_metadata)) - if template_name not in self.ALL_TEMPLATES: - ALL_TEMPLATES.append(template_name) - - except Exception as e: - LOG.exception( - "Failed to parse Metadata yml: " + file + "\n" + str(e) - ) - return templates_metada + templates_metadata = [] + for key, value in self.loaded_resenv_metadata.items(): + templates_metadata.append(value.json_string) + return templates_metadata def get_templates_by_template(self, template_name): get_url = f"{self.RE_BACKEND_URL}{self.TEMPLATES_URL}/{template_name}" @@ -1876,8 +1853,8 @@ def add_udp_security_group(self, server_id): ip_base = ( list(self.conn.compute.server_ips(server=server_id))[0] - .to_dict()["address"] - .split(".")[-1] + .to_dict()["address"] + .split(".")[-1] ) x = int(ip_base) # noqa F841 udp_port_start = eval(self.UDP_FORMULAR) @@ -2001,9 +1978,9 @@ def get_active_image_by_os_version(self, os_version, os_distro): image_os_distro = metadata.get("os_distro", None) base_image_ref = metadata.get("base_image_ref", None) if ( - os_version == image_os_version - and image.status == "active" - and base_image_ref is None + os_version == image_os_version + and image.status == "active" + and base_image_ref is None ): if os_distro and os_distro == image_os_distro: return image @@ -2022,17 +1999,17 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, ): LOG.info(f"Add machine to {cluster_id} - {key_name}") image = self.get_image(image=image) @@ -2081,7 +2058,7 @@ def add_cluster_machine( return server["id"] def scale_up_cluster( - self, cluster_id, image, flavor, count, names, start_idx, batch_index + self, cluster_id, image, flavor, count, names, start_idx, batch_index ): cluster_info = self.get_cluster_info(cluster_id=cluster_id) image = self.get_image(image=image) @@ -2365,9 +2342,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2376,7 +2353,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec.name != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec.name + and "bibigrid" not in sec.name ] if security_groups is not None: for sg in security_groups: @@ -2527,15 +2504,15 @@ def resume_server(self, openstack_id): return False def create_security_group( - self, - name, - udp_port_start=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port_start=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2677,6 +2654,12 @@ def get_limits(self): "totalGigabytesUsed": str(limits["totalGigabytesUsed"]), } + def install_ansible_galaxy_requirements(self): + LOG.info("Installing Ansible galaxy requirements..") + stream = os.popen(f'ansible-galaxy install -r {PLAYBOOKS_DIR}/packer/requirements.yml') + output = stream.read() + LOG.info(output) + def update_playbooks(self): if self.GITHUB_PLAYBOOKS_REPO is None: LOG.info( @@ -2685,16 +2668,20 @@ def update_playbooks(self): return LOG.info(f"STARTED update of playbooks from - {self.GITHUB_PLAYBOOKS_REPO}") r = req.get(self.GITHUB_PLAYBOOKS_REPO) - contents = json.loads(r.content) - # Todo maybe clone entire direcotry - for f in contents: - if f["name"] != "LICENSE": - LOG.info("started download of " + f["name"]) - download_link = f["download_url"] - file_request = req.get(download_link) - filename = "/code/VirtualMachineService/ancon/playbooks/" + f["name"] - with open(filename, "w") as playbook_file: - playbook_file.write(file_request.content.decode("utf-8")) + filename = "resenv_repo" + with open(filename, 'wb') as output_file: + output_file.write(r.content) + LOG.info('Downloading Completed') + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(PLAYBOOKS_DIR) + + resenvs_unziped_dir = next(filter(lambda f: os.path.isdir(f) and "resenvs" in f, glob.glob(PLAYBOOKS_DIR + '*'))) + shutil.copytree(resenvs_unziped_dir, PLAYBOOKS_DIR, dirs_exist_ok=True) + shutil.rmtree(resenvs_unziped_dir, ignore_errors=True) + self.ALL_TEMPLATES = [name for name in os.listdir(PLAYBOOKS_DIR) if + name != "packer" and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name))] + LOG.info(self.ALL_TEMPLATES) + templates_metadata = self.load_resenv_metadata() for template_metadata in templates_metadata: try: @@ -2707,6 +2694,7 @@ def update_playbooks(self): template_metadata[DIRECTION], template_metadata[PROTOCOL], template_metadata[INFORMATION_FOR_DISPLAY], + json_string=json.dumps(template_metadata) ) self.update_forc_allowed(template_metadata) if metadata.name not in list(self.loaded_resenv_metadata.keys()): @@ -2722,13 +2710,14 @@ def update_playbooks(self): + "\n" + str(e) ) + self.install_ansible_galaxy_requirements() LOG.info(self.loaded_resenv_metadata) def load_resenv_metadata(self): templates_metada = [] - for file in os.listdir(PLAYBOOKS_DIR): - if "_metadata.yml" in file: - with open(PLAYBOOKS_DIR + file) as template_metadata: + for template in self.ALL_TEMPLATES: + try: + with open(f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml") as template_metadata: try: loaded_metadata = yaml.load( template_metadata, Loader=yaml.FullLoader @@ -2742,6 +2731,8 @@ def load_resenv_metadata(self): LOG.exception( "Failed to parse Metadata yml: " + file + "\n" + str(e) ) + except Exception as e: + LOG.exception(f"No Metadatafile found for {template} - {e}") return templates_metada def update_forc_allowed(self, template_metadata): @@ -2770,15 +2761,16 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + json_string ): self.name = name self.port = port @@ -2788,3 +2780,4 @@ def __init__( self.direction = direction self.protocol = protocol self.information_for_display = information_for_display + self.json_string = json_string diff --git a/VirtualMachineService/ancon/Playbook.py b/VirtualMachineService/ancon/Playbook.py index f99c8eec..d25cd971 100644 --- a/VirtualMachineService/ancon/Playbook.py +++ b/VirtualMachineService/ancon/Playbook.py @@ -35,15 +35,15 @@ class Playbook(object): PLAYBOOK_FAILED = "PLAYBOOK_FAILED" def __init__( - self, - ip, - port, - playbooks_information, - osi_private_key, - public_key, - pool, - loaded_metadata_keys, - cloud_site, + self, + ip, + port, + playbooks_information, + osi_private_key, + public_key, + pool, + loaded_metadata_keys, + cloud_site, ): self.loaded_metadata_keys = loaded_metadata_keys self.cloud_site = cloud_site @@ -88,10 +88,10 @@ def __init__( ) inventory_string = ( - "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " - "ansible_ssh_private_key_file=" - + self.private_key.name - + " ansible_python_interpreter=/usr/bin/python3" + "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " + "ansible_ssh_private_key_file=" + + self.private_key.name + + " ansible_python_interpreter=/usr/bin/python3" ) self.inventory.write(inventory_string) self.inventory.close() @@ -107,12 +107,12 @@ def copy_playbooks_and_init(self, playbooks_information, public_key): self.playbooks_dir + "/change_key_vars_file.yml", self.directory.name ) with open( - self.directory.name + "/change_key_vars_file.yml", mode="r" + self.directory.name + "/change_key_vars_file.yml", mode="r" ) as key_file: data_ck = self.yaml_exec.load(key_file) data_ck["change_key_vars"]["key"] = public_key.strip('"') with open( - self.directory.name + "/change_key_vars_file.yml", mode="w" + self.directory.name + "/change_key_vars_file.yml", mode="w" ) as key_file: self.yaml_exec.dump(data_ck, key_file) self.add_to_playbook_always_lists("change_key") @@ -122,14 +122,14 @@ def copy_playbooks_and_init(self, playbooks_information, public_key): self.playbooks_dir + "/" + self.playbook_exec_name, self.directory.name ) with open( - self.directory.name + "/" + self.playbook_exec_name, mode="r" + self.directory.name + "/" + self.playbook_exec_name, mode="r" ) as generic_playbook: data_gp = self.yaml_exec.load(generic_playbook) data_gp[0]["vars_files"] = self.vars_files data_gp[0]["tasks"][0]["block"] = self.tasks data_gp[0]["tasks"][0]["always"] = self.always_tasks with open( - self.directory.name + "/" + self.playbook_exec_name, mode="w" + self.directory.name + "/" + self.playbook_exec_name, mode="w" ) as generic_playbook: self.yaml_exec.dump(data_gp, generic_playbook) @@ -160,36 +160,32 @@ def load_vars(): if k == MOSH: data[playbook_name + "_defined"][k] = v + # copy whole directory + shutil.copytree(f"{self.playbooks_dir}/{playbook_name}", self.directory.name,dirs_exist_ok=True) + site_specific_yml = f"/{playbook_name}{'-' + self.cloud_site}.yml" playbook_name_local = playbook_name - if os.path.isfile(self.playbooks_dir + site_specific_yml): + if os.path.isfile(self.directory.name + site_specific_yml): playbook_name_local = playbook_name + "-" + self.cloud_site - playbook_yml = f"/{playbook_name_local}.yml" playbook_var_yml = f"/{playbook_name}_vars_file.yml" + try: - shutil.copy(self.playbooks_dir + playbook_yml, self.directory.name) - try: - shutil.copy(self.playbooks_dir + playbook_var_yml, self.directory.name) - with open( + with open( self.directory.name + playbook_var_yml, mode="r" - ) as variables: - data = self.yaml_exec.load(variables) - load_vars() - with open( + ) as variables: + data = self.yaml_exec.load(variables) + load_vars() + with open( self.directory.name + playbook_var_yml, mode="w" - ) as variables: - self.yaml_exec.dump(data, variables) - self.add_to_playbook_lists(playbook_name_local, playbook_name) - except shutil.Error as e: - LOG.exception(e) - self.add_tasks_only(playbook_name_local) - except IOError as e: - LOG.exception(e) - self.add_tasks_only(playbook_name_local) + ) as variables: + self.yaml_exec.dump(data, variables) + self.add_to_playbook_lists(playbook_name_local, playbook_name) except shutil.Error as e: LOG.exception(e) + self.add_tasks_only(playbook_name_local) except IOError as e: LOG.exception(e) + self.add_tasks_only(playbook_name_local) def add_to_playbook_lists(self, playbook_name_local, playbook_name): self.vars_files.append(playbook_name + "_vars_file.yml") diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/LICENSE b/VirtualMachineService/ancon/playbooks/roles/guacamole/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/README.md b/VirtualMachineService/ancon/playbooks/roles/guacamole/README.md deleted file mode 100644 index 36f9576d..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/README.md +++ /dev/null @@ -1,66 +0,0 @@ -guacamolerdp-ansible -========= - -This role prepares a fresh Ubuntu 18.04 instance to be a fully fledged working environment via Xfce4 and XRDP. -XRDP gets bundled with guacamole, a clientless remote desktop web gateway. - -Aim of this is, that we can "reverse proxy" a remote desktop session to a privileged user with a remoteproxy webserver -provisioned with [de.NBI FORC](https://github.com/deNBI/simpleVMWebGateway). - -**For security reasons, you should execute this role on a VM, which is not publicly reachable via internet. Protect the VM with authentication via ReverseProxy, firewall etc.** - -Also an important security notification: - -Guacamole needs a valid unix user and password to automatically create and connect to a valid rdp session. -This role creates a default user with a default password described in `vars/main.yml`. You have been warned. -For more see the `Role Variables` section. - -Requirements ------------- - -* Ubuntu 18.04 -* Internet connection on the target -* Guacamole runs on port `8080`, make sure its not in use already. - -Role Variables --------------- - -**Again: If the targeted machine is not externaly protected or not used in a FORC environment with appropriate firewall rules, change these values!!!** - -**vars/main.yml** - -| Variable | Description | Default | Mandatory | -| ------------- |------------- | ----- | --- | -| DEFAULT_USER | Default unix user on which guacamole connects to | ubuntu | Yes | -| DEFAULT_PASSWORD | Default password of the unix user. Change it when target is not externally protected via ReverseProxy or other. | ogvkyf | Yes | -| DEFAULT_PASSWORD_HASHED | Hashed password of DEFAULT_PASSWORD | $6$iRrIJogr... | Yes | -| GUAC_USER | Default guacamole user | denbi | Yes | -| GUAC_PASSWORD | Default guacamole password | denbi | Yes | - - -Dependencies ------------- - -* No dependencies. - -Example Playbook ----------------- - -Make sure to include `become: yes`. Using this role in a playbook is straight forward: - - - hosts: servers - become: yes - roles: - - guacamolerdp-ansible - -License -------- - -Apache 2.0 - -Author Information ------------------- - -Alex Walender - -de.NBI Cloud Bielefeld diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/defaults/main.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/defaults/main.yml deleted file mode 100644 index d531a940..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# defaults file for guacamolerdp-ansible diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/files/guacamole.properties b/VirtualMachineService/ancon/playbooks/roles/guacamole/files/guacamole.properties deleted file mode 100644 index c14fbfa1..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/files/guacamole.properties +++ /dev/null @@ -1 +0,0 @@ -basic-user-mapping: /etc/guacamole/user-mapping.xml diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/handlers/main.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/handlers/main.yml deleted file mode 100644 index 304216fd..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/handlers/main.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# handlers file for guacamolerdp-ansible - -- name: Restart xrdp - systemd: - name: xrdp - state: restarted - -- name: Restart guacd - systemd: - name: guacd - state: restarted - -- name: Restart Tomcat - systemd: - name: tomcat9 - state: restarted - -- name: Reload systemd - systemd: - daemon_reload: yes diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/meta/main.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/meta/main.yml deleted file mode 100644 index 9d275b84..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/meta/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -galaxy_info: - author: Alex Walender - description: Installs guacamole, xrdp and xfce4 for a target which will act as a SimpleVM in the de.NBI Cloud SimpleVM context. - company: de.NBi Cloud Bielefeld - license: Apache - min_ansible_version: 2.4 - - platforms: - - name: Ubuntu - versions: - - bionic - - galaxy_tags: - - xfce - - xrdp - - guacamole - -dependencies: [] diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/005-desktop.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/005-desktop.yml deleted file mode 100644 index 81fe3212..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/005-desktop.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- - -- name: Update apt cache - apt: - cache_valid_time: 7600 - -- name: Install xfce4 - apt: - name: xfce4* - state: latest - -- name: Install Firefox - apt: - name: firefox - state: latest - -- name: Install xrdp and other goodies - apt: - name: - - xrdp - - xorg - - dbus-x11 - - x11-xserver-utils - state: latest - -- name: Config xrdp to start xfce4 - lineinfile: - path: /etc/xrdp/xrdp.ini - line: "exec startxfce4" - notify: Restart xrdp - -- name: Setup password for default user - user: - name: "{{ DEFAULT_USER }}" - password: "{{ DEFAULT_PASSWORD_HASHED }}" diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/010-guacamole.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/010-guacamole.yml deleted file mode 100644 index 94405ee1..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/010-guacamole.yml +++ /dev/null @@ -1,116 +0,0 @@ - - -- name: Install needed libraries and tools - apt: - name: - - make - - gcc - - vim - - curl - - wget - - g++ - - libcairo2-dev - - libjpeg-turbo8-dev - - libpng-dev - - libtool-bin - - libossp-uuid-dev - - libavcodec-dev - - libavutil-dev - - libswscale-dev - - build-essential - - libpango1.0-dev - - libssh2-1-dev - - libvncserver-dev - - libtelnet-dev - - freerdp2-dev - - libwebsockets-dev - - libssl-dev - - libvorbis-dev - - libwebp-dev - - tomcat9 - - tomcat9-admin - - tomcat9-user - state: latest - -- name: Download guacamole src - get_url: - url: http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.4.0/source/guacamole-server-1.4.0.tar.gz - dest: /usr/share/guacamole.tar.gz - -- name: Download guacamole war file - get_url: - url: http://apache.org/dyn/closer.cgi?action=download&filename=guacamole/1.4.0/binary/guacamole-1.4.0.war - dest: /usr/share/guacamole.war - -- name: Prepare build directory - file: - state: directory - path: /usr/share/guacamole/ - -- name: Unarchive source files - unarchive: - remote_src: yes - src: /usr/share/guacamole.tar.gz - dest: /usr/share/guacamole - -- name: Configure systemd unit - shell: "./configure --with-systemd-dir=/etc/systemd/system" - args: - chdir: /usr/share/guacamole/guacamole-server-1.4.0/ - creates: /usr/share/guacamole/guacamole-server-1.4.0/Makefile - -- name: Compile guacamole - make: - chdir: /usr/share/guacamole/guacamole-server-1.4.0/ - -- name: Make install - make: - chdir: /usr/share/guacamole/guacamole-server-1.4.0/ - target: install - -- name: Link libraries - shell: ldconfig - args: - chdir: /usr/share/guacamole/guacamole-server-1.4.0/ - -- name: Enable guacamole daemon - systemd: - name: guacd - enabled: yes - -- name: Create guacamole config folder - file: - state: directory - path: /etc/guacamole - -- name: Create guacamole.properties - copy: - src: guacamole.properties - dest: /etc/guacamole/guacamole.properties - notify: Restart guacd - -- name: Generate guacamole mapping - template: - src: user-mapping.xml.j2 - dest: /etc/guacamole/user-mapping.xml - notify: Restart guacd - -- name: Register path in env - lineinfile: - path: /etc/environment - line: 'GUACAMOLE_HOME="/etc/guacamole"' - state: present - -- name: Copy tomcat guacamole client - copy: - remote_src: yes - src: /usr/share/guacamole.war - dest: /var/lib/tomcat9/webapps/guacamole.war - notify: Restart Tomcat - -- name: Link configuration directory to tomcat - file: - state: link - src: /etc/guacamole - dest: /usr/share/tomcat9/.guacamole - notify: Reload systemd diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/main.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/main.yml deleted file mode 100644 index a234de77..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# tasks file for guacamolerdp-ansible - -- include: 005-desktop.yml - tags: ['guacamole_desktop'] - -- include: 010-guacamole.yml - tags: ['guacamole_guacamole'] diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/templates/user-mapping.xml.j2 b/VirtualMachineService/ancon/playbooks/roles/guacamole/templates/user-mapping.xml.j2 deleted file mode 100644 index db819dde..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/templates/user-mapping.xml.j2 +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - rdp - 127.0.0.1 - 3389 - {{ DEFAULT_USER }} - {{ DEFAULT_PASSWORD }} - - - - - - diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/inventory b/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/inventory deleted file mode 100644 index 2fbb50c4..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/inventory +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/test.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/test.yml deleted file mode 100644 index f08718ea..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/tests/test.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- hosts: localhost - remote_user: root - roles: - - guacamolerdp-ansible diff --git a/VirtualMachineService/ancon/playbooks/roles/guacamole/vars/main.yml b/VirtualMachineService/ancon/playbooks/roles/guacamole/vars/main.yml deleted file mode 100644 index d8ce58e5..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/guacamole/vars/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# vars file for guacamolerdp-ansible - -DEFAULT_USER: ubuntu -DEFAULT_PASSWORD: ogvkyf -# you can create password hashes like here https://stackoverflow.com/questions/19292899/creating-a-new-user-and-password-with-ansible -DEFAULT_PASSWORD_HASHED: $6$iRrIJogrR0N2ZVdb$oL5XzFqcTC.O3g4DS945a5K7nRvO0LqY.ugyjDINUA347qRYXe1YXag8cRlLf9PVwmNrRYU3LTNGyCIPbFvz4/ -GUAC_USER: denbi -GUAC_PASSWORD: denbi diff --git a/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.service.j2 b/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.service.j2 deleted file mode 100644 index 6e2aba99..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.service.j2 +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Theia-IDE service for user {{ theia_ide_user }} -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=simple -Restart=always -RestartSec=1 -User={{ theia_ide_user }} -ExecStart={{ theia_ide_install_dir }}/theia-ide.sh {{ theia_ide_workspace }} {{ theia_ide_bind_address }} {{ theia_ide_bind_port }} - -[Install] -WantedBy=multi-user.target diff --git a/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.sh.j2 b/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.sh.j2 deleted file mode 100644 index b67fdec1..00000000 --- a/VirtualMachineService/ancon/playbooks/roles/theia/templates/theia-ide.sh.j2 +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source {{ nvm_install_dir }}/nvm.sh -cd $(dirname ${0}) -yarn theia start ${1} --hostname ${2} --port ${3} From 1061c98552b2a73ca0c6c954077a4ef909c3a09f Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 16 May 2022 20:04:19 +0000 Subject: [PATCH 02/12] feat(Dependencies): Update dependency pyvim to v3.0.3 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | pyvim | 3.0.2 | 3.0.3 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c823b88e..f1766c87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ Click==8.1.3 flake8==4.0.1 paramiko==2.10.4 ruamel.yaml==0.17.21 -pyvim==3.0.2 +pyvim==3.0.3 redis==4.3.1 requests==2.27.1 pyyaml==6.0 From ee170330949fde09ee82b3da488eef81a501a434 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 16 May 2022 22:58:18 +0000 Subject: [PATCH 03/12] feat(Dependencies): Update dependency setuptools to v62.3.0 | datasource | package | from | to | | ---------- | ---------- | ------ | ------ | | pypi | setuptools | 62.2.0 | 62.3.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f1766c87..f750b658 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==62.2.0 +setuptools==62.3.0 thrift==0.16.0 python-keystoneclient openstacksdk ==0.61.0 From b65706e40d7bdfa9289ab30df5f96d061802fea7 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 17 May 2022 01:47:50 +0000 Subject: [PATCH 04/12] feat(Dependencies): Update dependency paramiko to v2.10.5 | datasource | package | from | to | | ---------- | -------- | ------ | ------ | | pypi | paramiko | 2.10.4 | 2.10.5 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f750b658..53fe0874 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ deprecated == 1.2.13 ansible==5.7.1 Click==8.1.3 flake8==4.0.1 -paramiko==2.10.4 +paramiko==2.10.5 ruamel.yaml==0.17.21 pyvim==3.0.3 redis==4.3.1 From 636aec8ae0149a0ffc22ab6877bc3aa4e3351f2c Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 17 May 2022 04:04:36 +0000 Subject: [PATCH 05/12] feat(Dependencies): Update dependency paramiko to v2.11.0 | datasource | package | from | to | | ---------- | -------- | ------ | ------ | | pypi | paramiko | 2.10.5 | 2.11.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53fe0874..7ab4d408 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ deprecated == 1.2.13 ansible==5.7.1 Click==8.1.3 flake8==4.0.1 -paramiko==2.10.5 +paramiko==2.11.0 ruamel.yaml==0.17.21 pyvim==3.0.3 redis==4.3.1 From 06a80a1154c12a620eac1164fe8007078fe8f9d5 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 17 May 2022 10:55:11 +0200 Subject: [PATCH 06/12] refactored conda --- .../VirtualMachineHandler.py | 42 +++++----- VirtualMachineService/ancon/Playbook.py | 8 +- .../ancon/playbooks/bioconda.yml | 80 ------------------- .../ancon/playbooks/bioconda_vars_file.yml | 12 --- requirements.yml | 6 ++ 5 files changed, 34 insertions(+), 114 deletions(-) delete mode 100644 VirtualMachineService/ancon/playbooks/bioconda.yml delete mode 100644 VirtualMachineService/ancon/playbooks/bioconda_vars_file.yml diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index e7bdce00..e4cc21da 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -98,6 +98,7 @@ PROTOCOL = "protocol" TEMPLATE_NAME = "template_name" INFORMATION_FOR_DISPLAY = "information_for_display" +NEEDS_FORC_SUPPORT = "needs_forc_support" FORC_VERSIONS = "forc_versions" @@ -1213,7 +1214,7 @@ def create_and_deploy_playbook( self, public_key, playbooks_information, openstack_id ): global active_playbooks - LOG.info(msg=f"Starting Playbook for (openstack_id): {openstack_id}") + LOG.info(msg=f"Starting Playbook for (openstack_id): {openstack_id} --> {playbooks_information}") port = self.get_vm_ports(openstack_id=openstack_id) key = self.redis.hget(openstack_id, "key").decode("utf-8") playbook = Playbook( @@ -1532,7 +1533,8 @@ def get_templates(self): def get_allowed_templates(self): templates_metadata = [] for key, value in self.loaded_resenv_metadata.items(): - templates_metadata.append(value.json_string) + if value.needs_forc_support: + templates_metadata.append(value.json_string) return templates_metadata def get_templates_by_template(self, template_name): @@ -2685,23 +2687,25 @@ def update_playbooks(self): templates_metadata = self.load_resenv_metadata() for template_metadata in templates_metadata: try: - metadata = ResenvMetadata( - template_metadata[TEMPLATE_NAME], - template_metadata[PORT], - template_metadata[SECURITYGROUP_NAME], - template_metadata[SECURITYGROUP_DESCRIPTION], - template_metadata[SECURITYGROUP_SSH], - template_metadata[DIRECTION], - template_metadata[PROTOCOL], - template_metadata[INFORMATION_FOR_DISPLAY], - json_string=json.dumps(template_metadata) - ) - self.update_forc_allowed(template_metadata) - if metadata.name not in list(self.loaded_resenv_metadata.keys()): - self.loaded_resenv_metadata[metadata.name] = metadata - else: - if self.loaded_resenv_metadata[metadata.name] != metadata: + if template_metadata.get(NEEDS_FORC_SUPPORT, False): + metadata = ResenvMetadata( + template_metadata[TEMPLATE_NAME], + template_metadata[PORT], + template_metadata[SECURITYGROUP_NAME], + template_metadata[SECURITYGROUP_DESCRIPTION], + template_metadata[SECURITYGROUP_SSH], + template_metadata[DIRECTION], + template_metadata[PROTOCOL], + template_metadata[INFORMATION_FOR_DISPLAY], + needs_forc_support=template_metadata.get(NEEDS_FORC_SUPPORT, False), + json_string=json.dumps(template_metadata) + ) + self.update_forc_allowed(template_metadata) + if metadata.name not in list(self.loaded_resenv_metadata.keys()): self.loaded_resenv_metadata[metadata.name] = metadata + else: + if self.loaded_resenv_metadata[metadata.name] != metadata: + self.loaded_resenv_metadata[metadata.name] = metadata except Exception as e: LOG.exception( @@ -2770,6 +2774,7 @@ def __init__( direction, protocol, information_for_display, + needs_forc_support, json_string ): self.name = name @@ -2781,3 +2786,4 @@ def __init__( self.protocol = protocol self.information_for_display = information_for_display self.json_string = json_string + self.needs_forc_support = needs_forc_support diff --git a/VirtualMachineService/ancon/Playbook.py b/VirtualMachineService/ancon/Playbook.py index d25cd971..45951ecd 100644 --- a/VirtualMachineService/ancon/Playbook.py +++ b/VirtualMachineService/ancon/Playbook.py @@ -8,11 +8,11 @@ import redis import ruamel.yaml -BIOCONDA = "bioconda" +CONDA = "conda" OPTIONAL = "optional" MOSH = "mosh" -ALL_TEMPLATES = [BIOCONDA] +ALL_TEMPLATES = [CONDA] LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -135,7 +135,7 @@ def copy_playbooks_and_init(self, playbooks_information, public_key): def copy_and_init(self, playbook_name, playbook_vars): def load_vars(): - if playbook_name == BIOCONDA: + if playbook_name == CONDA: for k, v in playbook_vars.items(): if k == "packages": p_array = [] @@ -144,7 +144,7 @@ def load_vars(): p_array.append(p.split("=")) for p in p_array: p_dict.update({p[0]: {"version": p[1]}}) - data[playbook_name + "_tools"][k] = p_dict + data[playbook_name + "_vars"][k] = p_dict if playbook_name in self.loaded_metadata_keys: for k, v in playbook_vars.items(): LOG.info(playbook_vars) diff --git a/VirtualMachineService/ancon/playbooks/bioconda.yml b/VirtualMachineService/ancon/playbooks/bioconda.yml deleted file mode 100644 index dee7ae1d..00000000 --- a/VirtualMachineService/ancon/playbooks/bioconda.yml +++ /dev/null @@ -1,80 +0,0 @@ -- name: Download miniconda install script - become_user: "{{ bioconda_user.name }}" - get_url: - args: - url: "{{ bioconda_folders.conda_installer_url }}" - dest: "{{ bioconda_folders.install_script }}" - mode: 0755 - timeout: 180 - force: no - -- name: Install miniconda - become_user: "{{ bioconda_user.name }}" - shell: "timeout 3m {{ bioconda_folders.install_script }} -b" - args: - executable: /bin/bash - creates: "{{ bioconda_folders.conda_dir }}" - -- name: Check for channels - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda config --get channels'" - register: added_channels - -- name: Add default channel - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda config --add channels default'" - args: - executable: /bin/bash - when: added_channels.stdout.find('default') == -1 - -- name: Add bioconda channel - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda config --add channels bioconda'" - args: - executable: /bin/bash - when: added_channels.stdout.find('bioconda') == -1 - -- name: Add conda-forge channel - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda config --add channels conda-forge'" - args: - executable: /bin/bash - when: added_channels.stdout.find('conda-forge') == -1 - -- name: Init .bashrc for conda - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda init'" - args: - executable: /bin/bash - -- name: Check for environment - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda info -e'" - register: added_envs - -- name: Create alias for environment - become_user: "{{ bioconda_user.name }}" - shell: "echo $ALIAS_VARIABLE > ~/.bash_aliases" - environment: - ALIAS_VARIABLE: 'alias {{ bioconda_tools.env | quote }}="conda activate {{ bioconda_tools.env | quote }}"' - when: added_envs.stdout.find(bioconda_tools.env) == -1 - -- name: Create environment - become_user: "{{ bioconda_user.name }}" - shell: "timeout 2m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda create --yes -n {{ bioconda_tools.env | quote}}'" - args: - executable: /bin/bash - when: added_envs.stdout.find(bioconda_tools.env) == -1 - -- name: Check for installed packages - become_user: "{{ bioconda_user.name }}" - shell: "timeout 1m bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda activate {{ bioconda_tools.env | quote}} && conda list'" - register: added_packages - -- name: Install chosen packages - become_user: "{{ bioconda_user.name }}" - shell: "timeout {{ bioconda_tools.timeout_length }} bash -c 'source {{ bioconda_folders.conda_dir }}/bin/activate && conda activate {{ bioconda_tools.env | quote}} && conda install --yes {{ item.key }}={{ item.value.version }}'" - args: - executable: /bin/bash - loop: "{{ q('dict', bioconda_tools.packages) }}" - when: added_packages.stdout.find(item.key) == -1 diff --git a/VirtualMachineService/ancon/playbooks/bioconda_vars_file.yml b/VirtualMachineService/ancon/playbooks/bioconda_vars_file.yml deleted file mode 100644 index 3439a158..00000000 --- a/VirtualMachineService/ancon/playbooks/bioconda_vars_file.yml +++ /dev/null @@ -1,12 +0,0 @@ -bioconda_tools: - packages: - env: "denbi" - timeout_length: "5m" - -bioconda_folders: - install_script: "/home/{{ bioconda_user.name }}/install_miniconda3.sh" - conda_dir: "/home/{{ bioconda_user.name }}/miniconda3" - conda_installer_url: "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" - -bioconda_user: - name: "ubuntu" diff --git a/requirements.yml b/requirements.yml index d74f30ee..7d22c965 100644 --- a/requirements.yml +++ b/requirements.yml @@ -4,3 +4,9 @@ roles: - name: oefenweb.latest_r version: v3.1.1 + + + - name: andrewrothstein.miniconda + version: v6.1.5 + + - name: andrewrothstein.conda-env From c34e8fe02aed922c0499abc88189d8ac710bcde8 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 17 May 2022 08:56:22 +0000 Subject: [PATCH 07/12] fix(Linting):blacked code --- .../VirtualMachineHandler.py | 268 ++++++++++-------- VirtualMachineService/ancon/Playbook.py | 48 ++-- 2 files changed, 166 insertions(+), 150 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index e4cc21da..9e5f33f9 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -244,9 +244,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -387,10 +387,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): metadata = img["metadata"] @@ -398,7 +398,7 @@ def get_Images(self): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -434,7 +434,7 @@ def prepare_image(self, img): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -470,11 +470,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -496,11 +496,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -553,10 +553,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags") if "resenv" in filter_list: @@ -739,7 +739,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -884,17 +884,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -967,18 +967,18 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https): return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1027,9 +1027,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1062,17 +1062,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1126,16 +1126,16 @@ def start_server( return {} def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1211,10 +1211,12 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks - LOG.info(msg=f"Starting Playbook for (openstack_id): {openstack_id} --> {playbooks_information}") + LOG.info( + msg=f"Starting Playbook for (openstack_id): {openstack_id} --> {playbooks_information}" + ) port = self.get_vm_ports(openstack_id=openstack_id) key = self.redis.hget(openstack_id, "key").decode("utf-8") playbook = Playbook( @@ -1260,8 +1262,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1855,8 +1857,8 @@ def add_udp_security_group(self, server_id): ip_base = ( list(self.conn.compute.server_ips(server=server_id))[0] - .to_dict()["address"] - .split(".")[-1] + .to_dict()["address"] + .split(".")[-1] ) x = int(ip_base) # noqa F841 udp_port_start = eval(self.UDP_FORMULAR) @@ -1980,9 +1982,9 @@ def get_active_image_by_os_version(self, os_version, os_distro): image_os_distro = metadata.get("os_distro", None) base_image_ref = metadata.get("base_image_ref", None) if ( - os_version == image_os_version - and image.status == "active" - and base_image_ref is None + os_version == image_os_version + and image.status == "active" + and base_image_ref is None ): if os_distro and os_distro == image_os_distro: return image @@ -2001,17 +2003,17 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, ): LOG.info(f"Add machine to {cluster_id} - {key_name}") image = self.get_image(image=image) @@ -2060,7 +2062,7 @@ def add_cluster_machine( return server["id"] def scale_up_cluster( - self, cluster_id, image, flavor, count, names, start_idx, batch_index + self, cluster_id, image, flavor, count, names, start_idx, batch_index ): cluster_info = self.get_cluster_info(cluster_id=cluster_id) image = self.get_image(image=image) @@ -2344,9 +2346,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2355,7 +2357,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec.name != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec.name + and "bibigrid" not in sec.name ] if security_groups is not None: for sg in security_groups: @@ -2506,15 +2508,15 @@ def resume_server(self, openstack_id): return False def create_security_group( - self, - name, - udp_port_start=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port_start=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2658,7 +2660,9 @@ def get_limits(self): def install_ansible_galaxy_requirements(self): LOG.info("Installing Ansible galaxy requirements..") - stream = os.popen(f'ansible-galaxy install -r {PLAYBOOKS_DIR}/packer/requirements.yml') + stream = os.popen( + f"ansible-galaxy install -r {PLAYBOOKS_DIR}/packer/requirements.yml" + ) output = stream.read() LOG.info(output) @@ -2671,17 +2675,25 @@ def update_playbooks(self): LOG.info(f"STARTED update of playbooks from - {self.GITHUB_PLAYBOOKS_REPO}") r = req.get(self.GITHUB_PLAYBOOKS_REPO) filename = "resenv_repo" - with open(filename, 'wb') as output_file: + with open(filename, "wb") as output_file: output_file.write(r.content) - LOG.info('Downloading Completed') - with zipfile.ZipFile(filename, 'r') as zip_ref: + LOG.info("Downloading Completed") + with zipfile.ZipFile(filename, "r") as zip_ref: zip_ref.extractall(PLAYBOOKS_DIR) - resenvs_unziped_dir = next(filter(lambda f: os.path.isdir(f) and "resenvs" in f, glob.glob(PLAYBOOKS_DIR + '*'))) + resenvs_unziped_dir = next( + filter( + lambda f: os.path.isdir(f) and "resenvs" in f, + glob.glob(PLAYBOOKS_DIR + "*"), + ) + ) shutil.copytree(resenvs_unziped_dir, PLAYBOOKS_DIR, dirs_exist_ok=True) shutil.rmtree(resenvs_unziped_dir, ignore_errors=True) - self.ALL_TEMPLATES = [name for name in os.listdir(PLAYBOOKS_DIR) if - name != "packer" and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name))] + self.ALL_TEMPLATES = [ + name + for name in os.listdir(PLAYBOOKS_DIR) + if name != "packer" and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + ] LOG.info(self.ALL_TEMPLATES) templates_metadata = self.load_resenv_metadata() @@ -2697,8 +2709,10 @@ def update_playbooks(self): template_metadata[DIRECTION], template_metadata[PROTOCOL], template_metadata[INFORMATION_FOR_DISPLAY], - needs_forc_support=template_metadata.get(NEEDS_FORC_SUPPORT, False), - json_string=json.dumps(template_metadata) + needs_forc_support=template_metadata.get( + NEEDS_FORC_SUPPORT, False + ), + json_string=json.dumps(template_metadata), ) self.update_forc_allowed(template_metadata) if metadata.name not in list(self.loaded_resenv_metadata.keys()): @@ -2721,7 +2735,9 @@ def load_resenv_metadata(self): templates_metada = [] for template in self.ALL_TEMPLATES: try: - with open(f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml") as template_metadata: + with open( + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + ) as template_metadata: try: loaded_metadata = yaml.load( template_metadata, Loader=yaml.FullLoader @@ -2765,17 +2781,17 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, - needs_forc_support, - json_string + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, + needs_forc_support, + json_string, ): self.name = name self.port = port diff --git a/VirtualMachineService/ancon/Playbook.py b/VirtualMachineService/ancon/Playbook.py index 45951ecd..cdbef8a9 100644 --- a/VirtualMachineService/ancon/Playbook.py +++ b/VirtualMachineService/ancon/Playbook.py @@ -35,15 +35,15 @@ class Playbook(object): PLAYBOOK_FAILED = "PLAYBOOK_FAILED" def __init__( - self, - ip, - port, - playbooks_information, - osi_private_key, - public_key, - pool, - loaded_metadata_keys, - cloud_site, + self, + ip, + port, + playbooks_information, + osi_private_key, + public_key, + pool, + loaded_metadata_keys, + cloud_site, ): self.loaded_metadata_keys = loaded_metadata_keys self.cloud_site = cloud_site @@ -88,10 +88,10 @@ def __init__( ) inventory_string = ( - "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " - "ansible_ssh_private_key_file=" - + self.private_key.name - + " ansible_python_interpreter=/usr/bin/python3" + "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " + "ansible_ssh_private_key_file=" + + self.private_key.name + + " ansible_python_interpreter=/usr/bin/python3" ) self.inventory.write(inventory_string) self.inventory.close() @@ -107,12 +107,12 @@ def copy_playbooks_and_init(self, playbooks_information, public_key): self.playbooks_dir + "/change_key_vars_file.yml", self.directory.name ) with open( - self.directory.name + "/change_key_vars_file.yml", mode="r" + self.directory.name + "/change_key_vars_file.yml", mode="r" ) as key_file: data_ck = self.yaml_exec.load(key_file) data_ck["change_key_vars"]["key"] = public_key.strip('"') with open( - self.directory.name + "/change_key_vars_file.yml", mode="w" + self.directory.name + "/change_key_vars_file.yml", mode="w" ) as key_file: self.yaml_exec.dump(data_ck, key_file) self.add_to_playbook_always_lists("change_key") @@ -122,14 +122,14 @@ def copy_playbooks_and_init(self, playbooks_information, public_key): self.playbooks_dir + "/" + self.playbook_exec_name, self.directory.name ) with open( - self.directory.name + "/" + self.playbook_exec_name, mode="r" + self.directory.name + "/" + self.playbook_exec_name, mode="r" ) as generic_playbook: data_gp = self.yaml_exec.load(generic_playbook) data_gp[0]["vars_files"] = self.vars_files data_gp[0]["tasks"][0]["block"] = self.tasks data_gp[0]["tasks"][0]["always"] = self.always_tasks with open( - self.directory.name + "/" + self.playbook_exec_name, mode="w" + self.directory.name + "/" + self.playbook_exec_name, mode="w" ) as generic_playbook: self.yaml_exec.dump(data_gp, generic_playbook) @@ -161,7 +161,11 @@ def load_vars(): data[playbook_name + "_defined"][k] = v # copy whole directory - shutil.copytree(f"{self.playbooks_dir}/{playbook_name}", self.directory.name,dirs_exist_ok=True) + shutil.copytree( + f"{self.playbooks_dir}/{playbook_name}", + self.directory.name, + dirs_exist_ok=True, + ) site_specific_yml = f"/{playbook_name}{'-' + self.cloud_site}.yml" playbook_name_local = playbook_name @@ -170,14 +174,10 @@ def load_vars(): playbook_var_yml = f"/{playbook_name}_vars_file.yml" try: - with open( - self.directory.name + playbook_var_yml, mode="r" - ) as variables: + with open(self.directory.name + playbook_var_yml, mode="r") as variables: data = self.yaml_exec.load(variables) load_vars() - with open( - self.directory.name + playbook_var_yml, mode="w" - ) as variables: + with open(self.directory.name + playbook_var_yml, mode="w") as variables: self.yaml_exec.dump(data, variables) self.add_to_playbook_lists(playbook_name_local, playbook_name) except shutil.Error as e: From 4bd1ff2703eab60460c8c04873847ab98ad68caa Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 17 May 2022 12:14:57 +0000 Subject: [PATCH 08/12] feat(Dependencies): Update dependency setuptools to v62.3.1 | datasource | package | from | to | | ---------- | ---------- | ------ | ------ | | pypi | setuptools | 62.3.0 | 62.3.1 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ab4d408..c58d16e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==62.3.0 +setuptools==62.3.1 thrift==0.16.0 python-keystoneclient openstacksdk ==0.61.0 From 8416e96de7da8198ca5fb1647539a5474b5a6a61 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 18 May 2022 17:14:29 +0000 Subject: [PATCH 09/12] feat(Dependencies): Update dependency ansible to v5.8.0 | datasource | package | from | to | | ---------- | ------- | ----- | ----- | | pypi | ansible | 5.7.1 | 5.8.0 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c58d16e3..38b1b4b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ thrift==0.16.0 python-keystoneclient openstacksdk ==0.61.0 deprecated == 1.2.13 -ansible==5.7.1 +ansible==5.8.0 Click==8.1.3 flake8==4.0.1 paramiko==2.11.0 From 34fef5f01069358d2b12f247464274e41edc7669 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 18 May 2022 19:54:25 +0000 Subject: [PATCH 10/12] feat(Dependencies): Update dependency setuptools to v62.3.2 | datasource | package | from | to | | ---------- | ---------- | ------ | ------ | | pypi | setuptools | 62.3.1 | 62.3.2 | --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38b1b4b1..08195a7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==62.3.1 +setuptools==62.3.2 thrift==0.16.0 python-keystoneclient openstacksdk ==0.61.0 From dc3b2f65c1a0c181866f4bb1a3b142e5d29c8095 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Thu, 19 May 2022 15:29:02 +0200 Subject: [PATCH 11/12] fixed --- VirtualMachineService/VirtualMachineHandler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index e4cc21da..cd690217 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -2726,14 +2726,12 @@ def load_resenv_metadata(self): loaded_metadata = yaml.load( template_metadata, Loader=yaml.FullLoader ) - template_name = loaded_metadata[TEMPLATE_NAME] templates_metada.append(loaded_metadata) - if template_name not in self.ALL_TEMPLATES: - ALL_TEMPLATES.append(template_name) + except Exception as e: LOG.exception( - "Failed to parse Metadata yml: " + file + "\n" + str(e) + "Failed to parse Metadata yml: " + template_metadata + "\n" + str(e) ) except Exception as e: LOG.exception(f"No Metadatafile found for {template} - {e}") From c3ac72717f47e6dd16cba06b17dbfac88323451d Mon Sep 17 00:00:00 2001 From: dweinholz Date: Fri, 20 May 2022 06:16:04 +0000 Subject: [PATCH 12/12] fix(Linting):blacked code --- VirtualMachineService/VirtualMachineHandler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 4cf6787f..f29de78c 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -2747,7 +2747,10 @@ def load_resenv_metadata(self): except Exception as e: LOG.exception( - "Failed to parse Metadata yml: " + template_metadata + "\n" + str(e) + "Failed to parse Metadata yml: " + + template_metadata + + "\n" + + str(e) ) except Exception as e: LOG.exception(f"No Metadatafile found for {template} - {e}")