diff --git a/.env.in b/.env.in index bff64806..5f86177b 100644 --- a/.env.in +++ b/.env.in @@ -1,6 +1,7 @@ # Client and Bibigrid CLOUD_PORTAL_CLIENT_TAG=0.1.0-beta.0.15.1 BIBIGRID_TAG=bibigrid-rest-2.3 +FORC_API_KEY= CLIENT_PORT= OS_AUTH_URL= @@ -11,9 +12,8 @@ OS_USER_DOMAIN_NAME= OS_PROJECT_DOMAIN_ID= OS_PASSWORD= - # Filebeat -FILEBEAT_TAG=7.14.1 +FILEBEAT_TAG=7.15.0 ELASTIC_USER= ELASTIC_PASSWORD= HOST=portal-dev.denbi.de diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index e6d205f3..a9a4371d 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -8,7 +8,7 @@ on: - 'dev' jobs: build: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Workflow run cleanup action diff --git a/Dockerfile.update b/Dockerfile.update index 69693fde..d48ce49f 100644 --- a/Dockerfile.update +++ b/Dockerfile.update @@ -1,3 +1,3 @@ FROM denbicloud/cloud-portal-client:0.1.0-beta.0.15.1 FROM bibiserv/bibigrid:bibigrid-rest-2.3 -FROM docker.elastic.co/beats/filebeat:7.14.1 +FROM docker.elastic.co/beats/filebeat:7.15.0 diff --git a/README.md b/README.md index 0b534e15..e66422fa 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ fill in the missing parameters. When starting with commandline you will need to export some of them manually. #### Security Groups -The client expects a security group with the name "defaultSimpleVM" to exist which will be assigned to each machine at startup. Also, each machine will have its own security group when it starts. +The config file contains a name for the default SimpleVM security group. +It can be configured via the `default_simple_vm_security_group_name` key. +The client will set this group for every SimpleVM machine. ##### Gateway diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 4b92a571..ce02df90 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -106,8 +106,6 @@ class VirtualMachineHandler(Iface): PREPARE_PLAYBOOK_BUILD = "PREPARE_PLAYBOOK_BUILD" BUILD_PLAYBOOK = "BUILD_PLAYBOOK" PLAYBOOK_FAILED = "PLAYBOOK_FAILED" - DEFAULT_SECURITY_GROUP = "defaultSimpleVM" - DEFAULT_SECURITY_GROUPS = [DEFAULT_SECURITY_GROUP] ALL_TEMPLATES = ALL_TEMPLATES loaded_resenv_metadata = {} @@ -164,6 +162,10 @@ def __init__(self, config): with open(config, "r") as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) + self.DEFAULT_SECURITY_GROUP_NAME = cfg["openstack_connection"][ + "default_simple_vm_security_group_name" + ] + self.DEFAULT_SECURITY_GROUPS = [self.DEFAULT_SECURITY_GROUP_NAME] self.USE_GATEWAY = cfg["openstack_connection"]["use_gateway"] self.NETWORK = cfg["openstack_connection"]["network"] self.FLOATING_IP_NETWORK = cfg["openstack_connection"][ @@ -219,13 +221,14 @@ def __init__(self, config): try: self.RE_BACKEND_URL = cfg["forc"]["forc_url"] - self.FORC_API_KEY = cfg["forc"]["forc_api_key"] + self.FORC_API_KEY = os.environ.get("FORC_API_KEY", None) self.FORC_ALLOWED = {} + self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg="Forc-Backend url loaded: {0}".format(self.RE_BACKEND_URL)) @@ -369,10 +372,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): metadata = img["metadata"] @@ -380,7 +383,7 @@ def get_Images(self): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) continue @@ -416,7 +419,7 @@ def prepare_image(self, img): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) return None @@ -452,11 +455,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -478,11 +481,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -535,10 +538,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags") if "resenv" in filter_list: @@ -790,7 +793,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info("create init script for volume ids:{}".format(volume_ids_path_new)) if not volume_ids_path_new and not volume_ids_path_attach: @@ -939,17 +942,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -992,12 +995,6 @@ def volume_ids( def prepare_security_groups_new_server(self, resenv, servername, http, https): custom_security_groups = [] - custom_security_groups.append( - self.create_security_group( - name=servername + "_ssh", description="Only SSH" - ).name - ) - if http or https: custom_security_groups.append( self.create_security_group( @@ -1028,18 +1025,18 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https): return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1088,9 +1085,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1123,17 +1120,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1187,16 +1184,16 @@ def start_server( return {} def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1272,7 +1269,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info(msg="Starting Playbook for (openstack_id): {0}".format(openstack_id)) @@ -1321,8 +1318,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1955,8 +1952,8 @@ def add_udp_security_group(self, server_id): ip_base = ( list(self.conn.compute.server_ips(server=server_id))[0] - .to_dict()["address"] - .split(".")[-1] + .to_dict()["address"] + .split(".")[-1] ) x = int(ip_base) udp_port_start = eval(self.UDP_PORT_CALCULATION) @@ -2079,7 +2076,7 @@ def get_clusters_info(self): return infos def scale_up_cluster( - self, cluster_id, image, flavor, count, names, start_idx, batch_index + self, cluster_id, image, flavor, count, names, start_idx, batch_index ): cluster_info = self.get_cluster_info(cluster_id=cluster_id) image = self.get_image(image=image) @@ -2355,9 +2352,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2365,8 +2362,8 @@ def delete_server(self, openstack_id): security_groups = [ sec for sec in security_groups - if sec.name != self.DEFAULT_SECURITY_GROUP - and not "bibigrid" in sec.name + if sec.name != self.DEFAULT_SECURITY_GROUP_NAME + and not "bibigrid" in sec.name ] if security_groups is not None: for sg in security_groups: @@ -2519,15 +2516,15 @@ def resume_server(self, openstack_id): return False def create_security_group( - self, - name, - udp_port_start=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port_start=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info("Create new security group {}".format(name)) sec = self.conn.get_security_group(name_or_id=name) @@ -2627,6 +2624,7 @@ def create_security_group( port_range_max=resenv_metadata.port, port_range_min=resenv_metadata.port, security_group_id=new_security_group["id"], + remote_group_id=self.FORC_REMOTE_ID, ) elif research_enviroment != "user_key_url": # Todo add mail for this logging as this should not happen @@ -2760,15 +2758,15 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, ): self.name = name self.port = port diff --git a/VirtualMachineService/VirtualMachineServer.py b/VirtualMachineService/VirtualMachineServer.py index ac2c65e1..2c7f4217 100644 --- a/VirtualMachineService/VirtualMachineServer.py +++ b/VirtualMachineService/VirtualMachineServer.py @@ -27,6 +27,7 @@ USER_DOMAIN_ID = "OS_USER_DOMAIN_NAME" AUTH_URL = "OS_AUTH_URL" PROJECT_DOMAIN_ID = "OS_PROJECT_DOMAIN_ID" +FORC_API_KEY= "FORC_API_KEY" environment_variables = [ USERNAME, @@ -36,6 +37,7 @@ USER_DOMAIN_ID, AUTH_URL, PROJECT_DOMAIN_ID, + FORC_API_KEY ] diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 745ea5fb..39f2ce7f 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -4,6 +4,7 @@ redis: password: "" openstack_connection: + default_simple_vm_security_group_name: "defaultSimpleVM" threads: 30 host: 0.0.0.0 # Client Port @@ -46,7 +47,7 @@ bibigrid: forc: forc_url: - forc_api_key: github_playbooks_repo: + forc_remote_id: cloud_site: bielefeld diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml index e4e3bb06..7ceb770f 100644 --- a/plays/setup_basics.yml +++ b/plays/setup_basics.yml @@ -11,15 +11,13 @@ pre_tasks: - - name: Verify Ansible version. + - name: Verify Ansible version assert: that: "ansible_version.full is version_compare('2.7', '>=')" msg: > "You must update Ansible to at least 2.7"L tags: always - - tasks: - name: Update cache apt: upgrade: yes @@ -27,6 +25,8 @@ cache_valid_time: 86400 #One day tags: always + + tasks: - name: Checkout cloud-portal-client repository git: repo: "git@github.com:deNBI/cloud-portal-client.git" @@ -37,6 +37,18 @@ become: no tags: always + - name: Import deadsnake PPA for python3.8 + ansible.builtin.apt_repository: + repo: ppa:deadsnakes/ppa + update_cache: yes + tags: always + + - name: Install python3.8 + apt: + state: latest + name: python3.8 + tags: always + - name: Install python3-pip apt: state: latest diff --git a/requirements.txt b/requirements.txt index a484bb89..61233210 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -setuptools==58.0.4 +setuptools==58.2.0 thrift >= 0.11.0,<0.20.0 python-keystoneclient openstacksdk ==0.59.0 deprecated == 1.2.13 -Click==8.0.1 -ansible==4.5.0 +Click==8.0.3 +ansible==4.6.0 flake8==3.9.2 ruamel.yaml<0.18.0 -paramiko==2.7.2 +paramiko==2.8.0 pyvim==3.0.2 redis==3.5.3 requests==2.26.0