From d6749db64c14aebefe8f1255b17223a99cfe3d9f Mon Sep 17 00:00:00 2001 From: qqmok Date: Thu, 23 Sep 2021 12:49:15 +0200 Subject: [PATCH 01/16] feat(cli): updated portal client playbook to automatically install python3.8 --- plays/setup_basics.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml index e4e3bb06..3ac6ec01 100644 --- a/plays/setup_basics.yml +++ b/plays/setup_basics.yml @@ -37,6 +37,18 @@ become: no tags: always + - name: Import deadsnake PPA for python3.8 + ansible.builtin.apt_repository: + repo: ppa:deadsnakes/ppa + update_cache: yes + tags: always + + - name: Install python3.8 + apt: + state: latest + name: python=3.8 + tags: always + - name: Install python3-pip apt: state: latest From 467bd8290c153a73b47ed60a88ba7845efcd76d5 Mon Sep 17 00:00:00 2001 From: qqmok Date: Fri, 24 Sep 2021 14:44:58 +0200 Subject: [PATCH 02/16] fix(cli): fixed version number inconsistent error --- plays/setup_basics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml index 3ac6ec01..2a10e052 100644 --- a/plays/setup_basics.yml +++ b/plays/setup_basics.yml @@ -46,7 +46,7 @@ - name: Install python3.8 apt: state: latest - name: python=3.8 + name: python3.8 tags: always - name: Install python3-pip From 17f85c3c1d19457b8a76103c5e38f7a76dbb4a8e Mon Sep 17 00:00:00 2001 From: qqmok Date: Fri, 24 Sep 2021 16:32:04 +0200 Subject: [PATCH 03/16] fix(cli): moved apt-update into pre_tasks to prevent error --- plays/setup_basics.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml index e4e3bb06..dc6188f7 100644 --- a/plays/setup_basics.yml +++ b/plays/setup_basics.yml @@ -11,15 +11,13 @@ pre_tasks: - - name: Verify Ansible version. + - name: Verify Ansible version assert: that: "ansible_version.full is version_compare('2.7', '>=')" msg: > "You must update Ansible to at least 2.7"L tags: always - - tasks: - name: Update cache apt: upgrade: yes @@ -27,6 +25,8 @@ cache_valid_time: 86400 #One day tags: always + + tasks: - name: Checkout cloud-portal-client repository git: repo: "git@github.com:deNBI/cloud-portal-client.git" From 9a166acd68470561fe8a0930e658639af4067711 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:00:39 +0000 Subject: [PATCH 04/16] feat(deps): bump setuptools from 58.0.4 to 58.1.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 58.0.4 to 58.1.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst) - [Commits](https://github.com/pypa/setuptools/compare/v58.0.4...v58.1.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a484bb89..9d92d022 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==58.0.4 +setuptools==58.1.0 thrift >= 0.11.0,<0.20.0 python-keystoneclient openstacksdk ==0.59.0 From f684c8b8c21a9a3fe9e1ce9fb4176d62e4ed0cea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:00:42 +0000 Subject: [PATCH 05/16] feat(deps): bump ansible from 4.5.0 to 4.6.0 Bumps [ansible](https://github.com/ansible/ansible) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a484bb89..716ccb02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ python-keystoneclient openstacksdk ==0.59.0 deprecated == 1.2.13 Click==8.0.1 -ansible==4.5.0 +ansible==4.6.0 flake8==3.9.2 ruamel.yaml<0.18.0 paramiko==2.7.2 From b6757c11acd2f33ce34c56ffcb4b0557a613baf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:02:07 +0000 Subject: [PATCH 06/16] feat(deps): bump beats/filebeat from 7.14.1 to 7.15.0 Bumps beats/filebeat from 7.14.1 to 7.15.0. --- updated-dependencies: - dependency-name: beats/filebeat dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile.update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.update b/Dockerfile.update index 69693fde..d48ce49f 100644 --- a/Dockerfile.update +++ b/Dockerfile.update @@ -1,3 +1,3 @@ FROM denbicloud/cloud-portal-client:0.1.0-beta.0.15.1 FROM bibiserv/bibigrid:bibigrid-rest-2.3 -FROM docker.elastic.co/beats/filebeat:7.14.1 +FROM docker.elastic.co/beats/filebeat:7.15.0 From a0cc6d224bb1cfde9d2a67a296733c7943eeb75f Mon Sep 17 00:00:00 2001 From: vktrrdk Date: Wed, 29 Sep 2021 12:44:10 +0200 Subject: [PATCH 07/16] fix(Security Groups): Adjusted set security groups for FORC --- README.md | 4 +++- VirtualMachineService/VirtualMachineHandler.py | 15 +++++++-------- VirtualMachineService/config/config.yml | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0b534e15..f67682a4 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ fill in the missing parameters. When starting with commandline you will need to export some of them manually. #### Security Groups -The client expects a security group with the name "defaultSimpleVM" to exist which will be assigned to each machine at startup. Also, each machine will have its own security group when it starts. +The config file contains a name for the default SimpleVM security group. +It can be configured via the `default_simple_vm_security_group_name` key. +The client will set this group for every SimpleVM machine. Also, each machine will have its own security group when it starts. ##### Gateway diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 4b92a571..f46dfaec 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -106,8 +106,6 @@ class VirtualMachineHandler(Iface): PREPARE_PLAYBOOK_BUILD = "PREPARE_PLAYBOOK_BUILD" BUILD_PLAYBOOK = "BUILD_PLAYBOOK" PLAYBOOK_FAILED = "PLAYBOOK_FAILED" - DEFAULT_SECURITY_GROUP = "defaultSimpleVM" - DEFAULT_SECURITY_GROUPS = [DEFAULT_SECURITY_GROUP] ALL_TEMPLATES = ALL_TEMPLATES loaded_resenv_metadata = {} @@ -162,8 +160,11 @@ def __init__(self, config): self.PROJECT_DOMAIN_ID = os.environ["OS_PROJECT_DOMAIN_ID"] self.SSH_PORT = 22 + with open(config, "r") as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) + self.DEFAULT_SECURITY_GROUP_NAME = cfg["openstack_connection"]["default_simple_vm_security_group_name"] + self.DEFAULT_SECURITY_GROUPS = [self.DEFAULT_SECURITY_GROUP_NAME] self.USE_GATEWAY = cfg["openstack_connection"]["use_gateway"] self.NETWORK = cfg["openstack_connection"]["network"] self.FLOATING_IP_NETWORK = cfg["openstack_connection"][ @@ -221,6 +222,7 @@ def __init__(self, config): self.RE_BACKEND_URL = cfg["forc"]["forc_url"] self.FORC_API_KEY = cfg["forc"]["forc_api_key"] self.FORC_ALLOWED = {} + self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( not self.RE_BACKEND_URL @@ -992,11 +994,6 @@ def volume_ids( def prepare_security_groups_new_server(self, resenv, servername, http, https): custom_security_groups = [] - custom_security_groups.append( - self.create_security_group( - name=servername + "_ssh", description="Only SSH" - ).name - ) if http or https: custom_security_groups.append( @@ -2365,7 +2362,7 @@ def delete_server(self, openstack_id): security_groups = [ sec for sec in security_groups - if sec.name != self.DEFAULT_SECURITY_GROUP + if sec.name != self.DEFAULT_SECURITY_GROUP_NAME and not "bibigrid" in sec.name ] if security_groups is not None: @@ -2627,6 +2624,8 @@ def create_security_group( port_range_max=resenv_metadata.port, port_range_min=resenv_metadata.port, security_group_id=new_security_group["id"], + remote_group_id=self.FORC_REMOTE_ID, + ) elif research_enviroment != "user_key_url": # Todo add mail for this logging as this should not happen diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 745ea5fb..37166e95 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -4,6 +4,7 @@ redis: password: "" openstack_connection: + default_simple_vm_security_group_name: "defaultSimpleVM" threads: 30 host: 0.0.0.0 # Client Port @@ -48,5 +49,6 @@ forc: forc_url: forc_api_key: github_playbooks_repo: + forc_remote_id: cloud_site: bielefeld From 6cdd13189ef52d88abefbb8348d6428bde8569f5 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Wed, 29 Sep 2021 14:04:48 +0200 Subject: [PATCH 08/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f67682a4..e66422fa 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ When starting with commandline you will need to export some of them manually. #### Security Groups The config file contains a name for the default SimpleVM security group. It can be configured via the `default_simple_vm_security_group_name` key. -The client will set this group for every SimpleVM machine. Also, each machine will have its own security group when it starts. +The client will set this group for every SimpleVM machine. ##### Gateway From 188eed8a10356d3fca0dbcae8b587acc4d981e9a Mon Sep 17 00:00:00 2001 From: dweinholz Date: Wed, 29 Sep 2021 12:05:48 +0000 Subject: [PATCH 09/16] fix(Linting):blacked code --- VirtualMachineService/VirtualMachineHandler.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index f46dfaec..5d27a659 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -160,10 +160,11 @@ def __init__(self, config): self.PROJECT_DOMAIN_ID = os.environ["OS_PROJECT_DOMAIN_ID"] self.SSH_PORT = 22 - with open(config, "r") as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) - self.DEFAULT_SECURITY_GROUP_NAME = cfg["openstack_connection"]["default_simple_vm_security_group_name"] + self.DEFAULT_SECURITY_GROUP_NAME = cfg["openstack_connection"][ + "default_simple_vm_security_group_name" + ] self.DEFAULT_SECURITY_GROUPS = [self.DEFAULT_SECURITY_GROUP_NAME] self.USE_GATEWAY = cfg["openstack_connection"]["use_gateway"] self.NETWORK = cfg["openstack_connection"]["network"] @@ -994,7 +995,6 @@ def volume_ids( def prepare_security_groups_new_server(self, resenv, servername, http, https): custom_security_groups = [] - if http or https: custom_security_groups.append( self.create_security_group( @@ -2625,7 +2625,6 @@ def create_security_group( port_range_min=resenv_metadata.port, security_group_id=new_security_group["id"], remote_group_id=self.FORC_REMOTE_ID, - ) elif research_enviroment != "user_key_url": # Todo add mail for this logging as this should not happen From 4c177c33efc606596d00ad858f47e5e75ca33880 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 12:00:37 +0000 Subject: [PATCH 10/16] feat(deps): bump setuptools from 58.1.0 to 58.2.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 58.1.0 to 58.2.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst) - [Commits](https://github.com/pypa/setuptools/compare/v58.1.0...v58.2.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 63405702..7ec47180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==58.1.0 +setuptools==58.2.0 thrift >= 0.11.0,<0.20.0 python-keystoneclient openstacksdk ==0.59.0 From bec7a288adeb1a054ee8173b06fdade147bb146d Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 5 Oct 2021 09:47:47 +0000 Subject: [PATCH 11/16] feat(Dockerfile):updated dependencies --- .env.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.in b/.env.in index bff64806..9b5b5187 100644 --- a/.env.in +++ b/.env.in @@ -13,7 +13,7 @@ OS_PASSWORD= # Filebeat -FILEBEAT_TAG=7.14.1 +FILEBEAT_TAG=7.15.0 ELASTIC_USER= ELASTIC_PASSWORD= HOST=portal-dev.denbi.de From cd4e6fde03a4b81aa20bde9a2f6dec83d05fe240 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 5 Oct 2021 12:58:38 +0200 Subject: [PATCH 12/16] Update .env.in --- .env.in | 1 - 1 file changed, 1 deletion(-) diff --git a/.env.in b/.env.in index 9b5b5187..26d28cd3 100644 --- a/.env.in +++ b/.env.in @@ -11,7 +11,6 @@ OS_USER_DOMAIN_NAME= OS_PROJECT_DOMAIN_ID= OS_PASSWORD= - # Filebeat FILEBEAT_TAG=7.15.0 ELASTIC_USER= From cb338b1ee321519ba4d6e5043f328db692d23ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 12:00:35 +0000 Subject: [PATCH 13/16] feat(deps): bump paramiko from 2.7.2 to 2.8.0 Bumps [paramiko](https://github.com/paramiko/paramiko) from 2.7.2 to 2.8.0. - [Release notes](https://github.com/paramiko/paramiko/releases) - [Changelog](https://github.com/paramiko/paramiko/blob/main/NEWS) - [Commits](https://github.com/paramiko/paramiko/compare/2.7.2...2.8.0) --- updated-dependencies: - dependency-name: paramiko dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ec47180..1d5a2d34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ Click==8.0.1 ansible==4.6.0 flake8==3.9.2 ruamel.yaml<0.18.0 -paramiko==2.7.2 +paramiko==2.8.0 pyvim==3.0.2 redis==3.5.3 requests==2.26.0 From c35f88f1cc70f3541b55989be8d9ff10a03ddf12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 12:00:38 +0000 Subject: [PATCH 14/16] feat(deps): bump click from 8.0.1 to 8.0.3 Bumps [click](https://github.com/pallets/click) from 8.0.1 to 8.0.3. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.0.1...8.0.3) --- updated-dependencies: - dependency-name: click dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ec47180..a5c7b31f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ thrift >= 0.11.0,<0.20.0 python-keystoneclient openstacksdk ==0.59.0 deprecated == 1.2.13 -Click==8.0.1 +Click==8.0.3 ansible==4.6.0 flake8==3.9.2 ruamel.yaml<0.18.0 From 23ff11664459026c458f3c78866ba280fc20db52 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 12 Oct 2021 09:40:22 +0200 Subject: [PATCH 15/16] fix(Forc):API key to secrets --- .env.in | 1 + .../VirtualMachineHandler.py | 200 +++++++++--------- VirtualMachineService/VirtualMachineServer.py | 2 + VirtualMachineService/config/config.yml | 1 - 4 files changed, 103 insertions(+), 101 deletions(-) diff --git a/.env.in b/.env.in index bff64806..c680d987 100644 --- a/.env.in +++ b/.env.in @@ -1,6 +1,7 @@ # Client and Bibigrid CLOUD_PORTAL_CLIENT_TAG=0.1.0-beta.0.15.1 BIBIGRID_TAG=bibigrid-rest-2.3 +FORC_API_KEY= CLIENT_PORT= OS_AUTH_URL= diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 5d27a659..ce02df90 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -221,14 +221,14 @@ def __init__(self, config): try: self.RE_BACKEND_URL = cfg["forc"]["forc_url"] - self.FORC_API_KEY = cfg["forc"]["forc_api_key"] + self.FORC_API_KEY = os.environ.get("FORC_API_KEY", None) self.FORC_ALLOWED = {} self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg="Forc-Backend url loaded: {0}".format(self.RE_BACKEND_URL)) @@ -372,10 +372,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): metadata = img["metadata"] @@ -383,7 +383,7 @@ def get_Images(self): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) continue @@ -419,7 +419,7 @@ def prepare_image(self, img): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) return None @@ -455,11 +455,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -481,11 +481,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -538,10 +538,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags") if "resenv" in filter_list: @@ -793,7 +793,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info("create init script for volume ids:{}".format(volume_ids_path_new)) if not volume_ids_path_new and not volume_ids_path_attach: @@ -942,17 +942,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -1025,18 +1025,18 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https): return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1085,9 +1085,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1120,17 +1120,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1184,16 +1184,16 @@ def start_server( return {} def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1269,7 +1269,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info(msg="Starting Playbook for (openstack_id): {0}".format(openstack_id)) @@ -1318,8 +1318,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1952,8 +1952,8 @@ def add_udp_security_group(self, server_id): ip_base = ( list(self.conn.compute.server_ips(server=server_id))[0] - .to_dict()["address"] - .split(".")[-1] + .to_dict()["address"] + .split(".")[-1] ) x = int(ip_base) udp_port_start = eval(self.UDP_PORT_CALCULATION) @@ -2076,7 +2076,7 @@ def get_clusters_info(self): return infos def scale_up_cluster( - self, cluster_id, image, flavor, count, names, start_idx, batch_index + self, cluster_id, image, flavor, count, names, start_idx, batch_index ): cluster_info = self.get_cluster_info(cluster_id=cluster_id) image = self.get_image(image=image) @@ -2352,9 +2352,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2363,7 +2363,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec.name != self.DEFAULT_SECURITY_GROUP_NAME - and not "bibigrid" in sec.name + and not "bibigrid" in sec.name ] if security_groups is not None: for sg in security_groups: @@ -2516,15 +2516,15 @@ def resume_server(self, openstack_id): return False def create_security_group( - self, - name, - udp_port_start=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port_start=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info("Create new security group {}".format(name)) sec = self.conn.get_security_group(name_or_id=name) @@ -2758,15 +2758,15 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, ): self.name = name self.port = port diff --git a/VirtualMachineService/VirtualMachineServer.py b/VirtualMachineService/VirtualMachineServer.py index ac2c65e1..2c7f4217 100644 --- a/VirtualMachineService/VirtualMachineServer.py +++ b/VirtualMachineService/VirtualMachineServer.py @@ -27,6 +27,7 @@ USER_DOMAIN_ID = "OS_USER_DOMAIN_NAME" AUTH_URL = "OS_AUTH_URL" PROJECT_DOMAIN_ID = "OS_PROJECT_DOMAIN_ID" +FORC_API_KEY= "FORC_API_KEY" environment_variables = [ USERNAME, @@ -36,6 +37,7 @@ USER_DOMAIN_ID, AUTH_URL, PROJECT_DOMAIN_ID, + FORC_API_KEY ] diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 37166e95..39f2ce7f 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -47,7 +47,6 @@ bibigrid: forc: forc_url: - forc_api_key: github_playbooks_repo: forc_remote_id: From a300f24b2f0fef29434f95fb212e6a4becbba225 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Tue, 12 Oct 2021 12:18:57 +0200 Subject: [PATCH 16/16] Update publish_docker.yml --- .github/workflows/publish_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index e6d205f3..a9a4371d 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -8,7 +8,7 @@ on: - 'dev' jobs: build: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Workflow run cleanup action