diff --git a/.github/workflows/blacked.yml b/.github/workflows/blacked.yml index aedd6e57..1189e65c 100644 --- a/.github/workflows/blacked.yml +++ b/.github/workflows/blacked.yml @@ -17,7 +17,7 @@ jobs: - name: Create Pull Request if: steps.git-check.outputs.modified == 'true' id: cpr - uses: peter-evans/create-pull-request@v4.0.3 + uses: peter-evans/create-pull-request@v4.0.4 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: fix(Linting):blacked code diff --git a/Dockerfile b/Dockerfile index 4a1c430b..7a533ee3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.4-buster +FROM python:3.10.5-buster RUN apt-get update -y RUN apt-get install -y build-essential WORKDIR /code diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 73f5785f..d7ae8d88 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -5,10 +5,8 @@ """ import math import sys -from uuid import uuid4 -from pathlib import Path import zipfile -import shutil +from uuid import uuid4 try: from ancon.Playbook import ALL_TEMPLATES, Playbook @@ -51,15 +49,16 @@ ) import datetime +import glob import json import logging import os +import shutil import socket import urllib from contextlib import closing from distutils.version import LooseVersion -import glob -import shutil + import redis import requests as req import yaml @@ -393,16 +392,19 @@ def get_Images(self): self.conn.list_images(), ): - metadata = img["metadata"] - description = metadata.get("description") - tags = img.get("tags") + properties = img.get("properties") + if not properties: + properties = {} + LOG.warning(f"Could not get properties for img: {img}") + description = properties.get("description", "") + tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue - image_type = img.get("image_type", "image") + image_type = properties.get("image_type", "image") if description is None: LOG.warning("No Description and for " + img["name"]) @@ -429,16 +431,19 @@ def get_Images(self): def prepare_image(self, img): try: - metadata = img["metadata"] - description = metadata.get("description") - tags = img.get("tags") + properties = img.get("properties") + if not properties: + properties = {} + LOG.warning(f"No properties found in image: {img}") + description = properties.get("description", "") + tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None - image_type = img.get("image_type", "image") + image_type = properties.get("image_type", "image") if description is None: LOG.warning("No Description for " + img["name"]) @@ -524,9 +529,12 @@ def get_Image_with_Tag(self, id): img = self.conn.get_image(name_or_id=id) if not img: return Image() - metadata = img["metadata"] - description = metadata.get("description") - tags = img.get("tags") + properties = img.get("properties") + if not properties: + properties = {} + LOG.warning(f"Could not get properties for image: {img}") + description = properties.get("description", "") + tags = img.get("tags", []) image = Image( name=img["name"], min_disk=img["min_disk"], @@ -558,15 +566,18 @@ def get_Images_by_filter(self, filter_list): and x["status"] == "active", self.conn.list_images(), ): - tags = img.get("tags") + tags = img.get("tags", []) if "resenv" in filter_list: modes = filter_list["resenv"].split(",") LOG.info(modes) if "resenv" in tags and not self.cross_check_forc_image(tags): continue - metadata = img["metadata"] - description = metadata.get("description") - image_type = img.get("image_type", "image") + properties = img.get("properties") + if not properties: + properties = {} + LOG.warning(f"Could not get properties for img: {img}") + description = properties.get("description", "") + image_type = properties.get("image_type", "image") if description is None: LOG.warning(f"No Description for {img['name']}") @@ -625,23 +636,22 @@ def import_keypair(self, keyname, public_key): LOG.exception(f"Import Keypair {keyname} error:{e}") return None - def openstack_flav_to_thrift_flav(self, flavor): + @staticmethod + def openstack_flav_to_thrift_flav(flavor): try: - if "id" in flavor: - flavor = self.conn.compute.get_flavor(flavor["id"]).to_dict() + if "name" in flavor: name = flavor["name"] - openstack_id = flavor["id"] - else: - # Giessen + elif "original_name" in flavor: name = flavor["original_name"] - openstack_id = None + else: + name = "NoNameFound" flav = Flavor( vcpus=flavor["vcpus"], ram=flavor["ram"], disk=flavor["disk"], name=name, - openstack_id=openstack_id, + openstack_id=None, ) return flav except Exception as e: @@ -1450,11 +1460,10 @@ def delete_backend(self, id): LOG.exception(e) return str(-1) - def add_user_to_backend(self, backend_id, owner_id, user_id): + def add_user_to_backend(self, backend_id, user_id): try: post_url = f"{self.RE_BACKEND_URL}{self.USERS_URL}/{backend_id}" user_info = { - "owner": owner_id, "user": user_id, } except Exception as e: @@ -1503,10 +1512,9 @@ def get_users_from_backend(self, backend_id): LOG.info(msg=f"Get users for backend timed out. {e}") return [] - def delete_user_from_backend(self, backend_id, owner_id, user_id): + def delete_user_from_backend(self, backend_id, user_id): delete_url = f"{self.RE_BACKEND_URL}{self.USERS_URL}/{backend_id}" user_info = { - "owner": owner_id, "user": user_id, } try: @@ -1628,7 +1636,7 @@ def get_volume(self, volume_id): os_volume = self.conn.get_volume_by_id(id=volume_id) LOG.info(os_volume) if os_volume.attachments: - device = os_volume.attachments[0].device + device = os_volume.attachments[0]["device"] else: device = None @@ -1770,7 +1778,6 @@ def openstack_server_to_thrift_server(self, server: Server) -> VM: ] except Exception as e: LOG.exception(f"Could not found volume {volume_id}: {e}") - flav = self.openstack_flav_to_thrift_flav(server["flavor"]) try: @@ -2368,12 +2375,12 @@ def delete_server(self, openstack_id): security_groups = [ sec for sec in security_groups - if sec.name != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec.name + if sec["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and "bibigrid" not in sec["name"] ] if security_groups is not None: for sg in security_groups: - LOG.info(f"Delete security group {sg.name}") + LOG.info(f"Delete security group {sg['name']}") self.conn.compute.remove_security_group_from_server( server=server, security_group=sg ) @@ -2654,7 +2661,8 @@ def get_limits(self): 'totalInstancesUsed': totalFlInstancesUsed} """ LOG.info("Get Limits") - limits = self.conn.get_compute_limits() + limits = {} + limits.update(self.conn.get_compute_limits()) limits.update(self.conn.get_volume_limits()["absolute"]) return { diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index 1f6a36a1..ca73d5f4 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -80,11 +80,9 @@ if len(sys.argv) <= 1 or sys.argv[1] == "--help": print(" get_backends_by_template(string template)") print(" Backend get_backend_by_id(i64 id)") print(" string delete_backend(i64 id)") - print(" add_user_to_backend(i64 backend_id, string owner_id, string user_id)") + print(" add_user_to_backend(i64 backend_id, string user_id)") print(" get_users_from_backend(i64 backend_id)") - print( - " delete_user_from_backend(i64 backend_id, string owner_id, string user_id)" - ) + print(" delete_user_from_backend(i64 backend_id, string user_id)") print(" get_templates()") print(" get_allowed_templates()") print(" get_templates_by_template(string template_name)") @@ -585,14 +583,13 @@ elif cmd == "delete_backend": ) elif cmd == "add_user_to_backend": - if len(args) != 3: - print("add_user_to_backend requires 3 args") + if len(args) != 2: + print("add_user_to_backend requires 2 args") sys.exit(1) pp.pprint( client.add_user_to_backend( eval(args[0]), args[1], - args[2], ) ) @@ -607,14 +604,13 @@ elif cmd == "get_users_from_backend": ) elif cmd == "delete_user_from_backend": - if len(args) != 3: - print("delete_user_from_backend requires 3 args") + if len(args) != 2: + print("delete_user_from_backend requires 2 args") sys.exit(1) pp.pprint( client.delete_user_from_backend( eval(args[0]), args[1], - args[2], ) ) diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index fb367e61..3a26a91c 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -414,13 +414,12 @@ def delete_backend(self, id): """ - def add_user_to_backend(self, backend_id, owner_id, user_id): + def add_user_to_backend(self, backend_id, user_id): """ Add a user to a backend Parameters: - backend_id - - owner_id - user_id """ @@ -434,13 +433,12 @@ def get_users_from_backend(self, backend_id): """ - def delete_user_from_backend(self, backend_id, owner_id, user_id): + def delete_user_from_backend(self, backend_id, user_id): """ Delete user from a backend Parameters: - backend_id - - owner_id - user_id """ @@ -2400,26 +2398,24 @@ def recv_delete_backend(self): "delete_backend failed: unknown result", ) - def add_user_to_backend(self, backend_id, owner_id, user_id): + def add_user_to_backend(self, backend_id, user_id): """ Add a user to a backend Parameters: - backend_id - - owner_id - user_id """ - self.send_add_user_to_backend(backend_id, owner_id, user_id) + self.send_add_user_to_backend(backend_id, user_id) return self.recv_add_user_to_backend() - def send_add_user_to_backend(self, backend_id, owner_id, user_id): + def send_add_user_to_backend(self, backend_id, user_id): self._oprot.writeMessageBegin( "add_user_to_backend", TMessageType.CALL, self._seqid ) args = add_user_to_backend_args() args.backend_id = backend_id - args.owner_id = owner_id args.user_id = user_id args.write(self._oprot) self._oprot.writeMessageEnd() @@ -2482,26 +2478,24 @@ def recv_get_users_from_backend(self): "get_users_from_backend failed: unknown result", ) - def delete_user_from_backend(self, backend_id, owner_id, user_id): + def delete_user_from_backend(self, backend_id, user_id): """ Delete user from a backend Parameters: - backend_id - - owner_id - user_id """ - self.send_delete_user_from_backend(backend_id, owner_id, user_id) + self.send_delete_user_from_backend(backend_id, user_id) return self.recv_delete_user_from_backend() - def send_delete_user_from_backend(self, backend_id, owner_id, user_id): + def send_delete_user_from_backend(self, backend_id, user_id): self._oprot.writeMessageBegin( "delete_user_from_backend", TMessageType.CALL, self._seqid ) args = delete_user_from_backend_args() args.backend_id = backend_id - args.owner_id = owner_id args.user_id = user_id args.write(self._oprot) self._oprot.writeMessageEnd() @@ -4936,7 +4930,7 @@ def process_add_user_to_backend(self, seqid, iprot, oprot): result = add_user_to_backend_result() try: result.success = self._handler.add_user_to_backend( - args.backend_id, args.owner_id, args.user_id + args.backend_id, args.user_id ) msg_type = TMessageType.REPLY except TTransport.TTransportException: @@ -4988,7 +4982,7 @@ def process_delete_user_from_backend(self, seqid, iprot, oprot): result = delete_user_from_backend_result() try: result.success = self._handler.delete_user_from_backend( - args.backend_id, args.owner_id, args.user_id + args.backend_id, args.user_id ) msg_type = TMessageType.REPLY except TTransport.TTransportException: @@ -13928,7 +13922,6 @@ class add_user_to_backend_args(object): """ Attributes: - backend_id - - owner_id - user_id """ @@ -13936,11 +13929,9 @@ class add_user_to_backend_args(object): def __init__( self, backend_id=None, - owner_id=None, user_id=None, ): self.backend_id = backend_id - self.owner_id = owner_id self.user_id = user_id def read(self, iprot): @@ -13962,15 +13953,6 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 2: - if ftype == TType.STRING: - self.owner_id = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 3: if ftype == TType.STRING: self.user_id = ( iprot.readString().decode("utf-8", errors="replace") @@ -13995,16 +13977,8 @@ def write(self, oprot): oprot.writeFieldBegin("backend_id", TType.I64, 1) oprot.writeI64(self.backend_id) oprot.writeFieldEnd() - if self.owner_id is not None: - oprot.writeFieldBegin("owner_id", TType.STRING, 2) - oprot.writeString( - self.owner_id.encode("utf-8") - if sys.version_info[0] == 2 - else self.owner_id - ) - oprot.writeFieldEnd() if self.user_id is not None: - oprot.writeFieldBegin("user_id", TType.STRING, 3) + oprot.writeFieldBegin("user_id", TType.STRING, 2) oprot.writeString( self.user_id.encode("utf-8") if sys.version_info[0] == 2 @@ -14041,17 +14015,10 @@ def __ne__(self, other): ( 2, TType.STRING, - "owner_id", - "UTF8", - None, - ), # 2 - ( - 3, - TType.STRING, "user_id", "UTF8", None, - ), # 3 + ), # 2 ) @@ -14324,7 +14291,6 @@ class delete_user_from_backend_args(object): """ Attributes: - backend_id - - owner_id - user_id """ @@ -14332,11 +14298,9 @@ class delete_user_from_backend_args(object): def __init__( self, backend_id=None, - owner_id=None, user_id=None, ): self.backend_id = backend_id - self.owner_id = owner_id self.user_id = user_id def read(self, iprot): @@ -14358,15 +14322,6 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 2: - if ftype == TType.STRING: - self.owner_id = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 3: if ftype == TType.STRING: self.user_id = ( iprot.readString().decode("utf-8", errors="replace") @@ -14391,16 +14346,8 @@ def write(self, oprot): oprot.writeFieldBegin("backend_id", TType.I64, 1) oprot.writeI64(self.backend_id) oprot.writeFieldEnd() - if self.owner_id is not None: - oprot.writeFieldBegin("owner_id", TType.STRING, 2) - oprot.writeString( - self.owner_id.encode("utf-8") - if sys.version_info[0] == 2 - else self.owner_id - ) - oprot.writeFieldEnd() if self.user_id is not None: - oprot.writeFieldBegin("user_id", TType.STRING, 3) + oprot.writeFieldBegin("user_id", TType.STRING, 2) oprot.writeString( self.user_id.encode("utf-8") if sys.version_info[0] == 2 @@ -14437,17 +14384,10 @@ def __ne__(self, other): ( 2, TType.STRING, - "owner_id", - "UTF8", - None, - ), # 2 - ( - 3, - TType.STRING, "user_id", "UTF8", None, - ), # 3 + ), # 2 ) diff --git a/VirtualMachineService/ancon/playbooks/optional.yml b/VirtualMachineService/ancon/playbooks/optional/optional.yml similarity index 100% rename from VirtualMachineService/ancon/playbooks/optional.yml rename to VirtualMachineService/ancon/playbooks/optional/optional.yml diff --git a/VirtualMachineService/ancon/playbooks/optional_vars_file.yml b/VirtualMachineService/ancon/playbooks/optional/optional_vars_file.yml similarity index 100% rename from VirtualMachineService/ancon/playbooks/optional_vars_file.yml rename to VirtualMachineService/ancon/playbooks/optional/optional_vars_file.yml diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index 6a3dbeb6..7e1b183a 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -17,7 +17,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.0 + image: redis:7.0.2 expose: - "6379" networks: @@ -40,7 +40,7 @@ services: # filebeat filebeat: - image: docker.elastic.co/beats/filebeat:8.2.2 + image: docker.elastic.co/beats/filebeat:8.2.3 env_file: - .env volumes: diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index eafbff65..a48557e2 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.0 + image: redis:7.0.2 expose: - "6379" networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 92f50977..70361933 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.0 + image: redis:7.0.2 expose: - "6379" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 3bd8b8b9..68021b4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.0 + image: redis:7.0.2 expose: - "6379" networks: @@ -26,7 +26,7 @@ services: # filebeat filebeat: container_name: client_filebeat - image: docker.elastic.co/beats/filebeat:8.2.2 + image: docker.elastic.co/beats/filebeat:8.2.3 env_file: - .env volumes: diff --git a/portal_client.thrift b/portal_client.thrift index 2654b5f0..c58235d4 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -545,8 +545,7 @@ service VirtualMachineService { /** Add a user to a backend*/ map add_user_to_backend( 1:i64 backend_id, - 2:string owner_id, - 3:string user_id + 2:string user_id ) /** Get users from a backend*/ @@ -557,8 +556,7 @@ service VirtualMachineService { /** Delete user from a backend*/ map delete_user_from_backend( 1:i64 backend_id, - 2:string owner_id, - 3:string user_id + 2:string user_id ) list> get_templates() diff --git a/requirements.txt b/requirements.txt index 61e53c5d..4da4a1f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -setuptools==62.3.2 +setuptools==62.3.4 thrift==0.16.0 -python-keystoneclient -openstacksdk ==0.61.0 +python-keystoneclient==4.5.0 +openstacksdk ==0.99.0 deprecated == 1.2.13 -ansible==5.8.0 +ansible==5.9.0 Click==8.1.3 flake8==4.0.1 paramiko==2.11.0 ruamel.yaml==0.17.21 pyvim==3.0.3 -redis==4.3.2 +redis==4.3.3 requests==2.27.1 pyyaml==6.0