diff --git a/.env.in b/.env.in index 5f86177b..e533cf0e 100644 --- a/.env.in +++ b/.env.in @@ -13,7 +13,7 @@ OS_PROJECT_DOMAIN_ID= OS_PASSWORD= # Filebeat -FILEBEAT_TAG=7.15.0 +FILEBEAT_TAG=7.15.1 ELASTIC_USER= ELASTIC_PASSWORD= HOST=portal-dev.denbi.de diff --git a/.github/workflows/blacked.yml b/.github/workflows/blacked.yml index 6c4b2754..3d138c7c 100644 --- a/.github/workflows/blacked.yml +++ b/.github/workflows/blacked.yml @@ -4,7 +4,7 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Checking out Branch run: git checkout "${GITHUB_REF:11}" - name: black diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index b53d9a1e..455355fe 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -8,5 +8,5 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - run: docker build --no-cache . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d5fbff23..e508d04e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/master-protection.yml b/.github/workflows/master-protection.yml index 80c19991..5376e4a7 100644 --- a/.github/workflows/master-protection.yml +++ b/.github/workflows/master-protection.yml @@ -14,7 +14,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Check Tag run: | diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index a9a4371d..c61f1ae8 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -15,7 +15,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml index 8b8a777d..7be6bf83 100644 --- a/.github/workflows/release_image.yml +++ b/.github/workflows/release_image.yml @@ -11,7 +11,7 @@ jobs: - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master with: diff --git a/.github/workflows/update_docker_tags.yml b/.github/workflows/update_docker_tags.yml index 3aca2041..43f5cf02 100644 --- a/.github/workflows/update_docker_tags.yml +++ b/.github/workflows/update_docker_tags.yml @@ -14,7 +14,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - run: | python3 update_env_by_dockerfile.py Dockerfile.update .env.in echo $? diff --git a/Dockerfile b/Dockerfile index 48eb5aae..0dec2d75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-buster +FROM python:3.10.0-buster RUN apt-get update -y RUN apt-get install -y build-essential WORKDIR /code diff --git a/Dockerfile.update b/Dockerfile.update index d48ce49f..ff2fc394 100644 --- a/Dockerfile.update +++ b/Dockerfile.update @@ -1,3 +1,3 @@ FROM denbicloud/cloud-portal-client:0.1.0-beta.0.15.1 FROM bibiserv/bibigrid:bibigrid-rest-2.3 -FROM docker.elastic.co/beats/filebeat:7.15.0 +FROM docker.elastic.co/beats/filebeat:7.15.1 diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index cdb75782..33617211 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,7 @@ Try to fulfill the following points before the Pull Request is merged: - [ ] The PR is reviewed by one of the team members. +- [ ] If a linting PR exists, it must be merged before this PR is allowed to be merged. - [ ] If the PR is merged in the master then a release should be be made. - [ ] If the new code is readable, if not it should be well commented diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index ce02df90..eeeee244 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -45,7 +45,6 @@ import json import logging import os -import parser import socket import time import urllib @@ -226,9 +225,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg="Forc-Backend url loaded: {0}".format(self.RE_BACKEND_URL)) @@ -256,8 +255,7 @@ def __init__(self, config): self.UDP_FORMULAR = cfg["openstack_connection"][ "udp_port_calc_formular" ] - self.SSH_PORT_CALCULATION = parser.expr(self.SSH_FORMULAR).compile() - self.UDP_PORT_CALCULATION = parser.expr(self.UDP_FORMULAR).compile() + LOG.info("Gateway IP is {}".format(self.GATEWAY_IP)) self.update_playbooks() self.conn = self.create_connection() @@ -372,10 +370,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): metadata = img["metadata"] @@ -383,7 +381,7 @@ def get_Images(self): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) continue @@ -419,7 +417,7 @@ def prepare_image(self, img): tags = img.get("tags") LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info("Resenv check: Skipping {0}.".format(img["name"])) return None @@ -455,11 +453,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -481,11 +479,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -526,7 +524,7 @@ def get_Image_with_Tag(self, id): return image except Exception as e: LOG.exception("Get Image {0} with Tag Error: {1}".format(id, e)) - return None + return Image() def get_Images_by_filter(self, filter_list): """ @@ -538,10 +536,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags") if "resenv" in filter_list: @@ -793,7 +791,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info("create init script for volume ids:{}".format(volume_ids_path_new)) if not volume_ids_path_new and not volume_ids_path_attach: @@ -942,17 +940,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -1025,18 +1023,18 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https): return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1085,9 +1083,9 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + init_script ) else: @@ -1120,17 +1118,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1184,16 +1182,16 @@ def start_server( return {} def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ @@ -1269,7 +1267,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info(msg="Starting Playbook for (openstack_id): {0}".format(openstack_id)) @@ -1318,8 +1316,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -1799,7 +1797,7 @@ def check_server_status(self, openstack_id): server_base = serv_cop.fixed_ip.split(".")[-1] x = int(server_base) host = str(self.GATEWAY_IP) - port = eval(self.SSH_PORT_CALCULATION) + port = eval(self.SSH_FORMULAR) elif self.get_server(openstack_id).floating_ip is None: host = self.add_floating_ip_to_server( openstack_id, self.FLOATING_IP_NETWORK @@ -1952,8 +1950,8 @@ def add_udp_security_group(self, server_id): ip_base = ( list(self.conn.compute.server_ips(server=server_id))[0] - .to_dict()["address"] - .split(".")[-1] + .to_dict()["address"] + .split(".")[-1] ) x = int(ip_base) udp_port_start = eval(self.UDP_PORT_CALCULATION) @@ -2013,8 +2011,8 @@ def get_vm_ports(self, openstack_id): server = self.get_server(openstack_id) server_base = server.fixed_ip.split(".")[-1] x = int(server_base) - port = eval(self.SSH_PORT_CALCULATION) - udp_port_start = eval(self.UDP_PORT_CALCULATION) + port = eval(self.SSH_FORMULAR) + udp_port_start = eval(self.UDP_FORMULAR) return {"port": str(port), "udp": str(udp_port_start)} def terminate_cluster(self, cluster_id): @@ -2076,7 +2074,7 @@ def get_clusters_info(self): return infos def scale_up_cluster( - self, cluster_id, image, flavor, count, names, start_idx, batch_index + self, cluster_id, image, flavor, count, names, start_idx, batch_index ): cluster_info = self.get_cluster_info(cluster_id=cluster_id) image = self.get_image(image=image) @@ -2352,9 +2350,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2363,7 +2361,7 @@ def delete_server(self, openstack_id): sec for sec in security_groups if sec.name != self.DEFAULT_SECURITY_GROUP_NAME - and not "bibigrid" in sec.name + and not "bibigrid" in sec.name ] if security_groups is not None: for sg in security_groups: @@ -2516,15 +2514,15 @@ def resume_server(self, openstack_id): return False def create_security_group( - self, - name, - udp_port_start=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port_start=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info("Create new security group {}".format(name)) sec = self.conn.get_security_group(name_or_id=name) @@ -2758,15 +2756,15 @@ def update_forc_allowed(self, template_metadata): class ResenvMetadata: def __init__( - self, - name, - port, - security_group_name, - security_group_description, - security_group_ssh, - direction, - protocol, - information_for_display, + self, + name, + port, + security_group_name, + security_group_description, + security_group_ssh, + direction, + protocol, + information_for_display, ): self.name = name self.port = port diff --git a/VirtualMachineService/VirtualMachineServer.py b/VirtualMachineService/VirtualMachineServer.py index 2c7f4217..f03f4cff 100644 --- a/VirtualMachineService/VirtualMachineServer.py +++ b/VirtualMachineService/VirtualMachineServer.py @@ -19,6 +19,7 @@ import yaml import click import signal +import ssl USERNAME = "OS_USERNAME" PASSWORD = "OS_PASSWORD" @@ -27,7 +28,7 @@ USER_DOMAIN_ID = "OS_USER_DOMAIN_NAME" AUTH_URL = "OS_AUTH_URL" PROJECT_DOMAIN_ID = "OS_PROJECT_DOMAIN_ID" -FORC_API_KEY= "FORC_API_KEY" +FORC_API_KEY = "FORC_API_KEY" environment_variables = [ USERNAME, @@ -37,7 +38,7 @@ USER_DOMAIN_ID, AUTH_URL, PROJECT_DOMAIN_ID, - FORC_API_KEY + FORC_API_KEY, ] @@ -71,7 +72,7 @@ def catch_shutdown(signal, frame): processor = Processor(handler) if USE_SSL: click.echo("Use SSL") - transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE) + transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE,ssl_version=ssl.PROTOCOL_TLS_SERVER) else: click.echo("Does not use SSL") transport = TSocket.TServerSocket(host=HOST, port=PORT) diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 39f2ce7f..2a100277 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -1,5 +1,5 @@ redis: - host: redis + host: client_redis port: 6379 password: "" diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index 40829c35..2b728c07 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -1,6 +1,7 @@ version: '3.5' services: portal-client: + container_name: client image: denbicloud/cloud-portal-client:${CLOUD_PORTAL_CLIENT_TAG} ports: - ${CLIENT_PORT}:9090 @@ -14,14 +15,16 @@ services: networks: - portal - redis: - image: redis:6.0.9 + client_redis: + container_name: client_redis + image: redis:6.2.6 expose: - "6379" networks: - portal bibigrid: + container_name: bibigrid image: bibiserv/bibigrid:${BIBIGRID_TAG} env_file: - .env diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index 4e56af03..e68fb65f 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -1,6 +1,7 @@ version: '3.5' services: portal-client: + container_name: client build: context: . dockerfile: Dockerfile @@ -15,14 +16,16 @@ services: networks: - portal - redis: - image: redis:6.0.9 + client_redis: + container_name: client_redis + image: redis:6.2.6 expose: - "6379" networks: - portal bibigrid: + container_name: bibigrid image: bibiserv/bibigrid:${BIBIGRID_TAG} env_file: - .env diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index bc3d628b..04fffbeb 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,6 +1,7 @@ version: '3.5' services: portal-client: + container_name: client build: context: . dockerfile: Dockerfile @@ -15,8 +16,9 @@ services: networks: - portal - redis: - image: redis:6.0.9 + client_redis: + container_name: client_redis + image: redis:6.2.6 expose: - "6379" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 876a80fd..1f814863 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.5' services: portal-client: + container_name: client image: denbicloud/cloud-portal-client:${CLOUD_PORTAL_CLIENT_TAG} ports: - ${CLIENT_PORT}:9090 @@ -14,8 +15,9 @@ services: networks: - portal - redis: - image: redis:6.0.9 + client_redis: + container_name: client_redis + image: redis:6.2.6 expose: - "6379" networks: @@ -23,6 +25,7 @@ services: # filebeat filebeat: + container_name: client_filebeat image: docker.elastic.co/beats/filebeat:${FILEBEAT_TAG} env_file: - .env diff --git a/requirements.txt b/requirements.txt index 61233210..94a583b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -setuptools==58.2.0 -thrift >= 0.11.0,<0.20.0 +setuptools==58.4.0 +thrift==0.15.0 python-keystoneclient openstacksdk ==0.59.0 deprecated == 1.2.13 Click==8.0.3 -ansible==4.6.0 -flake8==3.9.2 +ansible==4.7.0 +flake8==4.0.1 ruamel.yaml<0.18.0 paramiko==2.8.0 pyvim==3.0.2 redis==3.5.3 requests==2.26.0 -pyyaml==5.4.1 \ No newline at end of file +pyyaml==6.0