From d53e30f4111a0d6799e85c6638b8c6ce06144602 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 14 Sep 2023 14:57:27 +0200 Subject: [PATCH] changed image selection. Now image can be a regex, too. --- bibigrid/core/actions/create.py | 25 +++------------- bibigrid/core/utility/ansible_configurator.py | 9 ++++-- bibigrid/core/utility/image_selection.py | 30 +++++++++++++++++++ 3 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 bibigrid/core/utility/image_selection.py diff --git a/bibigrid/core/actions/create.py b/bibigrid/core/actions/create.py index 7f524a26..4945df2b 100644 --- a/bibigrid/core/actions/create.py +++ b/bibigrid/core/actions/create.py @@ -2,27 +2,25 @@ The cluster creation (master's creation, key creation, ansible setup and execution, ...) is done here """ -import difflib import logging import os import subprocess import threading import traceback -import re from functools import partial import paramiko import yaml from bibigrid.core.actions import terminate_cluster -from bibigrid.core.utility import ansible_configurator +from bibigrid.core.utility import ansible_configurator, image_selection from bibigrid.core.utility import id_generation from bibigrid.core.utility.handler import ssh_handler from bibigrid.core.utility.paths import ansible_resources_path as aRP from bibigrid.core.utility.paths import bin_path as biRP from bibigrid.models import exceptions from bibigrid.models import return_threading -from bibigrid.models.exceptions import ExecutionException, ConfigurationException, ImageDeactivatedException +from bibigrid.models.exceptions import ExecutionException, ConfigurationException PREFIX = "bibigrid" SEPARATOR = "-" @@ -156,7 +154,7 @@ def generate_security_groups(self): _ = provider.create_security_group(name=self.wireguard_security_group_name)["id"] configuration["security_groups"].append(self.wireguard_security_group_name) # store in configuration - def start_vpn_or_master_instance(self, configuration, provider): # pylint: disable=too-many-branches + def start_vpn_or_master_instance(self, configuration, provider): # pylint: disable=too-many-branches """ Start master/vpn-worker of a provider :param configuration: dict configuration of said provider @@ -176,22 +174,7 @@ def start_vpn_or_master_instance(self, configuration, provider): # pylint: disab flavor = instance_type["type"] network = configuration["network"] image = instance_type["image"] - # check if image is active - active_images = provider.get_active_images() - if image not in active_images: - old_image = image - image = None - LOG.warning(f"Image '{image}' not active or doesn't exist.") - if isinstance(configuration.get("fallbackOnOtherImage"), str): - image = next(elem for elem in active_images if re.match(configuration["fallbackOnOtherImage"], elem)) - LOG.info(f"Taking first regex ('{configuration['fallbackOnOtherImage']}') match '{image}'") - elif configuration.get("fallbackOnOtherImage", False): - image = difflib.get_close_matches(old_image, active_images)[0] - LOG.info(f"Taking closest (in name) active image '{image}'.") - if configuration.get("fallbackOnOtherImage", False) and image: - LOG.warning(f"Taking '{image}' instead of '{old_image}' as that image doesn't exist or is not active.") - else: - raise ImageDeactivatedException(f"Image {old_image} no longer active or doesn't exist.") + image = image_selection.select_image(configuration, provider, image, LOG) # create a server and block until it is up and running server = provider.create_server(name=name, flavor=flavor, key_name=self.key_name, image=image, network=network, diff --git a/bibigrid/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py index cbd909c4..1255cd22 100644 --- a/bibigrid/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -95,7 +95,8 @@ def write_host_and_group_vars(configurations, providers, cluster_id): # pylint: worker_dict = {"name": name, "regexp": regexp, "image": worker["image"], "network": configuration["network"], "flavor": flavor_dict, "gateway_ip": configuration["private_v4"], - "cloud_identifier": configuration["cloud_identifier"]} + "cloud_identifier": configuration["cloud_identifier"], + "fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)} worker_features = worker.get("features", []) if isinstance(worker_features, str): @@ -120,7 +121,8 @@ def write_host_and_group_vars(configurations, providers, cluster_id): # pylint: "flavor": flavor_dict, "wireguard_ip": wireguard_ip, "cloud_identifier": configuration[ - "cloud_identifier"]} + "cloud_identifier"], + "fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)} if configuration.get("wireguard_peer"): vpngtw_dict["wireguard"] = {"ip": wireguard_ip, "peer": configuration.get( @@ -136,7 +138,8 @@ def write_host_and_group_vars(configurations, providers, cluster_id): # pylint: "floating_ip": configuration["floating_ip"], "flavor": flavor_dict, "private_v4": configuration["private_v4"], "cloud_identifier": configuration["cloud_identifier"], - "volumes": configuration["volumes"]} + "volumes": configuration["volumes"], + "fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)} if configuration.get("wireguard_peer"): master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")} write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, "master.yml"), master_dict) diff --git a/bibigrid/core/utility/image_selection.py b/bibigrid/core/utility/image_selection.py new file mode 100644 index 00000000..9a419789 --- /dev/null +++ b/bibigrid/core/utility/image_selection.py @@ -0,0 +1,30 @@ +""" +Methods for image selection +""" +import difflib +import re + +from bibigrid.models.exceptions import ImageDeactivatedException + + +def select_image(configuration, provider, image, log): + # check if image is active + active_images = provider.get_active_images() + if image not in active_images: + old_image = image + log.info(f"Image '{old_image}' has no direct match. Maybe it's a regex? Trying regex match.") + image = next((elem for elem in active_images if re.match(image, elem)), None) + if not image: + log.warning(f"Couldn't find image '{old_image}'.") + if isinstance(configuration.get("fallbackOnOtherImage"), str): + image = next(elem for elem in active_images if re.match(configuration["fallbackOnOtherImage"], elem)) + log.info(f"Taking first regex ('{configuration['fallbackOnOtherImage']}') match '{image}'.") + elif configuration.get("fallbackOnOtherImage", False): + image = difflib.get_close_matches(old_image, active_images)[0] + log.info(f"Taking closest active image (in name) '{image}' instead.") + else: + raise ImageDeactivatedException(f"Image {old_image} no longer active or doesn't exist.") + log.info(f"Using alternative '{image}' instead of '{old_image}'. You should change the configuration.") + else: + log.info(f"Taking first regex match: '{image}'") + return image