From 73494c61cf3f34db8a9b1477643927323feba964 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Mon, 11 Nov 2024 12:56:17 +0100 Subject: [PATCH] volumes are now validated and fixed old state of masterInstance in validate_schema.py --- .../core/utility/validate_configuration.py | 72 +++++++++++++------ bibigrid/core/utility/validate_schema.py | 4 +- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/bibigrid/core/utility/validate_configuration.py b/bibigrid/core/utility/validate_configuration.py index 4451b632..f3d6d4af 100644 --- a/bibigrid/core/utility/validate_configuration.py +++ b/bibigrid/core/utility/validate_configuration.py @@ -325,38 +325,63 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p self.required_resources_dict[provider.cloud_specification['identifier']]["total_cores"] += flavor["vcpus"] return success - def check_volumes(self): + def check_volumes(self): # pylint: disable=too-many-branches """ Checking if volume or snapshot exists for all volumes @return: True if all snapshot and volumes are found. Else false. """ self.log.info("Checking volumes...") success = True - for configuration, provider in zip(self.configurations, self.providers): - volume_identifiers = [masterMount["name"] for masterMount in configuration.get("masterMounts", [])] - if volume_identifiers: - # check individually if volumes exist - for volume_identifier in volume_identifiers: - if ":" in volume_identifier: - volume_name_or_id = volume_identifier[:volume_identifier.index(":")] - else: - volume_name_or_id = volume_identifier - volume = provider.get_volume_by_id_or_name(volume_name_or_id) - if not volume: - snapshot = provider.get_volume_snapshot_by_id_or_name(volume_name_or_id) - if not snapshot: - self.log.warning(f"Neither Volume nor Snapshot '{volume_name_or_id}' found on " - f"{provider.cloud_specification['identifier']}") + for configuration, provider in zip(self.configurations, self.providers): # pylint: disable=too-many-nested-blocks,too-many-branches + master_volumes = ( + 1, configuration.get("masterInstance", []) and configuration["masterInstance"].get("volumes", + [])) + worker_volumes = configuration.get("workerInstances", (1, [])) and [ + (worker_instance.get("count", 1), worker_instance.get("volumes", [])) for + worker_instance in configuration["workerInstances"]] + volume_groups = [master_volumes] + worker_volumes + + for count, volume_group in volume_groups: + for volume in volume_group: + if volume.get("exists"): + if volume.get("name"): + volume_object = provider.get_volume_by_id_or_name(volume["name"]) + if volume_object: + self.log.debug( + f"Found volume {volume['name']} on cloud " + f"{provider.cloud_specification['identifier']}.") + else: + self.log.warning( + f"Couldn't find volume {volume['name']} on cloud " + f"{provider.cloud_specification['identifier']}. " + "No size added to resource requirements dict." + ) + success = False + else: + self.log.warning( + f"Key exists is set, but no name is given for {volume}. " + "No size added to resource requirements dict.") success = False + else: + self.required_resources_dict[provider.cloud_specification['identifier']]["volumes"] += count + + if volume.get("snapshot"): + snapshot_object = provider.get_volume_snapshot_by_id_or_name(volume["snapshot"]) + if snapshot_object: + self.log.debug( + f"Found snapshot {volume['snapshot']} on cloud " + f"{provider.cloud_specification['identifier']}.") + self.required_resources_dict[provider.cloud_specification['identifier']][ + "volume_gigabytes"] += snapshot_object["size"] * count + else: + self.log.warning( + f"Couldn't find snapshot {volume['snapshot']} on cloud " + f"{provider.cloud_specification['identifier']}. " + "No size added to resource requirements dict.") + success = False else: - self.log.info(f"Snapshot '{volume_name_or_id}' found on " - f"{provider.cloud_specification['identifier']}.") - self.required_resources_dict[provider.cloud_specification['identifier']]["volumes"] += 1 self.required_resources_dict[provider.cloud_specification['identifier']][ - "volume_gigabytes"] += snapshot["size"] - else: - self.log.info(f"Volume '{volume_name_or_id}' found on " - f"{provider.cloud_specification['identifier']}.") + "volume_gigabytes"] += volume.get("size", 50) * count return success def check_network(self): @@ -424,6 +449,7 @@ def check_quotas(self): self.log.info("required/available") for provider in self.providers: free_resources_dict = provider.get_free_resources() + print("REQ_REC", self.required_resources_dict) for key, value in self.required_resources_dict[provider.cloud_specification['identifier']].items(): success = has_enough(free_resources_dict[key], value, f"Project {provider.cloud_specification['identifier']}", key, self.log) and success diff --git a/bibigrid/core/utility/validate_schema.py b/bibigrid/core/utility/validate_schema.py index b5e8e8ff..695e62ec 100644 --- a/bibigrid/core/utility/validate_schema.py +++ b/bibigrid/core/utility/validate_schema.py @@ -30,7 +30,7 @@ Optional('terminate'): bool, Optional('size'): int }, - Optional('volumes'): { + Optional('volumes'): [{ Optional('snapshot'): str, # optional; to create volume from # one or none of these Optional('permanent'): bool, @@ -38,7 +38,7 @@ Optional('mountPoint'): str, Optional('size'): int, Optional('fstype'): str, - Optional('type'): str} + Optional('type'): str}] } # Define the schema for the configuration file