Skip to content

Commit

Permalink
removed unnecessary line in provider.py and added cloud information t…
Browse files Browse the repository at this point in the history
…o every log in validate_configuration.py for easier fixing.
  • Loading branch information
XaverStiensmeier committed Jan 12, 2024
1 parent 0c048b7 commit 989f9c4
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 40 deletions.
1 change: 0 additions & 1 deletion bibigrid/core/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def __init__(self, cloud_specification):
Call necessary methods to create a connection and save cloud_specification data as needed.
"""
self.cloud_specification = cloud_specification # contains sensitive information!
self.cloud_specification["identifier"] = self.cloud_specification['identifier']

@abstractmethod
def create_application_credential(self, name=None):
Expand Down
98 changes: 59 additions & 39 deletions bibigrid/core/utility/validate_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,19 +147,21 @@ def check_cloud_yaml(cloud_specification, log):
"application_credential_secret" in auth_keys):
log.warning("Insufficient authentication information. Needs either password and username or "
"if using application credentials: "
"auth_type, application_credential_id and application_credential_secret.")
"auth_type, application_credential_id and application_credential_secret. "
f"In cloud specification {cloud_specification.get('identifier')}")
success = False
if "auth_url" not in auth_keys:
log.warning("Authentification URL auth_url is missing.")
log.warning(f"Authentication URL auth_url is missing in cloud specification "
f"{cloud_specification.get('identifier')}")
success = False
else:
log.warning("Missing all auth information!")
log.warning(f"Missing all auth information in cloud specification {cloud_specification.get('identifier')}!")
success = False
if "region_name" not in keys:
log.warning("region_name is missing.")
log.warning(f"region_name is missing in cloud specification {cloud_specification.get('identifier')}.")
success = False
else:
log.warning("Missing all cloud_specification information!")
log.warning(f"{cloud_specification.get('identifier')} missing all cloud_specification information!")
return success


Expand Down Expand Up @@ -222,10 +224,12 @@ def check_master_vpn_worker(self):
"""
self.log.info("Checking master/vpn")
success = True
if not self.configurations[0].get("masterInstance") or self.configurations[0].get("vpnInstance"):
if not self.configurations[0].get("masterInstance"):
self.log.warning(f"{self.configurations[0].get('cloud')} has no master instance!")
success = False
for configuration in self.configurations[1:]:
if not configuration.get("vpnInstance") or configuration.get("masterInstance"):
if not configuration.get("vpnInstance"):
self.log.warning(f"{configuration.get('cloud')} has no vpn instance!")
success = False
return success

Expand All @@ -238,10 +242,12 @@ def check_provider_connections(self):
providers_unconnectable = []
for provider in self.providers:
if not provider.conn:
self.log.warning(f"API connection to {providers_unconnectable} not successful. "
f"Please check your configuration for cloud "
f"{provider.cloud_specification['identifier']}.")
providers_unconnectable.append(provider.cloud_specification["identifier"])
if providers_unconnectable:
self.log.warning("API connection to %s not successful. Please check your configuration.",
providers_unconnectable)
self.log.warning(f"Unconnected clouds: {providers_unconnectable}")
success = False
return success

Expand All @@ -252,9 +258,8 @@ def check_instances(self):
"""
self.log.info("Checking instance images and type")
success = True
configuration = None
try:
for configuration, provider in zip(self.configurations, self.providers):
for configuration, provider in zip(self.configurations, self.providers):
try:
self.required_resources_dict["floating_ips"] += 1
if configuration.get("masterInstance"):
success = self.check_instance("masterInstance", configuration["masterInstance"],
Expand All @@ -263,9 +268,10 @@ def check_instances(self):
success = self.check_instance("vpnInstance", configuration["vpnInstance"], provider) and success
for worker in configuration.get("workerInstances", []):
success = self.check_instance("workerInstance", worker, provider) and success
except KeyError as exc:
self.log.warning("Not found %s, but required in configuration %s.", str(exc), configuration)
success = False
except KeyError as exc:
self.log.warning("Not found %s, but required on %s.", str(exc),
provider.cloud_specification['identifier'])
success = False
return success

def check_instance(self, instance_name, instance, provider):
Expand All @@ -280,13 +286,15 @@ def check_instance(self, instance_name, instance, provider):
instance_image_id_or_name = instance["image"]
try:
instance_image = image_selection.select_image(provider, instance_image_id_or_name, self.log)
self.log.info("Instance %s image: %s found", instance_name, instance_image_id_or_name)
self.log.info(
f"Instance {instance_name} image: {instance_image_id_or_name} found on "
f"{provider.cloud_specification['identifier']}")
instance_type = instance["type"]
except ImageNotActiveException:
self.log.warning("Instance %s image: %s not found among active images.",
instance_name, instance_image_id_or_name)
self.log.log(42, "Available active images:")
self.log.log(42, "\n".join(provider.get_active_images()))
active_images = '\n'.join(provider.get_active_images())
self.log.warning(f"Instance {instance_name} image: {instance_image_id_or_name} not found among"
f" active images on {provider.cloud_specification['identifier']}.\n"
f"Available active images:\n{active_images}")
return False
return self.check_instance_type_image_combination(instance_type, instance_image, provider)

Expand All @@ -302,9 +310,9 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p
# check
flavor = provider.get_flavor(instance_type)
if not flavor:
self.log.warning("Flavor %s does not exist.", instance_type)
self.log.log(42, "Available flavors:")
self.log.log(42, "\n".join(provider.get_active_flavors()))
available_flavors = '\n'.join(provider.get_active_flavors())
self.log.warning(f"Flavor {instance_type} does not exist on {provider.cloud_specification['identifier']}.\n"
f"Available flavors:\n{available_flavors}")
return False
type_max_disk_space = flavor["disk"]
type_max_ram = flavor["ram"]
Expand Down Expand Up @@ -338,14 +346,17 @@ def check_volumes(self):
if not volume:
snapshot = provider.get_volume_snapshot_by_id_or_name(volume_name_or_id)
if not snapshot:
self.log.warning("Neither Volume nor Snapshot '%s' found", volume_name_or_id)
self.log.warning(f"Neither Volume nor Snapshot '{volume_name_or_id}' found on "
f"{provider.cloud_specification['identifier']}")
success = False
else:
self.log.info("Snapshot '%s' found", volume_name_or_id)
self.log.info(f"Snapshot '{volume_name_or_id}' found on "
f"{provider.cloud_specification['identifier']}.")
self.required_resources_dict["Volumes"] += 1
self.required_resources_dict["VolumeGigabytes"] += snapshot["size"]
else:
self.log.info(f"Volume '{volume_name_or_id}' found")
self.log.info(f"Volume '{volume_name_or_id}' found on "
f"{provider.cloud_specification['identifier']}.")
return success

def check_network(self):
Expand All @@ -357,22 +368,29 @@ def check_network(self):
success = True
for configuration, provider in zip(self.configurations, self.providers):
network_name_or_id = configuration.get("network")
subnet_name_or_id = configuration.get("subnet")
if network_name_or_id:
network = provider.get_network_by_id_or_name(network_name_or_id)
if not network:
self.log.warning(f"Network '{network_name_or_id}' not found", network_name_or_id)
self.log.warning(
f"Network '{network_name_or_id}' not found on {provider.cloud_specification['identifier']}")
success = False
else:
self.log.info(f"Network '{network_name_or_id}' found")
subnet_name_or_id = configuration.get("subnet")
if subnet_name_or_id:
self.log.info(
f"Network '{network_name_or_id}' found on {provider.cloud_specification['identifier']}")
elif subnet_name_or_id:
subnet = provider.get_subnet_by_id_or_name(subnet_name_or_id)
if not subnet:
self.log.warning(f"Subnet '{subnet_name_or_id}' not found")
self.log.warning(
f"Subnet '{subnet_name_or_id}' not found on {provider.cloud_specification['identifier']}")
success = False
else:
self.log.info(f"Subnet '{subnet_name_or_id}' found")
return bool(success and (network_name_or_id or subnet_name_or_id))
self.log.info(f"Subnet '{subnet_name_or_id}' found on {provider.cloud_specification['identifier']}")
else:
self.log.warning(f"Neither 'network' nor 'subnet' defined in configuration on "
f"{provider.cloud_specification['identifier']}.")
success = False
return success

def check_server_group(self):
"""
Expand All @@ -384,10 +402,12 @@ def check_server_group(self):
if server_group_name_or_id:
server_group = provider.get_server_group_by_id_or_name(server_group_name_or_id)
if not server_group:
self.log.warning("ServerGroup '%s' not found", server_group_name_or_id)
self.log.warning(f"ServerGroup '{server_group_name_or_id}' not found on "
f"{provider.cloud_specification['identifier']}")
success = False
else:
self.log.info("ServerGroup '%s' found", server_group_name_or_id)
self.log.info(f"ServerGroup '{server_group_name_or_id}' found on "
f"{provider.cloud_specification['identifier']}")
return success

def check_quotas(self):
Expand Down Expand Up @@ -419,10 +439,11 @@ def check_ssh_public_key_files(self):
for configuration in self.configurations:
for ssh_public_key_file in configuration.get("sshPublicKeyFiles") or []:
if not os.path.isfile(ssh_public_key_file):
self.log.warning("sshPublicKeyFile '%s' not found", ssh_public_key_file)
self.log.warning(
f"sshPublicKeyFile '{ssh_public_key_file}' not found on {configuration.get('cloud')}")
success = False
else:
self.log.info("sshPublicKeyFile '%s' found", ssh_public_key_file)
self.log.info(f"sshPublicKeyFile '{ssh_public_key_file}' found on {configuration.get('cloud')}")
success = evaluate_ssh_public_key_file_security(ssh_public_key_file, self.log) and success
return success

Expand All @@ -437,8 +458,7 @@ def check_clouds_yamls(self):
for index, cloud_specification in enumerate(cloud_specifications):
if not check_cloud_yaml(cloud_specification, self.log):
success = False
self.log.warning("Cloud specification %s is faulty. BiBiGrid understood %s.", index,
cloud_specification)
self.log.warning(f"Cloud specification {cloud_specification.get('identifier', index)} is faulty.")
success = check_clouds_yaml_security(self.log) and success
return success

Expand Down

0 comments on commit 989f9c4

Please sign in to comment.