Skip to content

Commit

Permalink
Merge branch 'dev' into yaml-error
Browse files Browse the repository at this point in the history
# Conflicts:
#	bibigrid/core/actions/ide.py
#	bibigrid/core/utility/handler/ssh_handler.py
  • Loading branch information
XaverStiensmeier committed Jan 10, 2024
2 parents fcf25a7 + 0c048b7 commit 7856940
Show file tree
Hide file tree
Showing 29 changed files with 1,303 additions and 1,239 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ resources/playbook/ansible_hosts
resources/playbook/vars/
resources/playbook/host_vars/
resources/playbook/group_vars/
tests/resources/*
!test/resources/test_configuration.yml

# any log files
*.log
Expand Down
1 change: 0 additions & 1 deletion bibigrid/core/actions/ide.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
LOCALHOST = "127.0.0.1"



def sigint_handler(caught_signal, frame): # pylint: disable=unused-argument
"""
Is called when SIGINT is thrown and terminates the program
Expand Down
4 changes: 2 additions & 2 deletions bibigrid/core/actions/list_clusters.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from bibigrid.core.actions import create

SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)\d+-([a-zA-Z0-9]+)-\d+)$")
SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)-([a-zA-Z0-9]+)-\d+)$")


def dict_clusters(providers, log):
Expand Down Expand Up @@ -69,7 +69,7 @@ def log_list(cluster_id, providers, log):
master_count, worker_count, vpn_count = get_size_overview(cluster_dict[cluster_id], log)
log.log(42, f"\tCluster has {master_count} master, {vpn_count} vpngtw and {worker_count} regular workers. "
f"The cluster is spread over {vpn_count + master_count} reachable provider(s).")
pprint.pprint(cluster_dict[cluster_id])
log.log(42, pprint.pformat(cluster_dict[cluster_id]))
else:
log.info("Cluster with cluster-id {cluster_id} not found.")
log.log(42, f"Cluster with cluster-id {cluster_id} not found.")
Expand Down
12 changes: 6 additions & 6 deletions bibigrid/core/actions/terminate.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
cluster_security_group_state = []
tmp_keyname = create.KEY_NAME.format(cluster_id=cluster_id)
local_keypairs_deleted = delete_local_keypairs(tmp_keyname, log)
if not assume_yes and (
local_keypairs_deleted or input(f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. "
f"This might not be your cluster. Are you sure you want to terminate it?\n"
f"Any non-empty input to shutdown cluster {cluster_id}. "
f"Empty input to exit with cluster still alive:")):
if assume_yes or local_keypairs_deleted or input(
f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. "
f"This might not be your cluster. Are you sure you want to terminate it?\n"
f"Any non-empty input to shutdown cluster {cluster_id}. "
f"Empty input to exit with cluster still alive:"):
for provider in providers:
log.info("Terminating cluster %s on cloud %s", cluster_id, provider.cloud_specification['identifier'])
server_list = provider.list_servers()
Expand All @@ -61,7 +61,7 @@ def terminate_servers(server_list, cluster_id, provider, log):
"""
log.info("Deleting servers on provider %s...", provider.cloud_specification['identifier'])
cluster_server_state = []
server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}+|(worker|vpngtw)-{cluster_id}+-\d+)$")
server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}|(worker|vpngtw)-{cluster_id}-\d+)$")
for server in server_list:
if server_regex.match(server["name"]):
log.info("Trying to terminate Server %s on cloud %s.", server['name'],
Expand Down
16 changes: 8 additions & 8 deletions bibigrid/core/utility/ansible_configurator.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ def write_host_and_group_vars(configurations, providers, cluster_id, log): # py
master_dict = {"name": name, "image": master["image"], "network": configuration["network"],
"network_cidrs": configuration["subnet_cidrs"], "floating_ip": configuration["floating_ip"],
"flavor": flavor_dict, "private_v4": configuration["private_v4"],
"cloud_identifier": configuration["cloud_identifier"],
"volumes": configuration["volumes"],
"cloud_identifier": configuration["cloud_identifier"], "volumes": configuration["volumes"],
"fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)}
if configuration.get("wireguard_peer"):
master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")}
Expand Down Expand Up @@ -165,8 +164,8 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us
master_configuration = configurations[0]
log.info("Generating common configuration file...")
# print(configuration.get("slurmConf", {}))
common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False),
"cluster_id": cluster_id, "cluster_cidrs": cidrs, "default_user": default_user,
common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id,
"cluster_cidrs": cidrs, "default_user": default_user,
"local_fs": master_configuration.get("localFS", False),
"local_dns_lookup": master_configuration.get("localDNSlookup", False),
"use_master_as_compute": master_configuration.get("useMasterAsCompute", True),
Expand Down Expand Up @@ -266,17 +265,18 @@ def get_cidrs(configurations):
"""
all_cidrs = []
for configuration in configurations:
subnet = configuration["subnet_cidrs"]
provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"], "provider_cidrs": subnet}
provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"],
"provider_cidrs": configuration["subnet_cidrs"]}
all_cidrs.append(provider_cidrs)
return all_cidrs


def get_ansible_roles(ansible_roles, log):
"""
Checks if ansible_roles have all necessary values and returns True if so.
:param ansible_roles: ansible_roles from master configuration (first configuration)
:return: list of valid ansible_roles
@param ansible_roles: ansible_roles from master configuration (first configuration)
@param log:
@return: list of valid ansible_roles
"""
ansible_roles_yaml = []
for ansible_role in (ansible_roles or []):
Expand Down
6 changes: 3 additions & 3 deletions bibigrid/core/utility/handler/ssh_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import time

import paramiko
import yaml
import sympy
import yaml

from bibigrid.core.utility import ansible_commands as aC
from bibigrid.models.exceptions import ConnectionException, ExecutionException
Expand Down Expand Up @@ -107,7 +107,7 @@ def is_active(client, floating_ip_address, private_key, username, log, gateway,
port = 22
if gateway:
log.info(f"Using SSH Gateway {gateway.get('ip')}")
octets = {f'oct{enum+1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))}
octets = {f'oct{enum + 1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))}
port = int(sympy.sympify(gateway["portFunction"]).subs(dict(octets)))
log.info(f"Port {port} will be used (see {gateway['portFunction']} and octets {octets}).")
client.connect(hostname=gateway.get("ip") or floating_ip_address, username=username,
Expand Down Expand Up @@ -159,7 +159,7 @@ def execute_ssh_cml_commands(client, commands, log):
:param log:
"""
for command in commands:
ssh_stdin, ssh_stdout, ssh_stderr = client.exec_command(command[0]) # pylint: disable=unused-variable
_, ssh_stdout, _ = client.exec_command(command[0])
ssh_stdout.channel.set_combine_stderr(True)
log.info(f"REMOTE: {command[1]}")

Expand Down
4 changes: 2 additions & 2 deletions bibigrid/core/utility/validate_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,8 +303,8 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p
flavor = provider.get_flavor(instance_type)
if not flavor:
self.log.warning("Flavor %s does not exist.", instance_type)
print("Available flavors:")
print("\n".join(provider.get_active_flavors()))
self.log.log(42, "Available flavors:")
self.log.log(42, "\n".join(provider.get_active_flavors()))
return False
type_max_disk_space = flavor["disk"]
type_max_ram = flavor["ram"]
Expand Down
2 changes: 2 additions & 0 deletions bibigrid/openstack/openstack_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ def create_server(self, name, flavor, image, network, key_name=None, wait=True,
except openstack.exceptions.BadRequestException as exc:
if "is not active" in str(exc):
raise ImageDeactivatedException() from exc
if "Invalid key_name provided" in str(exc):
raise ExecutionException() from exc
raise ConnectionError() from exc
except openstack.exceptions.SDKException as exc:
raise ExecutionException() from exc
Expand Down
61 changes: 36 additions & 25 deletions tests/provider/test_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
Module containing integration and unit tests regarding the provider
"""

import logging
import os
import unittest

import bibigrid.core.utility.handler.configuration_handler as configurationHandler
import bibigrid.core.utility.handler.provider_handler as providerHandler
import bibigrid.core.utility.paths.basic_path as bP
from bibigrid.core.utility.handler import configuration_handler
from bibigrid.core.utility.handler import provider_handler
from bibigrid.models.exceptions import ExecutionException

SERVER_KEYS = {'id', 'name', 'flavor', 'image', 'block_device_mapping', 'location', 'volumes', 'has_config_drive',
'host_id', 'progress', 'disk_config', 'power_state', 'task_state', 'vm_state', 'launched_at',
Expand All @@ -18,7 +20,12 @@
'updated', 'user_id', 'tags', 'interface_ip', 'properties', 'hostId', 'config_drive', 'project_id',
'tenant_id', 'region', 'cloud', 'az', 'OS-DCF:diskConfig', 'OS-EXT-AZ:availability_zone',
'OS-SRV-USG:launched_at', 'OS-SRV-USG:terminated_at', 'OS-EXT-STS:task_state', 'OS-EXT-STS:vm_state',
'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached'}
'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached', 'OS-EXT-SRV-ATTR:ramdisk_id',
'max_count', 'trusted_image_certificates', 'OS-SCH-HNT:scheduler_hints',
'OS-EXT-SRV-ATTR:reservation_id', 'OS-EXT-SRV-ATTR:host', 'locked', 'host_status',
'OS-EXT-SRV-ATTR:hypervisor_hostname', 'OS-EXT-SRV-ATTR:launch_index',
'OS-EXT-SRV-ATTR:root_device_name', 'OS-EXT-SRV-ATTR:instance_name', 'OS-EXT-SRV-ATTR:user_data',
'min_count', 'OS-EXT-SRV-ATTR:kernel_id', 'OS-EXT-SRV-ATTR:hostname'}
FLOATING_IP_KEYS = {'attached', 'fixed_ip_address', 'floating_ip_address', 'id', 'location', 'network', 'port',
'router', 'status', 'created_at', 'updated_at', 'description', 'revision_number', 'properties',
'port_id', 'router_id', 'project_id', 'tenant_id', 'floating_network_id', 'port_details',
Expand All @@ -39,7 +46,7 @@
'is_protected', 'locations', 'properties', 'is_public', 'visibility', 'description',
'owner_specified.openstack.md5', 'owner_specified.openstack.object', 'owner_specified.openstack.sha256',
'os_hidden', 'os_hash_algo', 'os_hash_value', 'os_distro', 'os_version', 'schema', 'protected',
'metadata', 'created', 'updated', 'minDisk', 'minRam'}
'metadata', 'created', 'updated', 'minDisk', 'minRam', 'stores'}

SNAPSHOT_KEYS = {'id', 'created_at', 'updated_at', 'name', 'description', 'volume_id', 'status', 'size', 'metadata',
'os-extended-snapshot-attributes:project_id', 'os-extended-snapshot-attributes:progress'}
Expand All @@ -61,9 +68,10 @@
"MFbUTTukAiDf4jAgvJkg7ayE0MPapGpI/OhSK2gyN45VAzs2m7uykun87B491JagZ57qr16vt8vxGYpFCEe8QqAcrUszUPqyPrb0auA8bz" \
"jO8S41Kx8FfG+7eTu4dQ0= user"

CONFIGURATIONS = configurationHandler.read_configuration(
os.path.join(bP.ROOT_PATH, "tests/resources/infrastructure_cloud.yml"))
PROVIDERS = providerHandler.get_providers(CONFIGURATIONS)
CONFIGURATIONS = configuration_handler.read_configuration(logging,
os.path.join(bP.ROOT_PATH,
"tests/resources/infrastructure_cloud.yml"))
PROVIDERS = provider_handler.get_providers(CONFIGURATIONS, logging)


class ProviderServer:
Expand All @@ -76,7 +84,7 @@ def __init__(self, provider, name, configuration, key_name=None):
self.name = name
self.server_dict = provider.create_server(name=self.name, flavor=configuration["flavor"],
image=configuration["image"], network=configuration["network"],
key_name=key_name)
key_name=key_name, security_groups=[])

def __enter__(self):
return self.server_dict
Expand All @@ -95,38 +103,41 @@ def test_get_free_resources(self):
with self.subTest(provider.NAME):
free_dict = provider.get_free_resources()
self.assertEqual(FREE_RESOURCES_KEYS, set(free_dict.keys()))
for value in free_dict.values():
self.assertLessEqual(0, value)
for key, value in free_dict.items():
if key != "floating_ips":
self.assertLessEqual(0, value)

def test_server_start_type_error(self):
for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
with self.subTest(provider.NAME):
with self.assertRaises(TypeError):
provider.create_server(name="name", flavor=configuration["flavor"],
network=configuration["network"])
network=configuration["network"], security_groups=[])
with self.assertRaises(TypeError):
provider.create_server(name="name", image=configuration["image"], network=configuration["network"])
provider.create_server(name="name", image=configuration["image"], network=configuration["network"],
security_groups=[])
with self.assertRaises(TypeError):
provider.create_server(flavor=configuration["flavor"], image=configuration["image"],
network=configuration["network"])
security_groups=[], network=configuration["network"])
with self.assertRaises(TypeError):
provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"])
provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
security_groups=[])

def test_server_start_attribute_error(self):
for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
with self.subTest(provider.NAME):
with self.assertRaises(AttributeError):
with self.assertRaises(ExecutionException):
provider.create_server(name="name", image="ERROR", flavor=configuration["flavor"],
network=configuration["network"])
with self.assertRaises(AttributeError):
network=configuration["network"], security_groups=[])
with self.assertRaises(ExecutionException):
provider.create_server(name="name", flavor="ERROR", image=configuration["image"],
network=configuration["network"])
with self.assertRaises(AttributeError):
network=configuration["network"], security_groups=[])
with self.assertRaises(ExecutionException):
provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
network="ERROR")
with self.assertRaises(AttributeError):
network="ERROR", security_groups=[])
with self.assertRaises(ExecutionException):
provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
network=configuration["network"], key_name="ERROR")
network=configuration["network"], key_name="ERROR", security_groups=[])

def test_create_keypair_create_delete_false_delete(self):
for provider in PROVIDERS:
Expand All @@ -141,8 +152,8 @@ def test_active_server_methods(self):
with self.subTest(provider.NAME):
with ProviderServer(provider, "bibigrid_test_server", configuration,
"bibigrid_test_keypair") as provider_server:
floating_ip = provider.create_floating_ip(provider.get_external_network(configuration["network"]),
provider_server)
floating_ip = provider.attach_available_floating_ip(
provider.get_external_network(configuration["network"]), provider_server)
server_list = provider.list_servers()
self.assertEqual(SERVER_KEYS, set(provider_server.keys()))
self.assertEqual("bibigrid_test_keypair", provider_server["key_name"])
Expand Down Expand Up @@ -209,7 +220,7 @@ def test_get_image_mismatch(self):
with self.subTest(provider.NAME):
self.assertIsNone(provider.get_image_by_id_or_name("NONE"))

if os.environ.get("OS_SNAPSHOT"):
if CONFIGURATIONS[0].get("snapshot_image"):
def test_get_snapshot(self):
for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
with self.subTest(provider.NAME):
Expand Down
10 changes: 0 additions & 10 deletions tests/resources/infrastructure_cloud.yml

This file was deleted.

9 changes: 4 additions & 5 deletions tests/startup_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ def suppress_stdout():
unittest.TextTestRunner(verbosity=2).run(suite)

# Provider-Test
## Configuration needs to contain providers and infrastructures
if os.environ.get("OS_KEY_NAME"):
suite = unittest.TestLoader().discover("./provider", pattern='test_*.py')
with suppress_stdout():
unittest.TextTestRunner(verbosity=2).run(suite)
# Configuration needs to contain providers and infrastructures
suite = unittest.TestLoader().discover("./provider", pattern='test_*.py')
with suppress_stdout():
unittest.TextTestRunner(verbosity=2).run(suite)
Loading

0 comments on commit 7856940

Please sign in to comment.