Updated all tests

.gitignore

@@ -8,6 +8,8 @@ resources/playbook/ansible_hosts
 resources/playbook/vars/
 resources/playbook/host_vars/
 resources/playbook/group_vars/
+tests/resources/*
+!test/resources/test_configuration.yml
 
 # any log files
 *.log

bibigrid/core/actions/ide.py

@@ -23,7 +23,6 @@
 LOCALHOST = "127.0.0.1"
 
 
-
 def sigint_handler(caught_signal, frame):  # pylint: disable=unused-argument
     """
     Is called when SIGINT is thrown and terminates the program
@@ -82,7 +81,7 @@ def ide(cluster_id, master_provider, master_configuration, log):
                                                   ssh_pkey=used_private_key,
                                                   local_bind_address=(LOCALHOST, used_local_bind_address),
                                                   remote_bind_address=(LOCALHOST, REMOTE_BIND_ADDRESS)) as server:
-                    print("CTRL+C to close port forwarding when you are done.")
+                    log.log(42, "CTRL+C to close port forwarding when you are done.")
                     with server:
                         # opens in existing window if any default program exists
                         webbrowser.open(f"http://localhost:{used_local_bind_address}", new=2)

bibigrid/core/actions/list_clusters.py

@@ -8,7 +8,7 @@
 
 from bibigrid.core.actions import create
 
-SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)\d+-([a-zA-Z0-9]+)-\d+)$")
+SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)-([a-zA-Z0-9]+)-\d+)$")
 
 
 def dict_clusters(providers, log):
@@ -69,7 +69,7 @@ def log_list(cluster_id, providers, log):
             master_count, worker_count, vpn_count = get_size_overview(cluster_dict[cluster_id], log)
             log.log(42, f"\tCluster has {master_count} master, {vpn_count} vpngtw and {worker_count} regular workers. "
                        f"The cluster is spread over {vpn_count + master_count} reachable provider(s).")
-            pprint.pprint(cluster_dict[cluster_id])
+            log.log(42, pprint.pformat(cluster_dict[cluster_id]))
         else:
             log.info("Cluster with cluster-id {cluster_id} not found.")
             log.log(42, f"Cluster with cluster-id {cluster_id} not found.")

bibigrid/core/actions/terminate.py

@@ -34,11 +34,11 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
     cluster_security_group_state = []
     tmp_keyname = create.KEY_NAME.format(cluster_id=cluster_id)
     local_keypairs_deleted = delete_local_keypairs(tmp_keyname, log)
-    if not assume_yes and (
-            local_keypairs_deleted or input(f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. "
-                                            f"This might not be your cluster. Are you sure you want to terminate it?\n"
-                                            f"Any non-empty input to shutdown cluster {cluster_id}. "
-                                            f"Empty input to exit with cluster still alive:")):
+    if assume_yes or local_keypairs_deleted or input(
+            f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. "
+            f"This might not be your cluster. Are you sure you want to terminate it?\n"
+            f"Any non-empty input to shutdown cluster {cluster_id}. "
+            f"Empty input to exit with cluster still alive:"):
         for provider in providers:
             log.info("Terminating cluster %s on cloud %s", cluster_id, provider.cloud_specification['identifier'])
             server_list = provider.list_servers()
@@ -61,7 +61,7 @@ def terminate_servers(server_list, cluster_id, provider, log):
     """
     log.info("Deleting servers on provider %s...", provider.cloud_specification['identifier'])
     cluster_server_state = []
-    server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}+|(worker|vpngtw)-{cluster_id}+-\d+)$")
+    server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}|(worker|vpngtw)-{cluster_id}-\d+)$")
     for server in server_list:
         if server_regex.match(server["name"]):
             log.info("Trying to terminate Server %s on cloud %s.", server['name'],

bibigrid/core/utility/ansible_configurator.py

@@ -128,8 +128,7 @@ def write_host_and_group_vars(configurations, providers, cluster_id, log):  # py
             master_dict = {"name": name, "image": master["image"], "network": configuration["network"],
                            "network_cidrs": configuration["subnet_cidrs"], "floating_ip": configuration["floating_ip"],
                            "flavor": flavor_dict, "private_v4": configuration["private_v4"],
-                           "cloud_identifier": configuration["cloud_identifier"],
-                           "volumes": configuration["volumes"],
+                           "cloud_identifier": configuration["cloud_identifier"], "volumes": configuration["volumes"],
                            "fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)}
             if configuration.get("wireguard_peer"):
                 master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")}
@@ -165,8 +164,8 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us
     master_configuration = configurations[0]
     log.info("Generating common configuration file...")
     # print(configuration.get("slurmConf", {}))
-    common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False),
-                                 "cluster_id": cluster_id, "cluster_cidrs": cidrs, "default_user": default_user,
+    common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id,
+                                 "cluster_cidrs": cidrs, "default_user": default_user,
                                  "local_fs": master_configuration.get("localFS", False),
                                  "local_dns_lookup": master_configuration.get("localDNSlookup", False),
                                  "use_master_as_compute": master_configuration.get("useMasterAsCompute", True),
@@ -266,17 +265,18 @@ def get_cidrs(configurations):
     """
     all_cidrs = []
     for configuration in configurations:
-        subnet = configuration["subnet_cidrs"]
-        provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"], "provider_cidrs": subnet}
+        provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"],
+                          "provider_cidrs": configuration["subnet_cidrs"]}
         all_cidrs.append(provider_cidrs)
     return all_cidrs
 
 
 def get_ansible_roles(ansible_roles, log):
     """
     Checks if ansible_roles have all necessary values and returns True if so.
-    :param ansible_roles: ansible_roles from master configuration (first configuration)
-    :return: list of valid ansible_roles
+    @param ansible_roles: ansible_roles from master configuration (first configuration)
+    @param log:
+    @return: list of valid ansible_roles
     """
     ansible_roles_yaml = []
     for ansible_role in (ansible_roles or []):

bibigrid/core/utility/handler/ssh_handler.py

@@ -7,8 +7,8 @@
 import time
 
 import paramiko
-import yaml
 import sympy
+import yaml
 
 from bibigrid.core.utility import ansible_commands as aC
 from bibigrid.models.exceptions import ConnectionException, ExecutionException
@@ -107,10 +107,10 @@ def is_active(client, floating_ip_address, private_key, username, log, gateway,
             port = 22
             if gateway:
                 log.info(f"Using SSH Gateway {gateway.get('ip')}")
-                octets = {f'oct{enum+1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))}
+                octets = {f'oct{enum + 1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))}
                 port = int(sympy.sympify(gateway["portFunction"]).subs(dict(octets)))
-            client.connect(hostname=gateway.get("ip") or floating_ip_address, username=username,
-                           pkey=private_key, timeout=7, auth_timeout=5, port=port)
+            client.connect(hostname=gateway.get("ip") or floating_ip_address, username=username, pkey=private_key,
+                           timeout=7, auth_timeout=5, port=port)
             establishing_connection = False
             log.info(f"Successfully connected to {floating_ip_address}")
         except paramiko.ssh_exception.NoValidConnectionsError as exc:
@@ -158,7 +158,7 @@ def execute_ssh_cml_commands(client, commands, log):
     :param log:
     """
     for command in commands:
-        ssh_stdin, ssh_stdout, ssh_stderr = client.exec_command(command[0])  # pylint: disable=unused-variable
+        _, ssh_stdout, _ = client.exec_command(command[0])
         ssh_stdout.channel.set_combine_stderr(True)
         log.info(f"REMOTE: {command[1]}")
 

bibigrid/core/utility/validate_configuration.py

@@ -303,8 +303,8 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p
         flavor = provider.get_flavor(instance_type)
         if not flavor:
             self.log.warning("Flavor %s does not exist.", instance_type)
-            print("Available flavors:")
-            print("\n".join(provider.get_active_flavors()))
+            self.log.log(42, "Available flavors:")
+            self.log.log(42, "\n".join(provider.get_active_flavors()))
             return False
         type_max_disk_space = flavor["disk"]
         type_max_ram = flavor["ram"]

bibigrid/openstack/openstack_provider.py

@@ -121,6 +121,8 @@ def create_server(self, name, flavor, image, network, key_name=None, wait=True,
         except openstack.exceptions.BadRequestException as exc:
             if "is not active" in str(exc):
                 raise ImageDeactivatedException() from exc
+            if "Invalid key_name provided" in str(exc):
+                raise ExecutionException() from exc
             raise ConnectionError() from exc
         except openstack.exceptions.SDKException as exc:
             raise ExecutionException() from exc

tests/provider/test_provider.py

@@ -2,12 +2,14 @@
 Module containing integration and unit tests regarding the provider
 """
 
+import logging
 import os
 import unittest
 
-import bibigrid.core.utility.handler.configuration_handler as configurationHandler
-import bibigrid.core.utility.handler.provider_handler as providerHandler
 import bibigrid.core.utility.paths.basic_path as bP
+from bibigrid.core.utility.handler import configuration_handler
+from bibigrid.core.utility.handler import provider_handler
+from bibigrid.models.exceptions import ExecutionException
 
 SERVER_KEYS = {'id', 'name', 'flavor', 'image', 'block_device_mapping', 'location', 'volumes', 'has_config_drive',
                'host_id', 'progress', 'disk_config', 'power_state', 'task_state', 'vm_state', 'launched_at',
@@ -18,7 +20,12 @@
                'updated', 'user_id', 'tags', 'interface_ip', 'properties', 'hostId', 'config_drive', 'project_id',
                'tenant_id', 'region', 'cloud', 'az', 'OS-DCF:diskConfig', 'OS-EXT-AZ:availability_zone',
                'OS-SRV-USG:launched_at', 'OS-SRV-USG:terminated_at', 'OS-EXT-STS:task_state', 'OS-EXT-STS:vm_state',
-               'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached'}
+               'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached', 'OS-EXT-SRV-ATTR:ramdisk_id',
+               'max_count', 'trusted_image_certificates', 'OS-SCH-HNT:scheduler_hints',
+               'OS-EXT-SRV-ATTR:reservation_id', 'OS-EXT-SRV-ATTR:host', 'locked', 'host_status',
+               'OS-EXT-SRV-ATTR:hypervisor_hostname', 'OS-EXT-SRV-ATTR:launch_index',
+               'OS-EXT-SRV-ATTR:root_device_name', 'OS-EXT-SRV-ATTR:instance_name', 'OS-EXT-SRV-ATTR:user_data',
+               'min_count', 'OS-EXT-SRV-ATTR:kernel_id', 'OS-EXT-SRV-ATTR:hostname'}
 FLOATING_IP_KEYS = {'attached', 'fixed_ip_address', 'floating_ip_address', 'id', 'location', 'network', 'port',
                     'router', 'status', 'created_at', 'updated_at', 'description', 'revision_number', 'properties',
                     'port_id', 'router_id', 'project_id', 'tenant_id', 'floating_network_id', 'port_details',
@@ -39,7 +46,7 @@
               'is_protected', 'locations', 'properties', 'is_public', 'visibility', 'description',
               'owner_specified.openstack.md5', 'owner_specified.openstack.object', 'owner_specified.openstack.sha256',
               'os_hidden', 'os_hash_algo', 'os_hash_value', 'os_distro', 'os_version', 'schema', 'protected',
-              'metadata', 'created', 'updated', 'minDisk', 'minRam'}
+              'metadata', 'created', 'updated', 'minDisk', 'minRam', 'stores'}
 
 SNAPSHOT_KEYS = {'id', 'created_at', 'updated_at', 'name', 'description', 'volume_id', 'status', 'size', 'metadata',
                  'os-extended-snapshot-attributes:project_id', 'os-extended-snapshot-attributes:progress'}
@@ -61,9 +68,10 @@
           "MFbUTTukAiDf4jAgvJkg7ayE0MPapGpI/OhSK2gyN45VAzs2m7uykun87B491JagZ57qr16vt8vxGYpFCEe8QqAcrUszUPqyPrb0auA8bz" \
           "jO8S41Kx8FfG+7eTu4dQ0= user"
 
-CONFIGURATIONS = configurationHandler.read_configuration(
-    os.path.join(bP.ROOT_PATH, "tests/resources/infrastructure_cloud.yml"))
-PROVIDERS = providerHandler.get_providers(CONFIGURATIONS)
+CONFIGURATIONS = configuration_handler.read_configuration(logging,
+                                                          os.path.join(bP.ROOT_PATH,
+                                                                       "tests/resources/infrastructure_cloud.yml"))
+PROVIDERS = provider_handler.get_providers(CONFIGURATIONS, logging)
 
 
 class ProviderServer:
@@ -76,7 +84,7 @@ def __init__(self, provider, name, configuration, key_name=None):
         self.name = name
         self.server_dict = provider.create_server(name=self.name, flavor=configuration["flavor"],
                                                   image=configuration["image"], network=configuration["network"],
-                                                  key_name=key_name)
+                                                  key_name=key_name, security_groups=[])
 
     def __enter__(self):
         return self.server_dict
@@ -95,38 +103,41 @@ def test_get_free_resources(self):
             with self.subTest(provider.NAME):
                 free_dict = provider.get_free_resources()
                 self.assertEqual(FREE_RESOURCES_KEYS, set(free_dict.keys()))
-                for value in free_dict.values():
-                    self.assertLessEqual(0, value)
+                for key, value in free_dict.items():
+                    if key != "floating_ips":
+                        self.assertLessEqual(0, value)
 
     def test_server_start_type_error(self):
         for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
             with self.subTest(provider.NAME):
                 with self.assertRaises(TypeError):
                     provider.create_server(name="name", flavor=configuration["flavor"],
-                                           network=configuration["network"])
+                                           network=configuration["network"], security_groups=[])
                 with self.assertRaises(TypeError):
-                    provider.create_server(name="name", image=configuration["image"], network=configuration["network"])
+                    provider.create_server(name="name", image=configuration["image"], network=configuration["network"],
+                                           security_groups=[])
                 with self.assertRaises(TypeError):
                     provider.create_server(flavor=configuration["flavor"], image=configuration["image"],
-                                           network=configuration["network"])
+                                           security_groups=[], network=configuration["network"])
                 with self.assertRaises(TypeError):
-                    provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"])
+                    provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
+                                           security_groups=[])
 
     def test_server_start_attribute_error(self):
         for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
             with self.subTest(provider.NAME):
-                with self.assertRaises(AttributeError):
+                with self.assertRaises(ExecutionException):
                     provider.create_server(name="name", image="ERROR", flavor=configuration["flavor"],
-                                           network=configuration["network"])
-                with self.assertRaises(AttributeError):
+                                           network=configuration["network"], security_groups=[])
+                with self.assertRaises(ExecutionException):
                     provider.create_server(name="name", flavor="ERROR", image=configuration["image"],
-                                           network=configuration["network"])
-                with self.assertRaises(AttributeError):
+                                           network=configuration["network"], security_groups=[])
+                with self.assertRaises(ExecutionException):
                     provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
-                                           network="ERROR")
-                with self.assertRaises(AttributeError):
+                                           network="ERROR", security_groups=[])
+                with self.assertRaises(ExecutionException):
                     provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"],
-                                           network=configuration["network"], key_name="ERROR")
+                                           network=configuration["network"], key_name="ERROR", security_groups=[])
 
     def test_create_keypair_create_delete_false_delete(self):
         for provider in PROVIDERS:
@@ -141,8 +152,8 @@ def test_active_server_methods(self):
             with self.subTest(provider.NAME):
                 with ProviderServer(provider, "bibigrid_test_server", configuration,
                                     "bibigrid_test_keypair") as provider_server:
-                    floating_ip = provider.create_floating_ip(provider.get_external_network(configuration["network"]),
-                                                              provider_server)
+                    floating_ip = provider.attach_available_floating_ip(
+                        provider.get_external_network(configuration["network"]), provider_server)
                     server_list = provider.list_servers()
                 self.assertEqual(SERVER_KEYS, set(provider_server.keys()))
                 self.assertEqual("bibigrid_test_keypair", provider_server["key_name"])
@@ -209,7 +220,7 @@ def test_get_image_mismatch(self):
             with self.subTest(provider.NAME):
                 self.assertIsNone(provider.get_image_by_id_or_name("NONE"))
 
-    if os.environ.get("OS_SNAPSHOT"):
+    if CONFIGURATIONS[0].get("snapshot_image"):
         def test_get_snapshot(self):
             for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
                 with self.subTest(provider.NAME):

tests/startup_tests.py

@@ -39,8 +39,7 @@ def suppress_stdout():
         unittest.TextTestRunner(verbosity=2).run(suite)
 
     # Provider-Test
-    ## Configuration needs to contain providers and infrastructures
-    if os.environ.get("OS_KEY_NAME"):
-        suite = unittest.TestLoader().discover("./provider", pattern='test_*.py')
-        with suppress_stdout():
-            unittest.TextTestRunner(verbosity=2).run(suite)
+    # Configuration needs to contain providers and infrastructures
+    suite = unittest.TestLoader().discover("./provider", pattern='test_*.py')
+    with suppress_stdout():
+        unittest.TextTestRunner(verbosity=2).run(suite)