Update tests

.gitignore

@@ -10,8 +10,7 @@ resources/playbook/ansible_hosts
 resources/playbook/vars/
 resources/playbook/host_vars/
 resources/playbook/group_vars/
-tests/resources/*
-!test/resources/test_configuration.yml
+resources/tests/bibigrid_test.yml
 
 # any log files
 *.log

bibigrid/core/actions/create.py

@@ -196,7 +196,6 @@ def start_vpn_or_master(self, configuration, provider):
         # create a server and block until it is up and running
         server = provider.create_server(name=name, flavor=flavor, key_name=self.key_name, image=image, network=network,
                                         volumes=volumes, security_groups=configuration["security_groups"], wait=True)
-        print("MASTER", server)
         configuration["private_v4"] = server["private_v4"]
         self.log.debug(f"Created Server {name}: {server['private_v4']}.")
         # get mac address for given private address

bibigrid/core/utility/ansible_configurator.py

@@ -55,9 +55,7 @@ def generate_site_file_yaml(custom_roles):
     @param custom_roles: ansibleRoles given by the config
     @return: site_yaml (dict)
     """
-    site_yaml = [{'hosts': 'master', "pre_tasks": [
-        {"name": "Print ansible.cfg timeout", "command": "ansible-config dump | grep 'DEFAULT_TIMEOUT'",
-         "register": "ansible_cfg_output"}, {"debug": {"msg": "{{ ansible_cfg_output.stdout }}"}}], "become": "yes",
+    site_yaml = [{'hosts': 'master', "become": "yes",
                   "vars_files": VARS_FILES, "roles": MASTER_ROLES},
                  {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": vpngtw_ROLES},
                  {"hosts": "workers", "become": "yes", "vars_files": VARS_FILES, "roles": WORKER_ROLES}]  # ,
@@ -173,7 +171,6 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us
     """
     master_configuration = configurations[0]
     log.info("Generating common configuration file...")
-    # print(configuration.get("slurmConf", {}))
     common_configuration_yaml = {"bibigrid_version": __version__,
                                  "auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id,
                                  "cluster_cidrs": cidrs, "default_user": default_user,

bibigrid/core/utility/handler/configuration_handler.py

@@ -92,6 +92,7 @@ def get_clouds_files(log):
         clouds_public = clouds_public_yaml.get(CLOUD_PUBLIC_ROOT_KEY)
         if not clouds_public:
             log.warning("%s is not valid. Must contain key '%s'", CLOUDS_PUBLIC_YAML, CLOUD_PUBLIC_ROOT_KEY)
+
     return clouds, clouds_public
 
 

bibigrid/core/utility/validate_configuration.py

@@ -181,7 +181,6 @@ def __init__(self, configurations, providers, log):
         self.log = log
         self.configurations = configurations
         self.providers = providers
-
         self.required_resources_dict = {
             provider.cloud_specification['identifier']: {'total_cores': 0, 'floating_ips': 0, 'instances': 0,
                                                          'total_ram': 0, 'volumes': 0, 'volume_gigabytes': 0,
@@ -227,11 +226,11 @@ def check_master_vpn_worker(self):
         """
         self.log.info("Checking master/vpn")
         success = True
-        if not self.configurations[0].get("masterInstance"):
+        if not self.configurations[0].get("masterInstance") or self.configurations[0].get("vpnInstance"):
             self.log.warning(f"{self.configurations[0].get('cloud')} has no master instance!")
             success = False
         for configuration in self.configurations[1:]:
-            if not configuration.get("vpnInstance"):
+            if not configuration.get("vpnInstance") or configuration.get("masterInstance"):
                 self.log.warning(f"{configuration.get('cloud')} has no vpn instance!")
                 success = False
         return success
@@ -356,9 +355,9 @@ def check_volumes(self):
                         else:
                             self.log.info(f"Snapshot '{volume_name_or_id}' found on "
                                           f"{provider.cloud_specification['identifier']}.")
-                            self.required_resources_dict[provider.cloud_specification['identifier']]["Volumes"] += 1
+                            self.required_resources_dict[provider.cloud_specification['identifier']]["volumes"] += 1
                             self.required_resources_dict[provider.cloud_specification['identifier']][
-                                "VolumeGigabytes"] += snapshot["size"]
+                                "volume_gigabytes"] += snapshot["size"]
                     else:
                         self.log.info(f"Volume '{volume_name_or_id}' found on "
                                       f"{provider.cloud_specification['identifier']}.")

resources/tests/bibigrid_test_example.yml

@@ -0,0 +1,9 @@
+- infrastructure: # former mode.
+  cloud: #credentials # name of clouds.yaml entry
+
+  flavor: de.NBI small
+  image: ^Ubuntu 22\.04 LTS \(.*\)$
+  sshUser: ubuntu
+  network: # network
+
+  snapshotImage: # name of a snapshot

tests/provider/test_provider.py

@@ -6,6 +6,8 @@
 import os
 import unittest
 
+from bibigrid.core import startup
+from bibigrid.core.utility import image_selection
 import bibigrid.core.utility.paths.basic_path as bP
 from bibigrid.core.utility.handler import configuration_handler
 from bibigrid.core.utility.handler import provider_handler
@@ -58,8 +60,8 @@
                'encrypted', 'multiattach', 'availability_zone', 'source_volid', 'user_id',
                'os-vol-tenant-attr:tenant_id'}
 
-FREE_RESOURCES_KEYS = {'total_cores', 'floating_ips', 'instances', 'total_ram', 'Volumes', 'VolumeGigabytes',
-                       'Snapshots', 'Backups', 'BackupGigabytes'}
+FREE_RESOURCES_KEYS = {'total_cores', 'floating_ips', 'instances', 'total_ram', 'volumes', 'volume_gigabytes',
+                       'snapshots', 'backups', 'backup_gigabytes'}
 
 KEYPAIR = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDORPauyW3O7M4Uk8/Qo557h2zxd9fwByljG9S1/zHKIEzOMcOBb7WUSmyNa5XHh5IB0/" \
           "BTsQvSag/O9IAhax2wlp9A2za6EkALYiRdEXeGOMNORw8yylRBqzLluKTErZ5sKYxENf1WGHsE3ifzct0G/moEPmIkixTHR9fZrZgOzQwj" \
@@ -70,7 +72,7 @@
 
 CONFIGURATIONS = configuration_handler.read_configuration(logging,
                                                           os.path.join(bP.ROOT_PATH,
-                                                                       "tests/resources/infrastructure_cloud.yml"))
+                                                                       "resources/tests/bibigrid_test.yml"))
 PROVIDERS = provider_handler.get_providers(CONFIGURATIONS, logging)
 
 
@@ -82,6 +84,9 @@ class ProviderServer:
     def __init__(self, provider, name, configuration, key_name=None):
         self.provider = provider
         self.name = name
+        self.log = startup.LOG
+        configuration["image"] = image_selection.select_image(provider=provider, image=configuration["image"],
+                                                              log=self.log)
         self.server_dict = provider.create_server(name=self.name, flavor=configuration["flavor"],
                                                   image=configuration["image"], network=configuration["network"],
                                                   key_name=key_name, security_groups=[])
@@ -166,6 +171,7 @@ def test_active_server_methods(self):
     def test_get_external_network(self):
         for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
             with self.subTest(provider.NAME):
+                print("FIRE", provider.get_external_network(configuration["network"]))
                 self.assertTrue(provider.get_external_network(configuration["network"]))
                 with self.assertRaises(TypeError):
                     provider.get_external_network("ERROR")
@@ -220,16 +226,17 @@ def test_get_image_mismatch(self):
             with self.subTest(provider.NAME):
                 self.assertIsNone(provider.get_image_by_id_or_name("NONE"))
 
-    if CONFIGURATIONS[0].get("snapshot_image"):
+    if CONFIGURATIONS[0].get("snapshotImage"):
         def test_get_snapshot(self):
             for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
                 with self.subTest(provider.NAME):
                     self.assertEqual(SNAPSHOT_KEYS, set(provider.get_volume_snapshot_by_id_or_name(
-                        configuration["snapshot_image"]).keys()))
+                        configuration["snapshotImage"]).keys()))
 
         def test_create_volume_from_snapshot(self):
             for provider, configuration in zip(PROVIDERS, CONFIGURATIONS):
                 with self.subTest(provider.NAME):
-                    volume_id = provider.create_volume_from_snapshot(configuration["snapshot_image"])
+                    volume_id = provider.create_volume_from_snapshot(configuration["snapshotImage"])
+                    print(volume_id)
                     volume = provider.get_volume_by_id_or_name(volume_id)
                     self.assertEqual(VOLUME_KEYS, set(volume.keys()))

tests/test_ansible_configurator.py

@@ -5,9 +5,10 @@
 from unittest.mock import MagicMock, Mock, patch, call, mock_open, ANY
 
 import bibigrid.core.utility.paths.ansible_resources_path as aRP
-from bibigrid.core.utility.yaml_dumper import NoAliasSafeDumper
 from bibigrid.core import startup
+from bibigrid.core.actions import version
 from bibigrid.core.utility import ansible_configurator
+from bibigrid.core.utility.yaml_dumper import NoAliasSafeDumper
 
 
 class TestAnsibleConfigurator(TestCase):
@@ -48,10 +49,12 @@ def test_generate_common_configuration_false(self):
         default_user = "ubuntu"
         ssh_user = "test"
         configuration = [{}]
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False,
-                                     'enable_nfs': False, 'enable_slurm': False, 'enable_zabbix': False,
-                                     'local_dns_lookup': False, 'local_fs': False, 'slurm': True,
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': False, 'enable_nfs': False,
+                                     'enable_slurm': False, 'enable_zabbix': False, 'local_dns_lookup': False,
+                                     'local_fs': False, 'slurm': True,
                                      'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm',
                                                     'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600,
                                                                            'TreeWidth': 128},
@@ -72,9 +75,11 @@ def test_generate_common_configuration_true(self):
         ssh_user = "test"
         configuration = [
             {elem: "True" for elem in ["localFS", "localDNSlookup", "useMasterAsCompute", "slurm", "zabbix", "ide"]}]
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': 'True',
-                                     'enable_nfs': False, 'enable_slurm': 'True', 'enable_zabbix': 'True',
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': 'True', 'enable_nfs': False,
+                                     'enable_slurm': 'True', 'enable_zabbix': 'True',
                                      'ide_conf': {'build': False, 'ide': False, 'port_end': 8383, 'port_start': 8181,
                                                   'workspace': '${HOME}'}, 'local_dns_lookup': 'True',
                                      'local_fs': 'True', 'slurm': 'True',
@@ -99,10 +104,12 @@ def test_generate_common_configuration_nfs_shares(self):
         cluster_id = "21"
         default_user = "ubuntu"
         ssh_user = "test"
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False,
-                                     'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False,
-                                     'ext_nfs_mounts': [], 'local_dns_lookup': False, 'local_fs': False,
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': False, 'enable_nfs': 'True',
+                                     'enable_slurm': False, 'enable_zabbix': False, 'ext_nfs_mounts': [],
+                                     'local_dns_lookup': False, 'local_fs': False,
                                      'nfs_mounts': [{'dst': '//vil/mil', 'src': '//vil/mil'},
                                                     {'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True,
                                      'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm',
@@ -123,10 +130,12 @@ def test_generate_common_configuration_nfs(self):
         cluster_id = "21"
         default_user = "ubuntu"
         ssh_user = "test"
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False,
-                                     'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False,
-                                     'ext_nfs_mounts': [], 'local_dns_lookup': False, 'local_fs': False,
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': False, 'enable_nfs': 'True',
+                                     'enable_slurm': False, 'enable_zabbix': False, 'ext_nfs_mounts': [],
+                                     'local_dns_lookup': False, 'local_fs': False,
                                      'nfs_mounts': [{'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True,
                                      'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm',
                                                     'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600,
@@ -146,9 +155,11 @@ def test_generate_common_configuration_ext_nfs_shares(self):
         cluster_id = "21"
         default_user = "ubuntu"
         ssh_user = "test"
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False,
-                                     'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False,
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': False, 'enable_nfs': 'True',
+                                     'enable_slurm': False, 'enable_zabbix': False,
                                      'ext_nfs_mounts': [{'dst': '/vil/mil', 'src': '/vil/mil'}],
                                      'local_dns_lookup': False, 'local_fs': False,
                                      'nfs_mounts': [{'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True,
@@ -162,6 +173,7 @@ def test_generate_common_configuration_ext_nfs_shares(self):
                                                                                                  default_user,
                                                                                                  startup.LOG)
         common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"]
+        print(generated_common_configuration)
         self.assertEqual(common_configuration_yaml, generated_common_configuration)
 
     def test_generate_common_configuration_ide(self):
@@ -170,10 +182,11 @@ def test_generate_common_configuration_ide(self):
         cluster_id = "21"
         default_user = "ubuntu"
         ssh_user = "test"
-        common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id,
-                                     'default_user': default_user, 'dns_server_list': ['8.8.8.8'],
-                                     'enable_ide': 'Some1', 'enable_nfs': False, 'enable_slurm': False,
-                                     'enable_zabbix': False,
+        common_configuration_yaml = {'auto_mount': False, 'bibigrid_version': version.__version__,
+                                     'cloud_scheduling': {'sshTimeout': 4}, 'cluster_cidrs': cidrs,
+                                     'cluster_id': cluster_id, 'default_user': default_user,
+                                     'dns_server_list': ['8.8.8.8'], 'enable_ide': 'Some1', 'enable_nfs': False,
+                                     'enable_slurm': False, 'enable_zabbix': False,
                                      'ide_conf': {'build': False, 'ide': False, 'key1': 'Some2', 'port_end': 8383,
                                                   'port_start': 8181, 'workspace': '${HOME}'},
                                      'local_dns_lookup': False, 'local_fs': False, 'slurm': True,

tests/test_configuration_handler.py

@@ -14,6 +14,7 @@ class TestConfigurationHandler(TestCase):
     """
     Class to test configuration_handler
     """
+
     # pylint: disable=R0904
     def test_get_list_by_name_none(self):
         configurations = [{}, {}]