diff --git a/bibigrid/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py index c2c95219..1c84f456 100644 --- a/bibigrid/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -64,6 +64,7 @@ def generate_site_file_yaml(user_roles): host_dict["vars_files"] = host_dict["vars_files"] + user_role.get("varsFiles", []) host_dict["roles"] = host_dict["roles"] + [{"role": role["name"], "tags": role.get("tags", [])} for role in user_role["roles"]] + return site_yaml @@ -280,26 +281,6 @@ def get_cidrs(configurations): return all_cidrs -def get_ansible_roles(ansible_roles, log): - """ - Checks if ansible_roles have all necessary values and returns True if so. - @param ansible_roles: ansible_roles from master configuration (first configuration) - @param log: - @return: list of valid ansible_roles - """ - ansible_roles_yaml = [] - for ansible_role in (ansible_roles or []): - if ansible_role.get("file") and ansible_role.get("hosts"): - ansible_role_dict = {"file": ansible_role["file"], "hosts": ansible_role["hosts"]} - for key in ["name", "vars", "vars_file"]: - if ansible_role.get(key): - ansible_role_dict[key] = ansible_role[key] - ansible_roles_yaml.append(ansible_role_dict) - else: - log.warning("Ansible role %s had neither galaxy,git nor url. Not added.", ansible_role) - return ansible_roles_yaml - - def get_ansible_galaxy_roles(ansible_galaxy_roles, log): """ Checks if ansible_galaxy_role have all necessary values and adds it to the return list if so. diff --git a/tests/test_ansible_configurator.py b/tests/test_ansible_configurator.py index 9bd92df0..ade2e245 100644 --- a/tests/test_ansible_configurator.py +++ b/tests/test_ansible_configurator.py @@ -29,19 +29,69 @@ def test_generate_site_file_yaml_empty(self): 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml([])) - def test_generate_site_file_yaml_role(self): - custom_roles = [{"file": "file", "hosts": "hosts", "name": "name", "vars": "vars", "vars_file": "varsFile"}] + def test_generate_site_file_yaml_master_role(self): + user_roles = [{'hosts': ['master'], 'roles': [{'name': 'resistance_nextflow', 'tags': ['rn']}]}] # vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] site_yaml = [{'become': 'yes', 'hosts': 'master', - 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}, 'additional/name'], - 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}, + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}, + {'role': 'resistance_nextflow', 'tags': ['rn']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'vpngtw', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'workers', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(user_roles)) + + def test_generate_site_file_yaml_vpngtw_role(self): + user_roles = [{'hosts': ['vpngtw'], 'roles': [{'name': 'resistance_nextflow'}], 'varsFiles': ['vars/rn']}] + # vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] + site_yaml = [{'become': 'yes', 'hosts': 'master', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'vpngtw', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}, + {'role': 'resistance_nextflow', 'tags': []}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'vars/rn']}, + {'become': 'yes', 'hosts': 'workers', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(user_roles)) + + def test_generate_site_file_yaml_workers_role(self): + user_roles = [{'hosts': ['workers'], 'roles': [{'name': 'resistance_nextflow'}]}] + # vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] + site_yaml = [{'become': 'yes', 'hosts': 'master', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'vpngtw', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'workers', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}, + {'role': 'resistance_nextflow', 'tags': []}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(user_roles)) + + def test_generate_site_file_yaml_all_role(self): + user_roles = [ + {'hosts': ['master', 'vpngtw', 'workers'], 'roles': [{'name': 'resistance_nextflow', 'tags': ['rn']}], + 'varsFiles': ['vars/rn']}] + # vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] + site_yaml = [{'become': 'yes', 'hosts': 'master', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}, + {'role': 'resistance_nextflow', 'tags': ['rn']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'vars/rn']}, {'become': 'yes', 'hosts': 'vpngtw', - 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}, 'additional/name'], - 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}, + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}, + {'role': 'resistance_nextflow', 'tags': ['rn']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'vars/rn']}, {'become': 'yes', 'hosts': 'workers', - 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}, 'additional/name'], - 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}] - self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(custom_roles)) + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}, + {'role': 'resistance_nextflow', 'tags': ['rn']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'vars/rn']}] + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(user_roles)) def test_generate_common_configuration_false(self): cidrs = "42" @@ -173,7 +223,6 @@ def test_generate_common_configuration_ext_nfs_shares(self): default_user, startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] - print(generated_common_configuration) self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_ide(self): @@ -202,32 +251,6 @@ def test_generate_common_configuration_ide(self): common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) - def test_generate_common_configuration_ansible_roles_mock(self): - cidrs = "42" - ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] - cluster_id = "21" - default_user = "ubuntu" - ssh_user = "test" - configuration = [{"ansibleRoles": ansible_roles}] - generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) - self.assertEqual(ansible_roles, generated_common_configuration["ansible_roles"]) - - def test_generate_common_configuration_ansible_galaxy_roles(self): - cidrs = "42" - cluster_id = "21" - default_user = "ubuntu" - ssh_user = "test" - galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] - configuration = [{"ansibleGalaxyRoles": galaxy_roles}] - generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) - self.assertEqual(galaxy_roles, generated_common_configuration["ansible_galaxy_roles"]) - @patch("bibigrid.core.utility.ansible_configurator.to_instance_host_dict") def test_generate_ansible_hosts(self, mock_instance_host_dict): cluster_id = "21" @@ -273,30 +296,6 @@ def test_get_cidrs(self): expected = [{'cloud_identifier': 13, 'provider_cidrs': [21]}] self.assertEqual(expected, ansible_configurator.get_cidrs(configuration)) - def test_get_ansible_roles_empty(self): - self.assertEqual([], ansible_configurator.get_ansible_roles([], startup.LOG)) - - def test_get_ansible_roles(self): - ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] - self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - - def test_get_ansible_roles_add(self): - ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] - ansible_roles_add = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file", "additional"]}] - self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles_add, startup.LOG)) - - def test_get_ansible_roles_minus(self): - ansible_roles = [{elem: elem for elem in ["file", "hosts"]}] - self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - - def test_get_ansible_roles_mismatch_hosts(self): - ansible_roles = [{"file": "file"}] - self.assertEqual([], ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - - def test_get_ansible_roles_mismatch_file(self): - ansible_roles = [{"hosts": "hosts"}] - self.assertEqual([], ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - def test_get_ansible_galaxy_roles_empty(self): self.assertEqual([], ansible_configurator.get_ansible_galaxy_roles([], startup.LOG)) @@ -349,26 +348,23 @@ def test_write_yaml_alias(self, mock_yaml): @patch("bibigrid.core.utility.ansible_configurator.generate_common_configuration_yaml") @patch("bibigrid.core.actions.list_clusters.dict_clusters") @patch("bibigrid.core.utility.ansible_configurator.generate_ansible_hosts_yaml") - @patch("bibigrid.core.utility.ansible_configurator.get_ansible_roles") @patch("bibigrid.core.utility.ansible_configurator.generate_site_file_yaml") @patch("bibigrid.core.utility.ansible_configurator.write_yaml") @patch("bibigrid.core.utility.ansible_configurator.get_cidrs") - def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, mock_list, + def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_hosts, mock_list, mock_common, mock_worker, mock_write): mock_cidrs.return_value = 421 mock_list.return_value = {2: 422} - mock_roles.return_value = 423 provider = MagicMock() provider.cloud_specification = {"auth": {"username": "Default"}} - configuration = [{"sshUser": 42, "ansibleRoles": 21}] + configuration = [{"sshUser": 42, "userRoles": 21}] cluster_id = 2 ansible_configurator.configure_ansible_yaml([provider], configuration, cluster_id, startup.LOG) mock_worker.assert_called_with(configuration, startup.LOG) mock_common.assert_called_with(cidrs=421, configurations=configuration, cluster_id=cluster_id, ssh_user=42, default_user="Default", log=startup.LOG) mock_hosts.assert_called_with(42, configuration, cluster_id, startup.LOG) - mock_site.assert_called_with(423) - mock_roles.assert_called_with(21, startup.LOG) + mock_site.assert_called_with(21) mock_cidrs.assert_called_with(configuration) mock_write.assert_called() expected = [call(aRP.WORKER_SPECIFICATION_FILE, mock_worker(), startup.LOG, False),