From d79d8047b6056f7dfc5685dcd334c857e65b1208 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 17 Oct 2023 18:29:37 +0200 Subject: [PATCH 01/34] updated most tests --- tests/test_ValidateConfiguration.py | 321 -------------------------- tests/test_validate_configuration.py | 333 +++++++++++++++++++++++++++ 2 files changed, 333 insertions(+), 321 deletions(-) delete mode 100644 tests/test_ValidateConfiguration.py create mode 100644 tests/test_validate_configuration.py diff --git a/tests/test_ValidateConfiguration.py b/tests/test_ValidateConfiguration.py deleted file mode 100644 index 96ffad26..00000000 --- a/tests/test_ValidateConfiguration.py +++ /dev/null @@ -1,321 +0,0 @@ -import os -from unittest import TestCase -from unittest.mock import Mock, patch, MagicMock, call - -from bibigrid.core.utility import validate_configuration - - -class TestValidateConfiguration(TestCase): - # pylint: disable=R0904 - def test_check_provider_data_count(self): - provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} - provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} - vc = validate_configuration - self.assertTrue(vc.check_provider_data([provider_data_1, provider_data_2], 2)) - self.assertFalse(vc.check_provider_data([provider_data_1, provider_data_2], 3)) - self.assertTrue(vc.check_provider_data([], 0)) - - def test_check_provider_data_unique(self): - provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} - provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} - vc = validate_configuration - self.assertTrue(vc.check_provider_data([provider_data_1, provider_data_2], 2)) - self.assertFalse(vc.check_provider_data([provider_data_1, provider_data_1], 2)) - self.assertTrue(vc.check_provider_data([], 0)) - - def test_check_master_vpn_worker_ordered(self): - master = {"masterInstance": "Value"} - vpn = {"vpnInstance": "Value"} - vpn_master = {} - vpn_master.update(master) - vpn_master.update(vpn) - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=[master]) - self.assertTrue(vc.check_master_vpn_worker()) - vc.configurations = [master, vpn] - self.assertTrue(vc.check_master_vpn_worker()) - vc.configurations = [vpn] - self.assertFalse(vc.check_master_vpn_worker()) - vc.configurations = [master, master] - self.assertFalse(vc.check_master_vpn_worker()) - - def test_check_master_vpn_worker_unique(self): - master = {"masterInstance": "Value"} - vpn = {"vpnInstance": "Value"} - vpn_master = {} - vpn_master.update(master) - vpn_master.update(vpn) - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=[vpn_master]) - self.assertFalse(vc.check_master_vpn_worker()) - vc.configurations = [master, vpn_master] - self.assertFalse(vc.check_master_vpn_worker()) - - def test_evaluate(self): - vc = validate_configuration - self.assertTrue(vc.evaluate("some", True)) - self.assertFalse(vc.evaluate("some", False)) - - def test_check_provider_connection(self): - mock = Mock() - mock.conn = False - vc = validate_configuration.ValidateConfiguration(providers=[mock], configurations=None) - self.assertFalse(vc.check_provider_connections()) - mock.conn = True - self.assertTrue(vc.check_provider_connections()) - - def test_check_instances_master(self): - vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"masterInstance": "42"}]) - with patch.object(vc, "check_instance") as mock: - vc.check_instances() - mock.assert_called_with("masterInstance", "42", "31") - - def test_check_instances_vpn(self): - vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"vpnInstance": "42"}]) - with patch.object(vc, "check_instance") as mock: - vc.check_instances() - mock.assert_called_with("vpnInstance", "42", "31") - - def test_check_instances_vpn_worker(self): - vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[ - {"masterInstance": "42", "workerInstances": ["42"]}]) - with patch.object(vc, "check_instance") as mock: - vc.check_instances() - mock.assert_called_with("workerInstance", "42", "31") - - def test_check_instances_vpn_master_missing(self): - vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{}]) - self.assertFalse(vc.check_instances()) - vc = validate_configuration.ValidateConfiguration(providers=["31"], - configurations=[{"workerInstances": ["42"]}]) - self.assertFalse(vc.check_instances()) - - def test_check_instances_vpn_master_count(self): - for i in range(3): - vc = validate_configuration.ValidateConfiguration(providers=["31"] * i, - configurations=[{"masterInstance": "42"}] * i) - # with patch.object(vc, "check_instance") as mock: - vc.check_instances() - self.assertTrue(vc.required_resources_dict["floating_ips"] == i) - - def test_check_instance_image_not_found(self): - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value=None) - self.assertFalse(vc.check_instance(None, {"count": 1, "image": 2}, provider)) - - def test_check_instance_image_not_active(self): - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value={"status": None}) - self.assertFalse(vc.check_instance(None, {"count": 1, "image": 2}, provider)) - - def test_check_instance_image_active_combination_call(self): - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value={"status": "active"}) - with patch.object(vc, "check_instance_type_image_combination") as mock: - vc.check_instance(42, {"count": 1, "image": 2, "type": 3}, provider) - mock.assert_called_with(3, {"status": "active"}, provider) - - def test_check_instance_image_not_found_count(self): - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value=None) - for i in range(1, 3): - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - vc.check_instance(None, {"count": i, "image": 2}, provider) - self.assertTrue(vc.required_resources_dict["instances"] == i) - - def test_check_instance_type_image_combination_has_enough_calls(self): - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - provider = MagicMock() - provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} - provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} - with patch.object(vc, "has_enough") as mock: - vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider) - self.assertEqual(call(42, 22, "Type de.NBI tiny", "disk space"), mock.call_args_list[0]) - self.assertEqual(call(32, 12, "Type de.NBI tiny", "ram"), mock.call_args_list[1]) - - def test_check_instance_type_image_combination_result(self): - provider = MagicMock() - provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} - provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - with patch.object(vc, "has_enough") as mock: - mock.side_effect = [True, True, False, False, True, False, False, True] - # True True - self.assertTrue(vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) - # False False - self.assertFalse(vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) - # True False - self.assertFalse(vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) - # False True - self.assertFalse(vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) - - def test_check_instance_type_image_combination_count(self): - for i in range(3): - provider = MagicMock() - provider.get_flavor.return_value = {"disk": 42, "ram": i * 32, "vcpus": i * 10} - provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} - vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) - with patch.object(vc, "has_enough") as mock: - vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider) - self.assertEqual(32 * i, vc.required_resources_dict["total_ram"]) - self.assertEqual(10 * i, vc.required_resources_dict["total_cores"]) - mock.assert_called_with(32 * i, 12, 'Type de.NBI tiny', 'ram') - - def test_check_volumes_none(self): - vc = validate_configuration.ValidateConfiguration(providers=[42], configurations=[{}]) - self.assertTrue(vc.check_volumes()) - - def test_check_volumes_mismatch(self): - provider = Mock() - provider.get_volume_by_id_or_name = MagicMock(return_value=None) - provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) - self.assertFalse(vc.check_volumes()) - - def test_check_volumes_match_snapshot(self): - provider = Mock() - provider.get_volume_by_id_or_name = MagicMock(return_value=None) - provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": 1}) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) - self.assertTrue(vc.check_volumes()) - - def test_check_volumes_match_snapshot_count(self): - for i in range(3): - provider = Mock() - provider.get_volume_by_id_or_name = MagicMock(return_value=None) - provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": i}) - vc = validate_configuration.ValidateConfiguration(providers=[provider] * i, - configurations=[{"masterMounts": ["Test"] * i}]) - self.assertTrue(vc.check_volumes()) - self.assertTrue(vc.required_resources_dict["Volumes"] == i) - self.assertTrue(vc.required_resources_dict["VolumeGigabytes"] == i ** 2) - - def test_check_volumes_match_volume(self): - provider = Mock() - provider.get_volume_by_id_or_name = MagicMock(return_value={"size": 1}) - provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) - self.assertTrue(vc.check_volumes()) - self.assertTrue(vc.required_resources_dict["Volumes"] == 0) - self.assertTrue(vc.required_resources_dict["VolumeGigabytes"] == 0) - - def test_check_network_none(self): - provider = Mock() - provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{}]) - self.assertFalse(vc.check_network()) - - def test_check_network_no_network(self): - provider = Mock() - provider.get_subnet_by_id_or_name = MagicMock(return_value="network") - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}]) - self.assertTrue(vc.check_network()) - provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") - - def test_check_network_no_network_mismatch_subnet(self): - provider = Mock() - provider.get_subnet_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}]) - self.assertFalse(vc.check_network()) - provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") - - def test_check_network_no_subnet_mismatch_network(self): - provider = Mock() - provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) - self.assertFalse(vc.check_network()) - provider.get_network_by_id_or_name.assert_called_with("network_name") - - def test_check_network_no_subnet(self): - provider = Mock() - provider.get_network_by_id_or_name = MagicMock(return_value="network") - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) - self.assertTrue(vc.check_network()) - provider.get_network_by_id_or_name.assert_called_with("network_name") - - def test_check_network_subnet_network(self): - provider = Mock() - provider.get_network_by_id_or_name = MagicMock(return_value="network") - provider.get_subnet_by_id_or_name = MagicMock(return_value="network") - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) - self.assertTrue(vc.check_network()) - provider.get_network_by_id_or_name.assert_called_with("network_name") - - def test_check_server_group_none(self): - provider = Mock() - provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{}]) - self.assertTrue(vc.check_server_group()) - - def test_check_server_group_mismatch(self): - provider = Mock() - provider.get_server_group_by_id_or_name = MagicMock(return_value=None) - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}]) - self.assertFalse(vc.check_server_group()) - provider.get_server_group_by_id_or_name.assert_called_with("GroupName") - - def test_check_server_group_match(self): - provider = Mock() - provider.get_server_group_by_id_or_name = MagicMock(return_value="Group") - vc = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}]) - self.assertTrue(vc.check_server_group()) - provider.get_server_group_by_id_or_name.assert_called_with("GroupName") - - def test_check_quotas_true(self): - provider = MagicMock() - provider.cloud_specification = {"auth": {"project_name": "name"}, "identifier": "identifier"} - test_dict = {'total_cores': 42, 'floating_ips': 42, 'instances': 42, 'total_ram': 42, - 'Volumes': 42, 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} - provider.get_free_resources.return_value = test_dict - vc = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None) - with patch.object(vc, "has_enough") as mock: - mock.side_effect = [True] * len(test_dict) - self.assertTrue(vc.check_quotas()) - provider.get_free_resources.assert_called() - for key in vc.required_resources_dict.keys(): - self.assertTrue(call(test_dict[key], vc.required_resources_dict[key], - "Project identifier", key) in mock.call_args_list) - - def test_check_quotas_false(self): - provider = MagicMock() - test_dict = {'total_cores': 42, 'floating_ips': 42, 'instances': 42, 'total_ram': 42, - 'Volumes': 42, 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} - provider.get_free_resources.return_value = test_dict - os.environ['OS_PROJECT_NAME'] = "name" - vc = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None) - with patch.object(vc, "has_enough") as mock: - mock.side_effect = [True] * (len(test_dict) - 1) + [False] - self.assertFalse(vc.check_quotas()) - provider.get_free_resources.assert_called() - mock.assert_called() - - def test_has_enough_lower(self): - vc = validate_configuration - self.assertTrue(vc.has_enough(2, 1, "", "")) - - def test_has_enough_equal(self): - vc = validate_configuration - self.assertTrue(vc.has_enough(2, 2, "", "")) - - def test_has_enough_higher(self): - vc = validate_configuration - self.assertFalse(vc.has_enough(1, 2, "", "")) diff --git a/tests/test_validate_configuration.py b/tests/test_validate_configuration.py new file mode 100644 index 00000000..c3611053 --- /dev/null +++ b/tests/test_validate_configuration.py @@ -0,0 +1,333 @@ +""" +Tests for validate configuration +""" + +import os +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock, call + +from bibigrid.core.utility import validate_configuration +from bibigrid.models.exceptions import ImageNotActiveException + + +class TestValidateConfiguration(TestCase): + """ + Class to test ValidateConfiguration + """ + # pylint: disable=R0904 + def test_check_provider_data_count(self): + provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} + provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} + self.assertTrue(validate_configuration.check_provider_data([provider_data_1, provider_data_2], 2, log=Mock())) + self.assertFalse(validate_configuration.check_provider_data([provider_data_1, provider_data_2], 3, log=Mock())) + self.assertTrue(validate_configuration.check_provider_data([], 0, log=Mock())) + + def test_check_provider_data_unique(self): + provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} + provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} + self.assertTrue(validate_configuration.check_provider_data([provider_data_1, provider_data_2], 2, log=Mock())) + self.assertFalse(validate_configuration.check_provider_data([provider_data_1, provider_data_1], 2, log=Mock())) + self.assertTrue(validate_configuration.check_provider_data([], 0, log=Mock())) + + @patch("bibigrid.core.utility.image_selection.select_image") + def test_check_master_vpn_worker_ordered(self, mock_select_image): # pylint: disable=unused-argument + master = {"masterInstance": "Value"} + vpn = {"vpnInstance": "Value"} + vpn_master = {} + vpn_master.update(master) + vpn_master.update(vpn) + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=[master], log=Mock()) + self.assertTrue(v_c.check_master_vpn_worker()) + v_c.configurations = [master, vpn] + self.assertTrue(v_c.check_master_vpn_worker()) + v_c.configurations = [vpn] + self.assertFalse(v_c.check_master_vpn_worker()) + v_c.configurations = [master, master] + self.assertFalse(v_c.check_master_vpn_worker()) + + def test_check_master_vpn_worker_unique(self): + master = {"masterInstance": "Value"} + vpn = {"vpnInstance": "Value"} + vpn_master = {} + vpn_master.update(master) + vpn_master.update(vpn) + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=[vpn_master], log=Mock()) + self.assertFalse(v_c.check_master_vpn_worker()) + v_c.configurations = [master, vpn_master] + self.assertFalse(v_c.check_master_vpn_worker()) + + def test_evaluate(self): + self.assertTrue(validate_configuration.evaluate("some", True, log=Mock())) + self.assertFalse(validate_configuration.evaluate("some", False, log=Mock())) + + def test_check_provider_connection(self): + mock = MagicMock() + mock.conn = False + v_c = validate_configuration.ValidateConfiguration(providers=[mock], configurations=None, log=Mock()) + self.assertFalse(v_c.check_provider_connections()) + mock.conn = True + self.assertTrue(v_c.check_provider_connections()) + + def test_check_instances_master(self): + v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"masterInstance": "42"}], + log=Mock()) + with patch.object(v_c, "check_instance") as mock: + v_c.check_instances() + mock.assert_called_with("masterInstance", "42", "31") + + def test_check_instances_vpn(self): + v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"vpnInstance": "42"}], + log=Mock()) + with patch.object(v_c, "check_instance") as mock: + v_c.check_instances() + mock.assert_called_with("vpnInstance", "42", "31") + + def test_check_instances_vpn_worker(self): + v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[ + {"masterInstance": "42", "workerInstances": ["42"]}], log=Mock()) + with patch.object(v_c, "check_instance") as mock: + v_c.check_instances() + mock.assert_called_with("workerInstance", "42", "31") + + def test_check_instances_vpn_master_missing(self): + v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{}], log=Mock()) + self.assertFalse(v_c.check_instances()) + v_c = validate_configuration.ValidateConfiguration(providers=["31"], + configurations=[{"workerInstances": ["42"]}], log=Mock()) + self.assertFalse(v_c.check_instances()) + + def test_check_instances_vpn_master_count(self): + for i in range(3): + v_c = validate_configuration.ValidateConfiguration(providers=["31"] * i, + configurations=[{"masterInstance": {"count": 1}}] * i, + log=Mock()) + # with patch.object(v_c, "check_instance") as mock: + v_c.check_instances() + print(v_c.required_resources_dict["floating_ips"]) + self.assertTrue(v_c.required_resources_dict["floating_ips"] == i) + + @patch("bibigrid.core.utility.image_selection.select_image") + def test_check_instance_image_not_found(self, mock_select_image): # pylint: disable=unused-argument + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) + provider = Mock() + provider.get_image_by_id_or_name = MagicMock(return_value=None) + self.assertFalse(v_c.check_instance(None, {"count": 1, "image": 2}, provider)) + + @patch("bibigrid.core.utility.image_selection.select_image") + def test_check_instance_image_not_active(self, mock_select_image): + mock_select_image.side_effect = ImageNotActiveException() + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) + provider = Mock() + provider.get_active_images.return_value = [] + provider.get_flavor = MagicMock(return_value={"disk": None, "ram": None}) + provider.get_image_by_id_or_name = MagicMock(return_value={"min_disk": None, "min_ram": None}) + self.assertFalse(v_c.check_instance(None, {"count": 1, "image": 2, "type": 3}, provider)) + + @patch("bibigrid.core.utility.image_selection.select_image") + def test_check_instance_image_active_combination_call(self, mock_select_image): + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) + provider = Mock() + provider.get_image_by_id_or_name = MagicMock(return_value={"status": "active"}) + with patch.object(v_c, "check_instance_type_image_combination") as mock: + v_c.check_instance(42, {"count": 1, "image": 2, "type": 3}, provider) + mock.assert_called_with(3, mock_select_image(2), provider) + + @patch("bibigrid.core.utility.image_selection.select_image") + def test_check_instance_image_not_found_count(self, mock_select_image): # pylint: disable=unused-argument + provider = Mock() + provider.get_image_by_id_or_name = MagicMock(return_value=None) + for i in range(1, 3): + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) + v_c.check_instance(None, {"count": i, "image": 2}, provider) + self.assertTrue(v_c.required_resources_dict["instances"] == i) + + def test_check_instance_type_image_combination_has_enough_calls(self): + log = Mock() + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=log) + provider = MagicMock() + provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} + provider.get_image_by_id_or_name.return_value = {"min_disk": 22, "min_ram": 12} + with patch.object(validate_configuration, "has_enough") as mock: + v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider) + self.assertEqual(call(42, 22, "Type de.NBI tiny", "disk space", log), mock.call_args_list[0]) + self.assertEqual(call(32, 12, "Type de.NBI tiny", "ram", log), mock.call_args_list[1]) + + def test_check_instance_type_image_combination_result(self): + provider = MagicMock() + provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} + provider.get_image_by_id_or_name.return_value = {"min_disk": 22, "min_ram": 12} + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) + with patch.object(validate_configuration, "has_enough") as mock: + mock.side_effect = [True, True, False, False, True, False, False, True] + # True True + self.assertTrue(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider)) + # False False + self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider)) + # True False + self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider)) + # False True + self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider)) + + def test_check_instance_type_image_combination_count(self): + for i in range(3): + provider = MagicMock() + provider.get_flavor.return_value = {"disk": 42, "ram": i * 32, "vcpus": i * 10} + provider.get_image_by_id_or_name.return_value = {"min_disk": 22, "min_ram": 12} + log = Mock() + v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=log) + with patch.object(validate_configuration, "has_enough") as mock: + v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", + provider=provider) + self.assertEqual(32 * i, v_c.required_resources_dict["total_ram"]) + self.assertEqual(10 * i, v_c.required_resources_dict["total_cores"]) + mock.assert_called_with(32 * i, 12, 'Type de.NBI tiny', 'ram', log) + + def test_check_volumes_none(self): + v_c = validate_configuration.ValidateConfiguration(providers=[42], configurations=[{}], log=Mock()) + self.assertTrue(v_c.check_volumes()) + + def test_check_volumes_mismatch(self): + provider = Mock() + provider.get_volume_by_id_or_name = MagicMock(return_value=None) + provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}], log=Mock()) + self.assertFalse(v_c.check_volumes()) + + def test_check_volumes_match_snapshot(self): + provider = Mock() + provider.get_volume_by_id_or_name = MagicMock(return_value=None) + provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": 1}) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}], log=Mock()) + self.assertTrue(v_c.check_volumes()) + + def test_check_volumes_match_snapshot_count(self): + for i in range(3): + provider = Mock() + provider.get_volume_by_id_or_name = MagicMock(return_value=None) + provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": i}) + v_c = validate_configuration.ValidateConfiguration(providers=[provider] * i, + configurations=[{"masterMounts": ["Test"] * i}], + log=Mock()) + self.assertTrue(v_c.check_volumes()) + self.assertTrue(v_c.required_resources_dict["Volumes"] == i) + self.assertTrue(v_c.required_resources_dict["VolumeGigabytes"] == i ** 2) + + def test_check_volumes_match_volume(self): + provider = Mock() + provider.get_volume_by_id_or_name = MagicMock(return_value={"size": 1}) + provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}], log=Mock()) + self.assertTrue(v_c.check_volumes()) + self.assertTrue(v_c.required_resources_dict["Volumes"] == 0) + self.assertTrue(v_c.required_resources_dict["VolumeGigabytes"] == 0) + + def test_check_network_none(self): + provider = Mock() + provider.get_network_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], configurations=[{}], log=Mock()) + self.assertFalse(v_c.check_network()) + + def test_check_network_no_network(self): + provider = Mock() + provider.get_subnet_by_id_or_name = MagicMock(return_value="network") + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"subnet": "subnet_name"}], log=Mock()) + self.assertTrue(v_c.check_network()) + provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") + + def test_check_network_no_network_mismatch_subnet(self): + provider = Mock() + provider.get_subnet_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"subnet": "subnet_name"}], log=Mock()) + self.assertFalse(v_c.check_network()) + provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") + + def test_check_network_no_subnet_mismatch_network(self): + provider = Mock() + provider.get_network_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}], log=Mock()) + self.assertFalse(v_c.check_network()) + provider.get_network_by_id_or_name.assert_called_with("network_name") + + def test_check_network_no_subnet(self): + provider = Mock() + provider.get_network_by_id_or_name = MagicMock(return_value="network") + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}], log=Mock()) + self.assertTrue(v_c.check_network()) + provider.get_network_by_id_or_name.assert_called_with("network_name") + + def test_check_network_subnet_network(self): + provider = Mock() + provider.get_network_by_id_or_name = MagicMock(return_value="network") + provider.get_subnet_by_id_or_name = MagicMock(return_value="network") + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}], log=Mock()) + self.assertTrue(v_c.check_network()) + provider.get_network_by_id_or_name.assert_called_with("network_name") + + def test_check_server_group_none(self): + provider = Mock() + provider.get_network_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], configurations=[{}], log=Mock()) + self.assertTrue(v_c.check_server_group()) + + def test_check_server_group_mismatch(self): + provider = Mock() + provider.get_server_group_by_id_or_name = MagicMock(return_value=None) + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"serverGroup": "GroupName"}], log=Mock()) + self.assertFalse(v_c.check_server_group()) + provider.get_server_group_by_id_or_name.assert_called_with("GroupName") + + def test_check_server_group_match(self): + provider = Mock() + provider.get_server_group_by_id_or_name = MagicMock(return_value="Group") + v_c = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"serverGroup": "GroupName"}], log=Mock()) + self.assertTrue(v_c.check_server_group()) + provider.get_server_group_by_id_or_name.assert_called_with("GroupName") + + def test_check_quotas_true(self): + provider = MagicMock() + provider.cloud_specification = {"auth": {"project_name": "name"}, "identifier": "identifier"} + test_dict = {'total_cores': 42, 'floating_ips': 42, 'instances': 42, 'total_ram': 42, 'Volumes': 42, + 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} + provider.get_free_resources.return_value = test_dict + v_c = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None, log=Mock()) + with patch.object(validate_configuration, "has_enough") as mock: + mock.side_effect = [True] * len(test_dict) + self.assertTrue(v_c.check_quotas()) + provider.get_free_resources.assert_called() + + def test_check_quotas_false(self): + provider = MagicMock() + test_dict = {'total_cores': 42, 'floating_ips': 42, 'instances': 42, 'total_ram': 42, 'Volumes': 42, + 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} + provider.get_free_resources.return_value = test_dict + os.environ['OS_PROJECT_NAME'] = "name" + v_c = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None, log=Mock()) + with patch.object(validate_configuration, "has_enough") as mock: + mock.side_effect = [True] * (len(test_dict) - 1) + [False] + self.assertFalse(v_c.check_quotas()) + provider.get_free_resources.assert_called() + mock.assert_called() + + def test_has_enough_lower(self): + self.assertTrue(validate_configuration.has_enough(2, 1, "", "", log=Mock())) + + def test_has_enough_equal(self): + self.assertTrue(validate_configuration.has_enough(2, 2, "", "", log=Mock())) + + def test_has_enough_higher(self): + self.assertFalse(validate_configuration.has_enough(1, 2, "", "", log=Mock())) From 1ce4f2e0775ee5793fad039b3373ae11dd4167d5 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 19 Oct 2023 11:43:32 +0200 Subject: [PATCH 02/34] fixed validate_configuration.py tests. --- tests/test_validate_configuration.py | 69 +++++++++++----------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/tests/test_validate_configuration.py b/tests/test_validate_configuration.py index c3611053..5deb65c8 100644 --- a/tests/test_validate_configuration.py +++ b/tests/test_validate_configuration.py @@ -14,6 +14,7 @@ class TestValidateConfiguration(TestCase): """ Class to test ValidateConfiguration """ + # pylint: disable=R0904 def test_check_provider_data_count(self): provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} @@ -70,14 +71,14 @@ def test_check_provider_connection(self): def test_check_instances_master(self): v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"masterInstance": "42"}], - log=Mock()) + log=Mock()) with patch.object(v_c, "check_instance") as mock: v_c.check_instances() mock.assert_called_with("masterInstance", "42", "31") def test_check_instances_vpn(self): v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"vpnInstance": "42"}], - log=Mock()) + log=Mock()) with patch.object(v_c, "check_instance") as mock: v_c.check_instances() mock.assert_called_with("vpnInstance", "42", "31") @@ -93,26 +94,19 @@ def test_check_instances_vpn_master_missing(self): v_c = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{}], log=Mock()) self.assertFalse(v_c.check_instances()) v_c = validate_configuration.ValidateConfiguration(providers=["31"], - configurations=[{"workerInstances": ["42"]}], log=Mock()) + configurations=[{"workerInstances": ["42"]}], log=Mock()) self.assertFalse(v_c.check_instances()) def test_check_instances_vpn_master_count(self): - for i in range(3): + for i in range(1, 4): v_c = validate_configuration.ValidateConfiguration(providers=["31"] * i, - configurations=[{"masterInstance": {"count": 1}}] * i, - log=Mock()) + configurations=[{"masterInstance": {"count": 1}}] + [ + {"vpnInstance": {"count": 1}}] * (i - 1), log=Mock()) # with patch.object(v_c, "check_instance") as mock: - v_c.check_instances() - print(v_c.required_resources_dict["floating_ips"]) + with patch.object(v_c, "check_instance", return_value=True): + v_c.check_instances() self.assertTrue(v_c.required_resources_dict["floating_ips"] == i) - @patch("bibigrid.core.utility.image_selection.select_image") - def test_check_instance_image_not_found(self, mock_select_image): # pylint: disable=unused-argument - v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value=None) - self.assertFalse(v_c.check_instance(None, {"count": 1, "image": 2}, provider)) - @patch("bibigrid.core.utility.image_selection.select_image") def test_check_instance_image_not_active(self, mock_select_image): mock_select_image.side_effect = ImageNotActiveException() @@ -132,15 +126,6 @@ def test_check_instance_image_active_combination_call(self, mock_select_image): v_c.check_instance(42, {"count": 1, "image": 2, "type": 3}, provider) mock.assert_called_with(3, mock_select_image(2), provider) - @patch("bibigrid.core.utility.image_selection.select_image") - def test_check_instance_image_not_found_count(self, mock_select_image): # pylint: disable=unused-argument - provider = Mock() - provider.get_image_by_id_or_name = MagicMock(return_value=None) - for i in range(1, 3): - v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=Mock()) - v_c.check_instance(None, {"count": i, "image": 2}, provider) - self.assertTrue(v_c.required_resources_dict["instances"] == i) - def test_check_instance_type_image_combination_has_enough_calls(self): log = Mock() v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=log) @@ -149,7 +134,7 @@ def test_check_instance_type_image_combination_has_enough_calls(self): provider.get_image_by_id_or_name.return_value = {"min_disk": 22, "min_ram": 12} with patch.object(validate_configuration, "has_enough") as mock: v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider) + provider=provider) self.assertEqual(call(42, 22, "Type de.NBI tiny", "disk space", log), mock.call_args_list[0]) self.assertEqual(call(32, 12, "Type de.NBI tiny", "ram", log), mock.call_args_list[1]) @@ -162,16 +147,16 @@ def test_check_instance_type_image_combination_result(self): mock.side_effect = [True, True, False, False, True, False, False, True] # True True self.assertTrue(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) + provider=provider)) # False False self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) + provider=provider)) # True False self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) + provider=provider)) # False True self.assertFalse(v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider)) + provider=provider)) def test_check_instance_type_image_combination_count(self): for i in range(3): @@ -182,7 +167,7 @@ def test_check_instance_type_image_combination_count(self): v_c = validate_configuration.ValidateConfiguration(providers=None, configurations=None, log=log) with patch.object(validate_configuration, "has_enough") as mock: v_c.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", - provider=provider) + provider=provider) self.assertEqual(32 * i, v_c.required_resources_dict["total_ram"]) self.assertEqual(10 * i, v_c.required_resources_dict["total_cores"]) mock.assert_called_with(32 * i, 12, 'Type de.NBI tiny', 'ram', log) @@ -196,7 +181,7 @@ def test_check_volumes_mismatch(self): provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}], log=Mock()) + configurations=[{"masterMounts": ["Test"]}], log=Mock()) self.assertFalse(v_c.check_volumes()) def test_check_volumes_match_snapshot(self): @@ -204,7 +189,7 @@ def test_check_volumes_match_snapshot(self): provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": 1}) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}], log=Mock()) + configurations=[{"masterMounts": ["Test"]}], log=Mock()) self.assertTrue(v_c.check_volumes()) def test_check_volumes_match_snapshot_count(self): @@ -213,8 +198,8 @@ def test_check_volumes_match_snapshot_count(self): provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": i}) v_c = validate_configuration.ValidateConfiguration(providers=[provider] * i, - configurations=[{"masterMounts": ["Test"] * i}], - log=Mock()) + configurations=[{"masterMounts": ["Test"] * i}], + log=Mock()) self.assertTrue(v_c.check_volumes()) self.assertTrue(v_c.required_resources_dict["Volumes"] == i) self.assertTrue(v_c.required_resources_dict["VolumeGigabytes"] == i ** 2) @@ -224,7 +209,7 @@ def test_check_volumes_match_volume(self): provider.get_volume_by_id_or_name = MagicMock(return_value={"size": 1}) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}], log=Mock()) + configurations=[{"masterMounts": ["Test"]}], log=Mock()) self.assertTrue(v_c.check_volumes()) self.assertTrue(v_c.required_resources_dict["Volumes"] == 0) self.assertTrue(v_c.required_resources_dict["VolumeGigabytes"] == 0) @@ -239,7 +224,7 @@ def test_check_network_no_network(self): provider = Mock() provider.get_subnet_by_id_or_name = MagicMock(return_value="network") v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}], log=Mock()) + configurations=[{"subnet": "subnet_name"}], log=Mock()) self.assertTrue(v_c.check_network()) provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") @@ -247,7 +232,7 @@ def test_check_network_no_network_mismatch_subnet(self): provider = Mock() provider.get_subnet_by_id_or_name = MagicMock(return_value=None) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}], log=Mock()) + configurations=[{"subnet": "subnet_name"}], log=Mock()) self.assertFalse(v_c.check_network()) provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") @@ -255,7 +240,7 @@ def test_check_network_no_subnet_mismatch_network(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value=None) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}], log=Mock()) + configurations=[{"network": "network_name"}], log=Mock()) self.assertFalse(v_c.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") @@ -263,7 +248,7 @@ def test_check_network_no_subnet(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value="network") v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}], log=Mock()) + configurations=[{"network": "network_name"}], log=Mock()) self.assertTrue(v_c.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") @@ -272,7 +257,7 @@ def test_check_network_subnet_network(self): provider.get_network_by_id_or_name = MagicMock(return_value="network") provider.get_subnet_by_id_or_name = MagicMock(return_value="network") v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}], log=Mock()) + configurations=[{"network": "network_name"}], log=Mock()) self.assertTrue(v_c.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") @@ -286,7 +271,7 @@ def test_check_server_group_mismatch(self): provider = Mock() provider.get_server_group_by_id_or_name = MagicMock(return_value=None) v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}], log=Mock()) + configurations=[{"serverGroup": "GroupName"}], log=Mock()) self.assertFalse(v_c.check_server_group()) provider.get_server_group_by_id_or_name.assert_called_with("GroupName") @@ -294,7 +279,7 @@ def test_check_server_group_match(self): provider = Mock() provider.get_server_group_by_id_or_name = MagicMock(return_value="Group") v_c = validate_configuration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}], log=Mock()) + configurations=[{"serverGroup": "GroupName"}], log=Mock()) self.assertTrue(v_c.check_server_group()) provider.get_server_group_by_id_or_name.assert_called_with("GroupName") From 661846296687b670743926a65898d04da35d7840 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 19 Oct 2023 12:24:39 +0200 Subject: [PATCH 03/34] Updated tests for startup.py --- bibigrid/core/startup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bibigrid/core/startup.py b/bibigrid/core/startup.py index 3a073f27..4676f44e 100755 --- a/bibigrid/core/startup.py +++ b/bibigrid/core/startup.py @@ -70,6 +70,7 @@ def run_action(args, configurations, config_path): exit_state = 0 try: providers = provider_handler.get_providers(configurations, LOG) + print(args) if providers: if args.list: LOG.info("Action list selected") From 2ff9b9899e4d6c18187eb86e72695acbdb63b9fa Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 19 Oct 2023 16:30:14 +0200 Subject: [PATCH 04/34] fixed bug in terminate that caused assume_yes to work as assume_no --- bibigrid/core/actions/terminate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bibigrid/core/actions/terminate.py b/bibigrid/core/actions/terminate.py index db812d7f..7a3fadb1 100644 --- a/bibigrid/core/actions/terminate.py +++ b/bibigrid/core/actions/terminate.py @@ -34,11 +34,11 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False): cluster_security_group_state = [] tmp_keyname = create.KEY_NAME.format(cluster_id=cluster_id) local_keypairs_deleted = delete_local_keypairs(tmp_keyname, log) - if not assume_yes and ( - local_keypairs_deleted or input(f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. " - f"This might not be your cluster. Are you sure you want to terminate it?\n" - f"Any non-empty input to shutdown cluster {cluster_id}. " - f"Empty input to exit with cluster still alive:")): + if assume_yes or local_keypairs_deleted or input( + f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. " + f"This might not be your cluster. Are you sure you want to terminate it?\n" + f"Any non-empty input to shutdown cluster {cluster_id}. " + f"Empty input to exit with cluster still alive:"): for provider in providers: log.info("Terminating cluster %s on cloud %s", cluster_id, provider.cloud_specification['identifier']) server_list = provider.list_servers() From e487ce0fb17259544d630cae2126dfc80c07c39c Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 19 Oct 2023 16:31:23 +0200 Subject: [PATCH 05/34] updated terminate_cluster tests. --- tests/test_startup.py | 60 +++++++++++++++++++-------------- tests/test_terminateCluster.py | 37 -------------------- tests/test_terminate_cluster.py | 50 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 63 deletions(-) mode change 100644 => 100755 tests/test_startup.py delete mode 100644 tests/test_terminateCluster.py create mode 100644 tests/test_terminate_cluster.py diff --git a/tests/test_startup.py b/tests/test_startup.py old mode 100644 new mode 100755 index 3baee61a..64f154d9 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -1,3 +1,7 @@ +""" +Modul to test startup +""" + from unittest import TestCase from unittest.mock import Mock, patch, MagicMock @@ -5,44 +9,48 @@ class TestStartup(TestCase): - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') - def test_provider(self, mock_get_providers): + """ + Class to test startup + """ + + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') + def test_provider_closing(self, mock_get_providers): args = Mock() - args.list_clusters = True + args.list = True args.version = False args.cluster_id = 12 provider = Mock provider.close = MagicMock() configurations = {} mock_get_providers.return_value = [provider] - with patch("bibigrid.core.actions.list_clusters.print_list_clusters") as mock_lc: + with patch("bibigrid.core.actions.list_clusters.log_list") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_get_providers.assert_called_with(configurations) + mock_get_providers.assert_called_with(configurations, startup.LOG) provider.close.assert_called() - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') def test_list_clusters(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() get_providers.return_value = [provider_mock] args = Mock() - args.list_clusters = True + args.list = True args.version = False args.cluster_id = 12 configurations = {} - with patch("bibigrid.core.actions.list_clusters.print_list_clusters") as mock_lc: + with patch("bibigrid.core.actions.list_clusters.log_list") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_lc.assert_called_with(12, [provider_mock]) + mock_lc.assert_called_with(12, [provider_mock], startup.LOG) - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') def test_check(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() get_providers.return_value = [provider_mock] args = Mock() - args.list_clusters = False + args.list = False args.version = False args.check = True args.cluster_id = 12 @@ -50,16 +58,16 @@ def test_check(self, get_providers): with patch("bibigrid.core.actions.check.check") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_lc.assert_called_with(configurations, [provider_mock]) + mock_lc.assert_called_with(configurations, [provider_mock], startup.LOG) - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') @patch('bibigrid.core.actions.create.Create') def test_create(self, mock_create, get_providers): provider_mock = Mock() provider_mock.close = Mock() get_providers.return_value = [provider_mock] args = Mock() - args.list_clusters = False + args.list = False args.version = False args.check = False args.create = True @@ -70,17 +78,17 @@ def test_create(self, mock_create, get_providers): creator.create = MagicMock(return_value=42) mock_create.return_value = creator self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_create.assert_called_with(providers=[provider_mock], configurations=configurations, debug=True, - config_path="") + mock_create.assert_called_with(providers=[provider_mock], configurations=configurations, log=startup.LOG, + debug=True, config_path="") creator.create.assert_called() - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') def test_terminate(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() get_providers.return_value = [provider_mock] args = Mock() - args.list_clusters = False + args.list = False args.version = False args.create = False args.check = False @@ -88,27 +96,27 @@ def test_terminate(self, get_providers): args.cluster_id = 12 args.debug = True configurations = {} - with patch("bibigrid.core.actions.terminateCluster.terminate_cluster") as mock_tc: + with patch("bibigrid.core.actions.terminate.terminate") as mock_tc: mock_tc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_tc.assert_called_with(12, [provider_mock], True) + mock_tc.assert_called_with(cluster_id=12, providers=[provider_mock], log=startup.LOG, debug=True) - @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.provider_handler.get_providers') @patch("bibigrid.core.actions.ide.ide") def test_ide(self, mock_ide, get_providers): - provider_mock = Mock() + provider_mock = MagicMock() provider_mock.close = Mock() get_providers.return_value = [provider_mock] args = Mock() - args.list_clusters = False + args.list = False args.version = False args.create = False args.check = False - args.terminate_cluster = False + args.terminate = False args.ide = True args.cluster_id = 12 args.debug = True - configurations = {} + configurations = [{"test_key": "test_value"}] mock_ide.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_ide.assert_called_with(12, [provider_mock], {}) + mock_ide.assert_called_with(12, provider_mock, {"test_key": "test_value"}, startup.LOG) diff --git a/tests/test_terminateCluster.py b/tests/test_terminateCluster.py deleted file mode 100644 index 4d9348a6..00000000 --- a/tests/test_terminateCluster.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase -from unittest.mock import MagicMock, patch - -from bibigrid.core.actions import create -from bibigrid.core.actions import terminate_cluster - - -class TestTerminate(TestCase): - - @patch("bibigrid.core.actions.terminate_cluster.terminate_output") - def test_terminate_cluster(self, _, mock_output): - provider = MagicMock() - provider.cloud_specification["auth"]["project_name"] = 32 - cluster_id = 42 - provider.list_servers.return_value = [ - {"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id), "id": 21}] - provider.delete_server.return_value = True - provider.delete_keypair.return_value = True - terminate_cluster.terminate_cluster(str(cluster_id), [provider], False) - provider.delete_server.assert_called_with(21) - provider.delete_keypair.assert_called_with( - create.KEY_PREFIX + provider.cloud_specification["auth"]["project_name"] + - create.SEPARATOR + str(cluster_id)) - mock_output.assert_called_with([provider.delete_server.return_value], - [provider.delete_keypair.return_value], str(cluster_id)) - - @patch("logging.info") - def test_terminate_cluster_none(self, _): - provider = MagicMock() - provider[0].specification["auth"]["project_name"] = "test_project_name" - cluster_id = 42 - provider.list_servers.return_value = [ - {"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id + 1), "id": 21}] - provider.delete_keypair.return_value = False - terminate_cluster.terminate_cluster(str(cluster_id), [provider], False) - provider.delete_server.assert_not_called() - provider.delete_keypair.assert_called_with('bibigrid42') # since keypair is not called diff --git a/tests/test_terminate_cluster.py b/tests/test_terminate_cluster.py new file mode 100644 index 00000000..949b0d1c --- /dev/null +++ b/tests/test_terminate_cluster.py @@ -0,0 +1,50 @@ +""" +Module to test terminate +""" +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from bibigrid.core import startup +from bibigrid.core.actions import create +from bibigrid.core.actions import terminate + + +class TestTerminate(TestCase): + """ + Class to test terminate. + """ + + @patch("bibigrid.core.actions.terminate.delete_local_keypairs") + @patch("bibigrid.core.actions.terminate.terminate_output") + def test_terminate(self, mock_output, mock_local): + mock_local.return_value = True + provider = MagicMock() + provider.cloud_specification["auth"]["project_name"] = 32 + cluster_id = 42 + provider.list_servers.return_value = [{"name": create.MASTER_IDENTIFIER(cluster_id=str(cluster_id)), "id": 21}] + provider.delete_server.return_value = True + provider.delete_keypair.return_value = True + provider.delete_security_group.return_value = True + provider.delete_application_credentials.return_value = True + terminate.terminate(str(cluster_id), [provider], startup.LOG, False, True) + provider.delete_server.assert_called_with(21) + provider.delete_keypair.assert_called_with(create.KEY_NAME.format(cluster_id=cluster_id)) + mock_output.assert_called_with([provider.delete_server.return_value], [provider.delete_keypair.return_value], + [provider.delete_security_group.return_value], + provider.delete_application_credentials.return_value, str(cluster_id), + startup.LOG) + + @patch("bibigrid.core.actions.terminate.delete_local_keypairs") + @patch("logging.info") + def test_terminate_none(self, _, mock_local): + mock_local.return_value = True + provider = MagicMock() + provider[0].specification["auth"]["project_name"] = "test_project_name" + cluster_id = 42 + provider.list_servers.return_value = [ + {"name": create.MASTER_IDENTIFIER(cluster_id=str(cluster_id + 1)), "id": 21}] + provider.delete_keypair.return_value = False + terminate.terminate(str(cluster_id), [provider], startup.LOG, False, True) + provider.delete_server.assert_not_called() + provider.delete_keypair.assert_called_with( + create.KEY_NAME.format(cluster_id=str(cluster_id))) # since keypair is not called From c3f79f160efc937eccebb07afc5e496dc30907f2 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 11:19:30 +0200 Subject: [PATCH 06/34] fixed formatting improved pylint --- bibigrid/core/utility/handler/ssh_handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bibigrid/core/utility/handler/ssh_handler.py b/bibigrid/core/utility/handler/ssh_handler.py index a705a405..d52cdde6 100644 --- a/bibigrid/core/utility/handler/ssh_handler.py +++ b/bibigrid/core/utility/handler/ssh_handler.py @@ -7,8 +7,8 @@ import time import paramiko -import yaml import sympy +import yaml from bibigrid.core.utility import ansible_commands as aC from bibigrid.models.exceptions import ConnectionException, ExecutionException @@ -107,10 +107,10 @@ def is_active(client, floating_ip_address, private_key, username, log, gateway, port = 22 if gateway: log.info(f"Using SSH Gateway {gateway.get('ip')}") - octets = {f'oct{enum+1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))} + octets = {f'oct{enum + 1}': int(elem) for enum, elem in enumerate(floating_ip_address.split("."))} port = int(sympy.sympify(gateway["portFunction"]).subs(dict(octets))) - client.connect(hostname=gateway.get("ip") or floating_ip_address, username=username, - pkey=private_key, timeout=7, auth_timeout=5, port=port) + client.connect(hostname=gateway.get("ip") or floating_ip_address, username=username, pkey=private_key, + timeout=7, auth_timeout=5, port=port) establishing_connection = False log.info(f"Successfully connected to {floating_ip_address}") except paramiko.ssh_exception.NoValidConnectionsError as exc: @@ -158,7 +158,7 @@ def execute_ssh_cml_commands(client, commands, log): :param log: """ for command in commands: - ssh_stdin, ssh_stdout, ssh_stderr = client.exec_command(command[0]) # pylint: disable=unused-variable + _, ssh_stdout, _ = client.exec_command(command[0]) ssh_stdout.channel.set_combine_stderr(True) log.info(f"REMOTE: {command[1]}") From dfbac4fb56139c2ead826fdeab2155102f7762d1 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 11:19:51 +0200 Subject: [PATCH 07/34] adapted tests --- tests/test_sshHandler.py | 104 ------------------------------- tests/test_ssh_handler.py | 128 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 104 deletions(-) delete mode 100644 tests/test_sshHandler.py create mode 100644 tests/test_ssh_handler.py diff --git a/tests/test_sshHandler.py b/tests/test_sshHandler.py deleted file mode 100644 index 7a9f05df..00000000 --- a/tests/test_sshHandler.py +++ /dev/null @@ -1,104 +0,0 @@ -import socket -from unittest import TestCase -from unittest.mock import mock_open, Mock, MagicMock, patch, call - -from paramiko.ssh_exception import NoValidConnectionsError - -import bibigrid.core.utility.handler.ssh_handler as sshHandler - - -class TestSshHandler(TestCase): - def test_get_add_ssh_public_key_commands_none(self): - ssh_public_key_files = [] - self.assertEqual([], sshHandler.get_add_ssh_public_key_commands(ssh_public_key_files)) - - def test_get_add_ssh_public_key_commands_line(self): - ssh_public_key_files = [42] - line = "42" - expected = [f"echo {line} >> .ssh/authorized_keys"] - with patch("builtins.open", mock_open(read_data=line)) as mock_file: - self.assertEqual(expected, sshHandler.get_add_ssh_public_key_commands(ssh_public_key_files)) - mock_file.assert_called_with(42) - - def test_copy_to_server_file(self): - sftp = Mock() - sftp.put = MagicMock(return_value=True) - with patch("os.path.isfile") as mock_isfile: - mock_isfile.return_value = True - sshHandler.copy_to_server(sftp, "Jim", "Joe") - sftp.put.assert_called_with("Jim", "Joe") - - @patch("os.listdir") - def test_copy_to_server_folder(self, mock_listdir): - sftp = Mock() - sftp.mkdir = MagicMock() - mock_listdir.return_value = [] - with patch("os.path.isfile") as mock_isfile: - mock_isfile.return_value = False - sshHandler.copy_to_server(sftp, "Jim", "Joe") - mock_listdir.assert_called_with("Jim") - sftp.mkdir.assert_called_with("Joe") - - @patch("logging.info") - def test_is_active(self, mock_log): - client = Mock() - client.connect = MagicMock(return_value=True) - self.assertFalse(sshHandler.is_active(client, 42, 32, 22, timeout=5)) - mock_log.assert_not_called() - - @patch("logging.info") - def test_is_active_second(self, mock_log): - client = Mock() - client.connect = MagicMock(side_effect=[NoValidConnectionsError({('127.0.0.1', 22): socket.error}), True]) - self.assertFalse(sshHandler.is_active(client, 42, 32, 22, timeout=5)) - mock_log.assert_called() - - @patch("logging.info") - def test_is_active_exception(self, mock_log): - client = Mock() - client.connect = MagicMock(side_effect=NoValidConnectionsError({('127.0.0.1', 22): socket.error})) - with self.assertRaises(ConnectionError): - sshHandler.is_active(client, 42, 32, 22, timeout=0) - client.connect.assert_called_with(hostname=42, username=22, pkey=32) - mock_log.assert_called() - - @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh_cml_commands") - @patch("paramiko.ECDSAKey.from_private_key_file") - @patch("paramiko.SSHClient") - def test_execute_ssh(self, mock_client, mock_paramiko_key, mock_exec): - mock_paramiko_key.return_value = 2 - client = Mock() - mock = Mock() - mock_client.return_value = mock - mock.__enter__ = client - mock.__exit__ = Mock(return_value=None) - with patch("bibigrid.core.utility.handler.sshHandler.is_active") as mock_active: - sshHandler.execute_ssh(42, 32, 22, [12], None) - mock_client.assert_called_with() - mock_active.assert_called_with(client=client(), floating_ip_address=42, username=22, private_key=2) - mock_exec.assert_called_with(client(), [12]) - mock_paramiko_key.assert_called_with(32) - - @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") - def test_ansible_preparation(self, mock_execute): - sshHandler.ansible_preparation(1, 2, 3, [], []) - mock_execute.assert_called_with(1, 2, 3, [] + sshHandler.ANSIBLE_SETUP, [(2, sshHandler.PRIVATE_KEY_FILE)]) - - @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") - def test_ansible_preparation_elem(self, mock_execute): - sshHandler.ansible_preparation(1, 2, 3, [42], [42]) - mock_execute.assert_called_with(1, 2, 3, sshHandler.ANSIBLE_SETUP + [42], - [42, (2, sshHandler.PRIVATE_KEY_FILE)]) - - @patch("logging.warning") - @patch("logging.info") - def test_execute_ssh_cml_commands(self, mock_log_info, mock_log_warning): - client = Mock() - stdout_mock = Mock() - stdout_mock.channel.recv_exit_status.side_effect = [0, 1] - stdout_mock.readlines.return_value = 49 - client.exec_command.return_value = (0, stdout_mock, 2) - commands = [42, 21] - sshHandler.execute_ssh_cml_commands(client, commands) - self.assertEqual([call('42:0')], mock_log_info.call_args_list) - self.assertEqual([call('21:1|49')], mock_log_warning.call_args_list) diff --git a/tests/test_ssh_handler.py b/tests/test_ssh_handler.py new file mode 100644 index 00000000..7c0cfd46 --- /dev/null +++ b/tests/test_ssh_handler.py @@ -0,0 +1,128 @@ +""" +Module to test ssh_handler +""" +import socket +from unittest import TestCase +from unittest.mock import mock_open, Mock, MagicMock, patch, call + +from paramiko.ssh_exception import NoValidConnectionsError + +from bibigrid.core import startup +from bibigrid.core.utility.handler import ssh_handler +from bibigrid.models.exceptions import ExecutionException, ConnectionException + + +class TestSshHandler(TestCase): + """ + Class to test ssh_handler + @todo: Test Gateway + """ + + def test_get_add_ssh_public_key_commands_none(self): + ssh_public_key_files = [] + self.assertEqual([], ssh_handler.get_add_ssh_public_key_commands(ssh_public_key_files)) + + def test_get_add_ssh_public_key_commands_line(self): + ssh_public_key_files = [42] + line = "42" + expected = [(f"echo {line} >> .ssh/authorized_keys", f"Add SSH Key {line}.")] + with patch("builtins.open", mock_open(read_data=line)) as mock_file: + self.assertEqual(expected, ssh_handler.get_add_ssh_public_key_commands(ssh_public_key_files)) + mock_file.assert_called_with(42, mode='r', encoding='UTF-8') + + def test_copy_to_server_file(self): + sftp = Mock() + sftp.put = MagicMock(return_value=True) + with patch("os.path.isfile") as mock_isfile: + mock_isfile.return_value = True + ssh_handler.copy_to_server(sftp, "Jim", "Joe", startup.LOG) + sftp.put.assert_called_with("Jim", "Joe") + + @patch("os.listdir") + def test_copy_to_server_folder(self, mock_listdir): + sftp = Mock() + sftp.mkdir = MagicMock() + mock_listdir.return_value = [] + with patch("os.path.isfile") as mock_isfile: + mock_isfile.return_value = False + ssh_handler.copy_to_server(sftp, "Jim", "Joe", startup.LOG) + mock_listdir.assert_called_with("Jim") + sftp.mkdir.assert_called_with("Joe") + + @patch("logging.info") + def test_is_active(self, mock_log): + client = Mock() + client.connect = MagicMock(return_value=True) + self.assertFalse(ssh_handler.is_active(client, 42, 32, 22, startup.LOG, {}, timeout=5)) + mock_log.assert_not_called() + + def test_is_active_on_second_attempt(self): + client = Mock() + client.connect = MagicMock(side_effect=[NoValidConnectionsError({('127.0.0.1', 22): socket.error}), True]) + self.assertFalse(ssh_handler.is_active(client, 42, 32, 22, startup.LOG, {}, timeout=5)) + + def test_is_active_exception(self): + client = Mock() + client.connect = MagicMock(side_effect=NoValidConnectionsError({('127.0.0.1', 22): socket.error})) + with self.assertRaises(ConnectionException): + ssh_handler.is_active(client, 42, 32, 22, startup.LOG, {}, timeout=0) + client.connect.assert_called_with(hostname=42, username=22, pkey=32, timeout=7, auth_timeout=5, port=22) + + @patch("bibigrid.core.utility.handler.ssh_handler.execute_ssh_cml_commands") + @patch("paramiko.ECDSAKey.from_private_key_file") + @patch("paramiko.SSHClient") + def test_execute_ssh(self, mock_client, mock_paramiko_key, mock_exec): + mock_paramiko_key.return_value = 2 + client = Mock() + mock = Mock() + mock_client.return_value = mock + mock.__enter__ = client + mock.__exit__ = Mock(return_value=None) + with patch("bibigrid.core.utility.handler.ssh_handler.is_active") as mock_active: + ssh_handler.execute_ssh(42, 32, 22, startup.LOG, {}, [12]) + mock_client.assert_called_with() + mock_active.assert_called_with(client=client(), floating_ip_address=42, username=22, private_key=2, + log=startup.LOG, gateway={}) + mock_exec.assert_called_with(client=client(), commands=[12], log=startup.LOG) + mock_paramiko_key.assert_called_with(32) + + @patch("bibigrid.core.utility.handler.ssh_handler.execute_ssh") + def test_ansible_preparation(self, mock_execute): + ssh_handler.ansible_preparation(1, 2, 3, startup.LOG, {}, [], []) + mock_execute.assert_called_with(1, 2, 3, startup.LOG, {}, ssh_handler.ANSIBLE_SETUP, + [(2, ssh_handler.PRIVATE_KEY_FILE)]) + + @patch("bibigrid.core.utility.handler.ssh_handler.execute_ssh") + def test_ansible_preparation_elem(self, mock_execute): + ssh_handler.ansible_preparation(1, 2, 3, startup.LOG, {}, [42], [42]) + mock_execute.assert_called_with(1, 2, 3, startup.LOG, {}, ssh_handler.ANSIBLE_SETUP + [42], + [42, (2, ssh_handler.PRIVATE_KEY_FILE)]) + + def test_execute_ssh_cml_commands(self): + client = Mock() + stdout_mock = Mock() + stdout_mock.channel.recv_exit_status.side_effect = [0, 0] + stdout_mock.readline.side_effect = ["First Line", "", "First Line", ""] + client.exec_command.return_value = (0, stdout_mock, 2) + commands = [(42, 0), (21, 1)] + ssh_handler.execute_ssh_cml_commands(client, commands, startup.LOG) + + stdout_mock.channel.recv_exit_status.assert_called() + stdout_mock.channel.recv_exit_status.call_count = 2 + stdout_mock.readline.assert_called() + assert stdout_mock.readline.call_count == 4 + client.exec_command.assert_has_calls([call(42), call(21)]) + assert client.exec_command.call_count == 2 + + def test_execute_ssh_cml_commands_execution_exception(self): + client = Mock() + stdout_mock = Mock() + stdout_mock.channel.recv_exit_status.side_effect = [0, 1] + stdout_mock.readline.side_effect = ["First Line", "", "First Line", ""] + client.exec_command.return_value = (0, stdout_mock, 2) + commands = [(42, 0), (21, 1)] + with self.assertRaises(ExecutionException): + ssh_handler.execute_ssh_cml_commands(client, commands, startup.LOG) + stdout_mock.channel.recv_exit_status.assert_called() + stdout_mock.readline.assert_called() + client.exec_command.assert_called_with(21) From c9b580f2520960b52af6953be300da3260b7b8cc Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 12:09:31 +0200 Subject: [PATCH 08/34] updated return threading test --- ...eturnThreading.py => test_return_threading.py} | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) rename tests/{test_returnThreading.py => test_return_threading.py} (63%) diff --git a/tests/test_returnThreading.py b/tests/test_return_threading.py similarity index 63% rename from tests/test_returnThreading.py rename to tests/test_return_threading.py index be9eeecd..8370f189 100644 --- a/tests/test_returnThreading.py +++ b/tests/test_return_threading.py @@ -1,17 +1,22 @@ +""" +Module to test return thread +""" from unittest import TestCase import bibigrid.models.return_threading as returnThreading -def test_method(x): - return (42, x) +def test_method(elem): + return 42, elem class TestReturnThread(TestCase): + """ + Class to test return thread + """ - def test_ReturnThread(self): - return_thread = returnThreading.ReturnThread(target=test_method, - args=[42]) + def test_return_thread(self): + return_thread = returnThreading.ReturnThread(target=test_method, args=[42]) return_thread.start() return_value = return_thread.join() self.assertTrue(return_value == (42, 42)) From ad3c1dd0eab99e04b7c8499c667e28b0be7cd719 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 12:15:59 +0200 Subject: [PATCH 09/34] updated provider_handler --- tests/test_providerHandler.py | 26 -------------------------- tests/test_provider_handler.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 26 deletions(-) delete mode 100644 tests/test_providerHandler.py create mode 100644 tests/test_provider_handler.py diff --git a/tests/test_providerHandler.py b/tests/test_providerHandler.py deleted file mode 100644 index 6735e77f..00000000 --- a/tests/test_providerHandler.py +++ /dev/null @@ -1,26 +0,0 @@ -from unittest import TestCase -from unittest.mock import MagicMock, patch - -import bibigrid.core.utility.handler.provider_handler as providerHandler - - -class TestProviderHandler(TestCase): - - @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specifications") - @patch("bibigrid.core.utility.handler.providerHandler.get_provider_list_by_name_list") - def test_get_providers(self, mock_provider_list, mock_get_cloud_specifications): - mock_get_cloud_specifications.return_value = True # for if not false - configurations = [{"infrastructure": "some"}] - mock_provider_list.return_value = 42 - with patch("bibigrid.core.utility.handler.configurationHandler.get_list_by_key") as mock_by_name: - self.assertEqual(42, providerHandler.get_providers(configurations)) - mock_by_name.assert_called_with(configurations, "infrastructure") - mock_get_cloud_specifications.assert_called_with(configurations) - - def test_get_provider_list_by_name_list(self): - keys = providerHandler.PROVIDER_NAME_DICT.keys() - values = [42] - with patch("bibigrid.core.utility.handler.providerHandler.get_provider_by_name") as mock_by_name: - mock_by_name.return_value = MagicMock(return_value=42) - self.assertEqual(providerHandler.get_provider_list_by_name_list(keys, "nonempty_specification"), values) - mock_by_name.assert_called_with(list(keys)[0]) diff --git a/tests/test_provider_handler.py b/tests/test_provider_handler.py new file mode 100644 index 00000000..6b37fec9 --- /dev/null +++ b/tests/test_provider_handler.py @@ -0,0 +1,33 @@ +""" +Module to test provider_handler +""" +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from bibigrid.core.utility.handler import provider_handler +from bibigrid.core import startup + + +class TestProviderHandler(TestCase): + """ + Class to test provider_handler + """ + + @patch("bibigrid.core.utility.handler.configuration_handler.get_cloud_specifications") + @patch("bibigrid.core.utility.handler.provider_handler.get_provider_list_by_name_list") + def test_get_providers(self, mock_provider_list, mock_get_cloud_specifications): + mock_get_cloud_specifications.return_value = True # for if not false + configurations = [{"infrastructure": "some"}] + mock_provider_list.return_value = 42 + with patch("bibigrid.core.utility.handler.configuration_handler.get_list_by_key") as mock_by_name: + self.assertEqual(42, provider_handler.get_providers(configurations, startup.LOG)) + mock_by_name.assert_called_with(configurations, "infrastructure") + mock_get_cloud_specifications.assert_called_with(configurations, startup.LOG) + + def test_get_provider_list_by_name_list(self): + keys = provider_handler.PROVIDER_NAME_DICT.keys() + values = [42] + with patch("bibigrid.core.utility.handler.provider_handler.get_provider_by_name") as mock_by_name: + mock_by_name.return_value = MagicMock(return_value=42) + self.assertEqual(provider_handler.get_provider_list_by_name_list(keys, "nonempty_specification"), values) + mock_by_name.assert_called_with(list(keys)[0]) From 0b441bb0c545328274ab13e494dfef731bd76b66 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 14:06:05 +0200 Subject: [PATCH 10/34] tests not finished yet --- ..._listClusters.py => test_list_clusters.py} | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) rename tests/{test_listClusters.py => test_list_clusters.py} (61%) diff --git a/tests/test_listClusters.py b/tests/test_list_clusters.py similarity index 61% rename from tests/test_listClusters.py rename to tests/test_list_clusters.py index 12744785..d6777146 100644 --- a/tests/test_listClusters.py +++ b/tests/test_list_clusters.py @@ -1,23 +1,29 @@ +""" +Module test list +""" from unittest import TestCase from unittest.mock import Mock +from bibigrid.core import startup from bibigrid.core.actions import create from bibigrid.core.actions import list_clusters -class TestDictClusters(TestCase): +class TestList(TestCase): + """ + Class test list + """ def test_setup(self): for identifier in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]: cluster_id = 42 - test_provider = Mock() - test_provider.name = "name" + provider = Mock() + provider.name = "name" + provider.cloud_specification = {"identifier": 21} cluster_dict = {} - server = {"name": identifier + create.SEPARATOR + str(cluster_id)} - self.assertEqual(str(cluster_id), - list_clusters.setup(server, - identifier, cluster_dict, test_provider)) + server = {"name": identifier(cluster_id=str(cluster_id))} + self.assertEqual(str(cluster_id), list_clusters.setup(server, identifier, cluster_dict, provider)) self.assertEqual({str(cluster_id): {'worker': [], 'vpngtw': []}}, cluster_dict) - self.assertEqual(test_provider, server["provider"]) + self.assertEqual(provider, server["provider"]) def test_setup_already(self): for identifier in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]: @@ -25,22 +31,18 @@ def test_setup_already(self): test_provider = Mock() test_provider.name = "name" cluster_dict = {str(cluster_id): {'worker': ["some"], 'vpngtw': ["some"]}} - server = {"name": identifier + create.SEPARATOR + str(cluster_id)} - self.assertEqual(str(cluster_id), - list_clusters.setup(server, - identifier, cluster_dict, test_provider)) + server = {"name": identifier(cluster_id=str(cluster_id))} + self.assertEqual(str(cluster_id), list_clusters.setup(server, identifier, cluster_dict, test_provider)) self.assertEqual({str(cluster_id): {'worker': ["some"], 'vpngtw': ["some"]}}, cluster_dict) self.assertEqual(test_provider, server["provider"]) def test_dict_clusters(self): cluster_id = 42 expected = {str(cluster_id): {'workers': [{'name': f'bibigrid-worker-{str(cluster_id)}', 'provider': 'Mock'}], - 'vpngtws': [ - {'name': f'bibigrid-vpngtw-{str(cluster_id)}', 'provider': 'Mock'}], + 'vpngtws': [{'name': f'bibigrid-vpngtw-{str(cluster_id)}', 'provider': 'Mock'}], 'master': {'name': f'bibigrid-master-{str(cluster_id)}', 'provider': 'Mock'}}} provider = Mock() - provider.list_servers.return_value = [{'name': identifier + create.SEPARATOR + str(cluster_id)} for identifier - in + provider.list_servers.return_value = [{'name': identifier(cluster_id=str(cluster_id))} for identifier in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]] - self.assertEqual(expected, list_clusters.dict_clusters([provider])) + self.assertEqual(expected, list_clusters.dict_clusters([provider], startup.LOG)) From 961bf0486d22b4964616461fcd35fb4ae5f0635e Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 16:03:01 +0200 Subject: [PATCH 11/34] Fixed server regex issue --- bibigrid/core/actions/list_clusters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bibigrid/core/actions/list_clusters.py b/bibigrid/core/actions/list_clusters.py index 1965dc39..99e1b6bc 100644 --- a/bibigrid/core/actions/list_clusters.py +++ b/bibigrid/core/actions/list_clusters.py @@ -8,7 +8,7 @@ from bibigrid.core.actions import create -SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)\d+-([a-zA-Z0-9]+)-\d+)$") +SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)-([a-zA-Z0-9]+)-\d+)$") def dict_clusters(providers, log): @@ -24,6 +24,8 @@ def dict_clusters(providers, log): servers = provider.list_servers() for server in servers: result = SERVER_REGEX.match(server["name"]) + print(server["name"]) + print(result) if result: identifier = result.group(4) or result.group(2) cluster_id = result.group(5) or result.group(3) From 83d4a2a4321e4d43f007b5f29b6c1d17ea9f64cf Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 16:03:30 +0200 Subject: [PATCH 12/34] test list clusters updated --- tests/test_list_clusters.py | 51 ++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tests/test_list_clusters.py b/tests/test_list_clusters.py index d6777146..b0a1ada4 100644 --- a/tests/test_list_clusters.py +++ b/tests/test_list_clusters.py @@ -13,36 +13,45 @@ class TestList(TestCase): """ Class test list """ + def test_setup(self): for identifier in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]: - cluster_id = 42 + cluster_id = "42" provider = Mock() provider.name = "name" - provider.cloud_specification = {"identifier": 21} + provider.cloud_specification = {"identifier": "21"} cluster_dict = {} - server = {"name": identifier(cluster_id=str(cluster_id))} - self.assertEqual(str(cluster_id), list_clusters.setup(server, identifier, cluster_dict, provider)) - self.assertEqual({str(cluster_id): {'worker': [], 'vpngtw': []}}, cluster_dict) - self.assertEqual(provider, server["provider"]) + server = {"name": identifier(cluster_id=cluster_id)} + list_clusters.setup(cluster_dict, str(cluster_id), server, provider) + + self.assertEqual({cluster_id: {'workers': [], 'vpngtws': []}}, cluster_dict) + self.assertEqual(provider.NAME, server["provider"]) + self.assertEqual("21", server["cloud_specification"]) def test_setup_already(self): for identifier in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]: - cluster_id = 42 - test_provider = Mock() - test_provider.name = "name" - cluster_dict = {str(cluster_id): {'worker': ["some"], 'vpngtw': ["some"]}} - server = {"name": identifier(cluster_id=str(cluster_id))} - self.assertEqual(str(cluster_id), list_clusters.setup(server, identifier, cluster_dict, test_provider)) - self.assertEqual({str(cluster_id): {'worker': ["some"], 'vpngtw': ["some"]}}, cluster_dict) - self.assertEqual(test_provider, server["provider"]) + cluster_id = "42" + provider = Mock() + provider.name = "name" + provider.cloud_specification = {"identifier": "21"} + cluster_dict = {cluster_id: {'workers': ["some"], 'vpngtws': ["some"]}} + server = {"name": identifier(cluster_id=cluster_id)} + list_clusters.setup(cluster_dict, cluster_id, server, provider) + + self.assertEqual({cluster_id: {'workers': ["some"], 'vpngtws': ["some"]}}, cluster_dict) + self.assertEqual(provider.NAME, server["provider"]) + self.assertEqual("21", server["cloud_specification"]) def test_dict_clusters(self): - cluster_id = 42 - expected = {str(cluster_id): {'workers': [{'name': f'bibigrid-worker-{str(cluster_id)}', 'provider': 'Mock'}], - 'vpngtws': [{'name': f'bibigrid-vpngtw-{str(cluster_id)}', 'provider': 'Mock'}], - 'master': {'name': f'bibigrid-master-{str(cluster_id)}', 'provider': 'Mock'}}} + cluster_id = "42" + expected = {cluster_id: { + 'workers': [{'cloud_specification': '21', 'name': f'bibigrid-worker-{cluster_id}-0', 'provider': 'Mock'}], + 'vpngtws': [{'cloud_specification': '21', 'name': f'bibigrid-vpngtw-{cluster_id}-0', 'provider': 'Mock'}], + 'master': {'cloud_specification': '21', 'name': f'bibigrid-master-{cluster_id}', 'provider': 'Mock'}}} provider = Mock() - provider.list_servers.return_value = [{'name': identifier(cluster_id=str(cluster_id))} for identifier in - [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, - create.MASTER_IDENTIFIER]] + provider.NAME = "Mock" + provider.cloud_specification = {"identifier": "21"} + provider.list_servers.return_value = [{'name': identifier(cluster_id=cluster_id) + "-0"} for identifier in + [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER]] + [ + {'name': create.MASTER_IDENTIFIER(cluster_id=cluster_id)}] self.assertEqual(expected, list_clusters.dict_clusters([provider], startup.LOG)) From d54ab93ab5d41717856e9e52d4647ca0bcdee229 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 16:04:35 +0200 Subject: [PATCH 13/34] fixed too open cluster_id regex --- bibigrid/core/actions/terminate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bibigrid/core/actions/terminate.py b/bibigrid/core/actions/terminate.py index 7a3fadb1..15a5a859 100644 --- a/bibigrid/core/actions/terminate.py +++ b/bibigrid/core/actions/terminate.py @@ -61,7 +61,7 @@ def terminate_servers(server_list, cluster_id, provider, log): """ log.info("Deleting servers on provider %s...", provider.cloud_specification['identifier']) cluster_server_state = [] - server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}+|(worker|vpngtw)-{cluster_id}+-\d+)$") + server_regex = re.compile(fr"^bibigrid-(master-{cluster_id}|(worker|vpngtw)-{cluster_id}-\d+)$") for server in server_list: if server_regex.match(server["name"]): log.info("Trying to terminate Server %s on cloud %s.", server['name'], From b78f63d2c05beeebd69840501081310244fc1bed Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 16:22:53 +0200 Subject: [PATCH 14/34] added missing "to" --- tests/test_list_clusters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_list_clusters.py b/tests/test_list_clusters.py index b0a1ada4..d7932d74 100644 --- a/tests/test_list_clusters.py +++ b/tests/test_list_clusters.py @@ -1,5 +1,5 @@ """ -Module test list +Module to test list """ from unittest import TestCase from unittest.mock import Mock @@ -11,7 +11,7 @@ class TestList(TestCase): """ - Class test list + Class to test list """ def test_setup(self): From 6ed924bdba3f91862a5daf993abc48630113fd41 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 16:23:29 +0200 Subject: [PATCH 15/34] fixed id_generation tests --- ...{test_idGeneration.py => test_id_generation.py} | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) rename tests/{test_idGeneration.py => test_id_generation.py} (82%) diff --git a/tests/test_idGeneration.py b/tests/test_id_generation.py similarity index 82% rename from tests/test_idGeneration.py rename to tests/test_id_generation.py index 6e694530..0e7c4009 100644 --- a/tests/test_idGeneration.py +++ b/tests/test_id_generation.py @@ -1,3 +1,6 @@ +""" +Module to test id_generation +""" from unittest import TestCase from unittest.mock import Mock, MagicMock, patch @@ -5,7 +8,10 @@ from bibigrid.core.utility import id_generation -class Test(TestCase): +class TestIDGeneration(TestCase): + """ + Class to test id_generation + """ def test_generate_cluster_id(self): """ @@ -26,10 +32,10 @@ def test_generate_safe_cluster_id(self, mock_generate_cluster_id): mock_is_unique.assert_called_with(21, [42]) def test_is_unique_cluster_id_duplicate(self): - cluster_id = 42 + cluster_id = "42" provider = Mock() provider.list_servers = MagicMock( - return_value=[{"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id)}]) + return_value=[{"name": create.MASTER_IDENTIFIER(cluster_id=cluster_id)}]) self.assertFalse(id_generation.is_unique_cluster_id(str(cluster_id), [provider])) provider.list_servers.assert_called() @@ -37,6 +43,6 @@ def test_is_unique_cluster_id_unique(self): cluster_id = 42 provider = Mock() provider.list_servers = MagicMock( - return_value=[{"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id + 1)}]) + return_value=[{"name": create.MASTER_IDENTIFIER(cluster_id=str(cluster_id + 1))}]) self.assertTrue(id_generation.is_unique_cluster_id(str(cluster_id), [provider])) provider.list_servers.assert_called() From f844f6862293339cf4068e4a16f63e4bfde141d7 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 17:31:24 +0200 Subject: [PATCH 16/34] renamed configuration handler to please linter --- tests/test_configurationHandler.py | 197 -------------------------- tests/test_configuration_handler.py | 208 ++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 197 deletions(-) delete mode 100644 tests/test_configurationHandler.py create mode 100644 tests/test_configuration_handler.py diff --git a/tests/test_configurationHandler.py b/tests/test_configurationHandler.py deleted file mode 100644 index f62161af..00000000 --- a/tests/test_configurationHandler.py +++ /dev/null @@ -1,197 +0,0 @@ -import os -from unittest import TestCase -from unittest.mock import patch, mock_open, MagicMock - -import bibigrid.core.utility.handler.configuration_handler as configurationHandler - - -class TestConfigurationHandler(TestCase): - # pylint: disable=R0904 - def test_get_list_by_name_none(self): - configurations = [{}, {}] - self.assertEqual([None, None], configurationHandler.get_list_by_key(configurations, "key1")) - self.assertEqual([], configurationHandler.get_list_by_key(configurations, "key1", False)) - - def test_get_list_by_name_empty(self): - configurations = [{"key1": "value1", "key2": "value1"}, {"key1": "value2"}] - self.assertEqual(["value1", "value2"], configurationHandler.get_list_by_key(configurations, "key1")) - self.assertEqual(["value1", "value2"], configurationHandler.get_list_by_key(configurations, "key1", False)) - self.assertEqual(["value1", None], configurationHandler.get_list_by_key(configurations, "key2")) - self.assertEqual(["value1"], configurationHandler.get_list_by_key(configurations, "key2", False)) - - @patch("os.path.isfile") - def test_read_configuration_no_file(self, mock_isfile): - mock_isfile.return_value = False - test = MagicMock() - configuration = "Test: 42" - expected_result = None - with patch("builtins.open", mock_open(test, read_data=configuration)): - result = configurationHandler.read_configuration("path") - mock_isfile.assert_called_with("path") - test.assert_not_called() - self.assertEqual(expected_result, result) - - @patch("os.path.isfile") - def test_read_configuration_file(self, mock_isfile): - mock_isfile.return_value = True - opener = MagicMock() - configuration = "Test: 42" - expected_result = {"Test": 42} - with patch("builtins.open", mock_open(opener, read_data=configuration)): - result = configurationHandler.read_configuration("path") - mock_isfile.assert_called_with("path") - opener.assert_called_with("path", "r") - self.assertEqual(expected_result, result) - - @patch("os.path.isfile") - def test_read_configuration_file_yaml_exception(self, mock_isfile): - mock_isfile.return_value = True - opener = MagicMock() - configuration = "]unbalanced brackets[" - expected_result = None - with patch("builtins.open", mock_open(opener, read_data=configuration)): - result = configurationHandler.read_configuration("path") - mock_isfile.assert_called_with("path") - opener.assert_called_with("path", "r") - self.assertEqual(expected_result, result) - - def test_find_file_in_folders_not_found_no_folder(self): - expected_result = None - result = configurationHandler.find_file_in_folders("true_file", []) - self.assertEqual(expected_result, result) - - def test_find_file_in_folders_not_found_no_file(self): - expected_result = None - with patch("os.path.isfile") as mock_isfile: - mock_isfile.return_value = False - result = configurationHandler.find_file_in_folders("false_file", ["or_false_folder"]) - self.assertEqual(expected_result, result) - mock_isfile.called_with(os.path.expanduser(os.path.join("or_false_folder", "false_file"))) - - @patch("os.path.isfile") - @patch("bibigrid.core.utility.handler.configurationHandler.read_configuration") - def test_find_file_in_folders(self, mock_read_configuration, mock_isfile): - expected_result = 42 - mock_isfile.return_value(True) - mock_read_configuration.return_value = 42 - result = configurationHandler.find_file_in_folders("true_file", ["true_folder"]) - self.assertEqual(expected_result, result) - mock_read_configuration.assert_called_with(os.path.expanduser(os.path.join("true_folder", "true_file"))) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files_none(self, mock_ffif): - mock_ffif.return_value = None - expected_result = None, None - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files_no_clouds_yaml(self, mock_ffif): - mock_ffif.side_effect = [None, {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] - expected_result = None, 42 - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files_no_public_clouds_yaml(self, mock_ffif): - mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 42}, None] - expected_result = 42, None - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files_no_root_key_public(self, mock_ffif): - mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 42}, {"name": 42}] - expected_result = 42, None - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files_no_root_key_cloud(self, mock_ffif): - mock_ffif.side_effect = [{"name": 42}, {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] - expected_result = None, 42 - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - - @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") - def test_get_cloud_files(self, mock_ffif): - mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 22}, - {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] - expected_result = 22, 42 - result = configurationHandler.get_clouds_files() - self.assertEqual(expected_result, result) - mock_ffif.assert_called_with(configurationHandler.CLOUDS_PUBLIC_YAML, configurationHandler.CLOUDS_YAML_PATHS) - - @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") - def test_get_cloud_specifications_none(self, mock_get_clouds_files, mock_get_clouds_specification): - mock_get_clouds_files.return_value = None, None - expected_result = [] - result = configurationHandler.get_cloud_specifications([{"cloud": 42}]) - self.assertEqual(expected_result, result) - mock_get_clouds_specification.assert_not_called() - mock_get_clouds_files.assert_called() - - @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") - def test_get_cloud_specifications_no_cloud_configuration_key(self, mock_get_clouds_files, - mock_get_clouds_specification): - mock_get_clouds_files.return_value = {"Some"}, {"Some"} - expected_result = [] - result = configurationHandler.get_cloud_specifications([{"no_cloud": 42}]) - self.assertEqual(expected_result, result) - mock_get_clouds_specification.assert_not_called() - mock_get_clouds_files.assert_called() - - @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") - def test_get_cloud_specifications_cloud(self, mock_get_clouds_files, mock_get_clouds_specification): - mock_get_clouds_files.return_value = {"1"}, {"2"} - mock_get_clouds_specification.return_value = 21 - expected_result = [21] - result = configurationHandler.get_cloud_specifications([{"cloud": 42}]) - self.assertEqual(expected_result, result) - mock_get_clouds_specification.assert_called_with(42, {"1"}, {"2"}) - mock_get_clouds_files.assert_called() - - @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") - def test_get_cloud_specifications_no_config(self, mock_get_clouds_files, mock_get_clouds_specification): - mock_get_clouds_files.return_value = {"1"}, {"2"} - mock_get_clouds_specification.return_value = 21 - expected_result = [] - result = configurationHandler.get_cloud_specifications([]) - self.assertEqual(expected_result, result) - mock_get_clouds_specification.assert_not_called() - mock_get_clouds_files.assert_called() - - def test_get_cloud_specification_no_matching_cloud(self): - expected_result = {} - result = configurationHandler.get_cloud_specification("some_name", {}, {"some_some": "public"}) - self.assertEqual(expected_result, result) - - def test_get_cloud_specification_cloud(self): - expected_result = {42: 42} - result = configurationHandler.get_cloud_specification("some_name", {"some_name": {42: 42}}, None) - self.assertEqual(expected_result, result) - - def test_get_cloud_specification_no_public_cloud(self): - expected_result = {42: 42, "profile": "name2"} - result = configurationHandler.get_cloud_specification("some_name", {"some_name": expected_result}, - {"not_name2": {21: 21}}) - self.assertEqual(expected_result, result) - - def test_get_cloud_specification(self): - cloud_private_specification = {42: 42, "profile": "name2", "test": {"recursive": "foo"}} - expected_result = {42: 42, "profile": "name2", "test": {"recursive": "foo"}, "additional": "value"} - result = configurationHandler.get_cloud_specification("some_name", {"some_name": cloud_private_specification}, - {"name2": {42: 21, "test": {"recursive": "oof"}, - "additional": "value"}}) - self.assertEqual(expected_result, result) - - def test_get_cloud_specification_type_exception(self): - cloud_private_specification = {42: 42, "profile": "name2", "test": {"recursive": "foo"}} - result = configurationHandler.get_cloud_specification("some_name", {"some_name": cloud_private_specification}, - {"name2": {42: 21, "test": ["recursive", 22], - "additional": "value"}}) - self.assertEqual({}, result) diff --git a/tests/test_configuration_handler.py b/tests/test_configuration_handler.py new file mode 100644 index 00000000..86564f84 --- /dev/null +++ b/tests/test_configuration_handler.py @@ -0,0 +1,208 @@ +""" +Module to test configuration_handler +""" + +import os +from unittest import TestCase +from unittest.mock import patch, mock_open, MagicMock + +from bibigrid.core import startup +from bibigrid.core.utility.handler import configuration_handler + + +class TestConfigurationHandler(TestCase): + """ + Class to test configuration_handler + """ + # pylint: disable=R0904 + def test_get_list_by_name_none(self): + configurations = [{}, {}] + self.assertEqual([None, None], configuration_handler.get_list_by_key(configurations, "key1")) + self.assertEqual([], configuration_handler.get_list_by_key(configurations, "key1", False)) + + def test_get_list_by_name_empty(self): + configurations = [{"key1": "value1", "key2": "value1"}, {"key1": "value2"}] + self.assertEqual(["value1", "value2"], configuration_handler.get_list_by_key(configurations, "key1")) + self.assertEqual(["value1", "value2"], configuration_handler.get_list_by_key(configurations, "key1", False)) + self.assertEqual(["value1", None], configuration_handler.get_list_by_key(configurations, "key2")) + self.assertEqual(["value1"], configuration_handler.get_list_by_key(configurations, "key2", False)) + + @patch("os.path.isfile") + def test_read_configuration_no_file(self, mock_isfile): + mock_isfile.return_value = False + test_open = MagicMock() + configuration = "Test: 42" + expected_result = [None] + with patch("builtins.open", mock_open(test_open, read_data=configuration)): + result = configuration_handler.read_configuration(startup.LOG, "path") + mock_isfile.assert_called_with("path") + test_open.assert_not_called() + self.assertEqual(expected_result, result) + + @patch("os.path.isfile") + def test_read_configuration_file(self, mock_isfile): + mock_isfile.return_value = True + opener = MagicMock() + configuration = "Test: 42" + expected_result = [{"Test": 42}] + with patch("builtins.open", mock_open(opener, read_data=configuration)): + result = configuration_handler.read_configuration(startup.LOG, "path") + mock_isfile.assert_called_with("path") + opener.assert_called_with("path", mode="r", encoding="UTF-8") + self.assertEqual(expected_result, result) + + @patch("os.path.isfile") + def test_read_configuration_file_yaml_exception(self, mock_isfile): + mock_isfile.return_value = True + opener = MagicMock() + configuration = "]unbalanced brackets[" + expected_result = [None] + with patch("builtins.open", mock_open(opener, read_data=configuration)): + result = configuration_handler.read_configuration(startup.LOG, "path") + mock_isfile.assert_called_with("path") + opener.assert_called_with("path", mode="r", encoding="UTF-8") + self.assertEqual(expected_result, result) + + def test_find_file_in_folders_not_found_no_folder(self): + expected_result = None + result = configuration_handler.find_file_in_folders("true_file", [], startup.LOG) + self.assertEqual(expected_result, result) + + def test_find_file_in_folders_not_found_no_file(self): + expected_result = None + with patch("os.path.isfile") as mock_isfile: + mock_isfile.return_value = False + result = configuration_handler.find_file_in_folders("false_file", ["or_false_folder"], startup.LOG) + self.assertEqual(expected_result, result) + mock_isfile.called_with(os.path.expanduser(os.path.join("or_false_folder", "false_file"))) + + @patch("os.path.isfile") + @patch("bibigrid.core.utility.handler.configuration_handler.read_configuration") + def test_find_file_in_folders(self, mock_read_configuration, mock_isfile): + expected_result = 42 + mock_isfile.return_value(True) + mock_read_configuration.return_value = 42 + result = configuration_handler.find_file_in_folders("true_file", ["true_folder"], startup.LOG) + self.assertEqual(expected_result, result) + mock_read_configuration.assert_called_with(startup.LOG, + os.path.expanduser(os.path.join("true_folder", "true_file")), False) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files_none(self, mock_ffif): + mock_ffif.return_value = None + expected_result = None, None + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files_no_clouds_yaml(self, mock_ffif): + mock_ffif.side_effect = [None, {configuration_handler.CLOUD_PUBLIC_ROOT_KEY: 42}] + expected_result = None, 42 + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files_no_public_clouds_yaml(self, mock_ffif): + mock_ffif.side_effect = [{configuration_handler.CLOUD_ROOT_KEY: 42}, None] + expected_result = 42, None + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files_no_root_key_public(self, mock_ffif): + mock_ffif.side_effect = [{configuration_handler.CLOUD_ROOT_KEY: 42}, {"name": 42}] + expected_result = 42, None + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files_no_root_key_cloud(self, mock_ffif): + mock_ffif.side_effect = [{"name": 42}, {configuration_handler.CLOUD_PUBLIC_ROOT_KEY: 42}] + expected_result = None, 42 + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + + @patch("bibigrid.core.utility.handler.configuration_handler.find_file_in_folders") + def test_get_cloud_files(self, mock_ffif): + mock_ffif.side_effect = [{configuration_handler.CLOUD_ROOT_KEY: 22}, + {configuration_handler.CLOUD_PUBLIC_ROOT_KEY: 42}] + expected_result = 22, 42 + result = configuration_handler.get_clouds_files(startup.LOG) + self.assertEqual(expected_result, result) + mock_ffif.assert_called_with(configuration_handler.CLOUDS_PUBLIC_YAML, configuration_handler.CLOUDS_YAML_PATHS, + startup.LOG) + + @patch("bibigrid.core.utility.handler.configuration_handler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configuration_handler.get_clouds_files") + def test_get_cloud_specifications_none(self, mock_get_clouds_files, mock_get_clouds_specification): + mock_get_clouds_files.return_value = None, None + expected_result = [] + result = configuration_handler.get_cloud_specifications([{"cloud": 42}], startup.LOG) + self.assertEqual(expected_result, result) + mock_get_clouds_specification.assert_not_called() + mock_get_clouds_files.assert_called() + + @patch("bibigrid.core.utility.handler.configuration_handler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configuration_handler.get_clouds_files") + def test_get_cloud_specifications_no_cloud_configuration_key(self, mock_get_clouds_files, + mock_get_clouds_specification): + mock_get_clouds_files.return_value = {"Some"}, {"Some"} + expected_result = [] + result = configuration_handler.get_cloud_specifications([{"no_cloud": 42}], startup.LOG) + self.assertEqual(expected_result, result) + mock_get_clouds_specification.assert_not_called() + mock_get_clouds_files.assert_called() + + @patch("bibigrid.core.utility.handler.configuration_handler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configuration_handler.get_clouds_files") + def test_get_cloud_specifications_cloud(self, mock_get_clouds_files, mock_get_clouds_specification): + mock_get_clouds_files.return_value = {"1": "1"}, {"2": "2"} + mock_get_clouds_specification.return_value = 21 + expected_result = [21] + result = configuration_handler.get_cloud_specifications([{"cloud": 42}], startup.LOG) + self.assertEqual(expected_result, result) + mock_get_clouds_specification.assert_called_with(42, {"1": "1"}, {"2": "2"}, startup.LOG) + mock_get_clouds_files.assert_called() + + @patch("bibigrid.core.utility.handler.configuration_handler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configuration_handler.get_clouds_files") + def test_get_cloud_specifications_no_config(self, mock_get_clouds_files, mock_get_clouds_specification): + mock_get_clouds_files.return_value = {"1": "1"}, {"2": "2"} + mock_get_clouds_specification.return_value = 21 + expected_result = [] + result = configuration_handler.get_cloud_specifications([], startup.LOG) + self.assertEqual(expected_result, result) + mock_get_clouds_specification.assert_not_called() + mock_get_clouds_files.assert_called() + + def test_get_cloud_specification_no_matching_cloud(self): + expected_result = {} + result = configuration_handler.get_cloud_specification("some_name", {}, {"some_some": "public"}, startup.LOG) + self.assertEqual(expected_result, result) + + def test_get_cloud_specification_cloud(self): + expected_result = {42: 42, "identifier": "some_name"} + result = configuration_handler.get_cloud_specification("some_name", {"some_name": {42: 42}}, None, startup.LOG) + self.assertEqual(expected_result, result) + + def test_get_cloud_specification_no_public_cloud(self): + expected_result = {42: 42, "profile": "name2", "identifier": "some_name"} + result = configuration_handler.get_cloud_specification("some_name", {"some_name": expected_result}, + {"not_name2": {21: 21}}, startup.LOG) + self.assertEqual(expected_result, result) + + def test_get_cloud_specification(self): + cloud_private_specification = {42: 42, "profile": "name2", "test": {"recursive": "oof"}} + expected_result = {42: 21, "profile": "name2", "test": {"recursive": "foo"}, "additional": "value", + "identifier": "some_name"} + result = configuration_handler.get_cloud_specification("some_name", {"some_name": cloud_private_specification}, + {"name2": {42: 21, "test": {"recursive": "foo"}, + "additional": "value"}}, startup.LOG) + self.assertEqual(expected_result, result) + + def test_get_cloud_specification_type_exception(self): + cloud_private_specification = {42: 42, "profile": "name2", "test": {"recursive": "foo"}} + result = configuration_handler.get_cloud_specification("some_name", {"some_name": cloud_private_specification}, + {"name2": {42: 21, "test": ["recursive", 22], + "additional": "value"}}, startup.LOG) + self.assertEqual({}, result) From 0c4a97fd8af58ffd4e70a3cfad9b94feb6ae499f Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 20 Oct 2023 17:31:43 +0200 Subject: [PATCH 17/34] removed unnecessary tests and updated remaining --- tests/test_check.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/tests/test_check.py b/tests/test_check.py index b3f04f31..fcaa5e11 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -1,34 +1,20 @@ +""" +Module to test check +""" from unittest import TestCase from unittest.mock import patch +from bibigrid.core import startup from bibigrid.core.actions import check -from bibigrid.core.utility import validate_configuration class TestCheck(TestCase): - - @patch("logging.info") - def test_check_true(self, mock_log): - providers = [42] - configurations = [32] - with patch.object(validate_configuration.ValidateConfiguration, "validate", return_value=True) as mock_validate: - self.assertFalse(check.check(configurations, providers)) - mock_validate.assert_called() - mock_log.assert_called_with("Total check returned True.") - - @patch("logging.info") - def test_check_false(self, mock_log): - providers = [42] - configurations = [32] - with patch.object(validate_configuration.ValidateConfiguration, "validate", - return_value=False) as mock_validate: - self.assertFalse(check.check(configurations, providers)) - mock_validate.assert_called() - mock_log.assert_called_with("Total check returned False.") - + """ + Class to test check + """ @patch("bibigrid.core.utility.validate_configuration.ValidateConfiguration") - def test_check_init(self, mock_validator): + def test_check_true(self, mock_validator): providers = [42] configurations = [32] - self.assertFalse(check.check(configurations, providers)) - mock_validator.assert_called_with(configurations, providers) + self.assertFalse(check.check(configurations, providers, startup.LOG)) + mock_validator.assert_called_once_with(configurations, providers, startup.LOG) From fbabea243ccdd165abfde022efeb5c33768ddcfc Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 26 Oct 2023 12:48:19 +0200 Subject: [PATCH 18/34] updated tests not finished yet --- tests/test_ansible_configurator.py | 274 +++++++++++++++++------------ 1 file changed, 165 insertions(+), 109 deletions(-) diff --git a/tests/test_ansible_configurator.py b/tests/test_ansible_configurator.py index db22411d..c3ae1851 100644 --- a/tests/test_ansible_configurator.py +++ b/tests/test_ansible_configurator.py @@ -7,129 +7,191 @@ import bibigrid.core.utility.ansible_configurator as ansibleConfigurator import bibigrid.core.utility.paths.ansible_resources_path as aRP import bibigrid.core.utility.yaml_dumper as yamlDumper +from bibigrid.core import startup + class TestAnsibleConfigurator(TestCase): """ Test ansible configurator test class """ + # pylint: disable=R0904 def test_generate_site_file_yaml_empty(self): - site_yaml = [{'hosts': 'master', "become": "yes", - "vars_files": ansibleConfigurator.VARS_FILES, "roles": ["common", "master"]}, - {"hosts": "worker", "become": "yes", "vars_files": - ansibleConfigurator.VARS_FILES, "roles": ["common", "worker"]}, - {"hosts": "vpngtw", "become": "yes", "vars_files": - ansibleConfigurator.VARS_FILES, "roles": ["common", "vpngtw"]}] + site_yaml = [{'become': 'yes', 'hosts': 'master', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'vpngtw', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}, + {'become': 'yes', 'hosts': 'workers', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] self.assertEqual(site_yaml, ansibleConfigurator.generate_site_file_yaml([])) def test_generate_site_file_yaml_role(self): custom_roles = [{"file": "file", "hosts": "hosts", "name": "name", "vars": "vars", "vars_file": "varsFile"}] - vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] - site_yaml = [{'hosts': 'master', "become": "yes", - "vars_files": vars_files, "roles": ["common", "master", "additional/name"]}, - {"hosts": "worker", "become": "yes", "vars_files": - vars_files, "roles": ["common", "worker", "additional/name"]}, - {"hosts": "vpngtw", "become": "yes", "vars_files": - vars_files, "roles": ["common", "vpngtw", "additional/name"]}] + # vars_files = ['vars/login.yml', 'vars/common_configuration.yml', 'varsFile'] + site_yaml = [{'become': 'yes', 'hosts': 'master', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-master']}, 'additional/name'], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}, + {'become': 'yes', 'hosts': 'vpngtw', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-vpngtw']}, 'additional/name'], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}, + {'become': 'yes', 'hosts': 'workers', + 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}, 'additional/name'], + 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}] self.assertEqual(site_yaml, ansibleConfigurator.generate_site_file_yaml(custom_roles)) - def test_generate_instances(self): - cluster_dict = object() - self.assertEqual(cluster_dict, ansibleConfigurator.generate_instances_yaml(cluster_dict)) - def test_generate_common_configuration_false(self): - cidrs = 42 - configuration = {} - common_configuration_yaml = {"cluster_cidrs": cidrs, - "local_fs": False, - "local_dns_lookup": False, - "use_master_as_compute": True, - "enable_slurm": False, - "enable_zabbix": False, - "enable_nfs": False, - "enable_ide": False - } - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + configuration = [{}] + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False, + 'enable_nfs': False, 'enable_slurm': False, 'enable_zabbix': False, + 'local_dns_lookup': False, 'local_fs': False, 'slurm': True, + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, + 'use_master_as_compute': True} + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + # munge key is randomly generated + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_true(self): - cidrs = 42 - configuration = {elem: "true" for elem in ["localFS", "localDNSlookup", "useMasterAsCompute", "slurm", - "zabbix", "ide"]} - common_configuration_yaml = {elem: "true" for elem in ["local_fs", "local_dns_lookup", "use_master_as_compute", - "enable_slurm", "enable_zabbix", "enable_ide"]} - common_configuration_yaml["cluster_cidrs"] = cidrs - common_configuration_yaml["enable_nfs"] = False - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + configuration = [ + {elem: "True" for elem in ["localFS", "localDNSlookup", "useMasterAsCompute", "slurm", "zabbix", "ide"]}] + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': 'True', + 'enable_nfs': False, 'enable_slurm': 'True', 'enable_zabbix': 'True', + 'ide_conf': {'build': False, 'ide': False, 'port_end': 8383, 'port_start': 8181, + 'workspace': '${HOME}'}, 'local_dns_lookup': 'True', + 'local_fs': 'True', 'slurm': 'True', + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, + 'use_master_as_compute': 'True', + 'zabbix_conf': {'admin_password': 'bibigrid', 'db': 'zabbix', + 'db_password': 'zabbix', 'db_user': 'zabbix', + 'server_name': 'bibigrid', 'timezone': 'Europe/Berlin'}} + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_nfs_shares(self): - cidrs = 42 - configuration = {"nfs": "True", "nfsShares": ["/vil/mil"]} - common_configuration_yaml = {'cluster_cidrs': 42, - 'enable_ide': False, - 'enable_nfs': 'True', - 'enable_slurm': False, - 'enable_zabbix': False, - 'ext_nfs_mounts': [], - 'local_dns_lookup': False, - 'local_fs': False, - 'nfs_mounts': [{'dst': '/vil/mil', 'src': '/vil/mil'}, - {'dst': '/vol/spool', 'src': '/vol/spool'}], + configuration = [{"nfs": "True", "nfsShares": ["/vil/mil"]}] + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False, + 'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False, + 'ext_nfs_mounts': [], 'local_dns_lookup': False, 'local_fs': False, + 'nfs_mounts': [{'dst': '//vil/mil', 'src': '//vil/mil'}, + {'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True, + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_nfs(self): - cidrs = 42 - configuration = {"nfs": "True"} - common_configuration_yaml = {'cluster_cidrs': 42, - 'enable_ide': False, - 'enable_nfs': 'True', - 'enable_slurm': False, - 'enable_zabbix': False, - 'ext_nfs_mounts': [], - 'local_dns_lookup': False, - 'local_fs': False, - 'nfs_mounts': [{'dst': '/vol/spool', 'src': '/vol/spool'}], + configuration = [{"nfs": "True"}] + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False, + 'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False, + 'ext_nfs_mounts': [], 'local_dns_lookup': False, 'local_fs': False, + 'nfs_mounts': [{'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True, + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_ext_nfs_shares(self): - cidrs = 42 - configuration = {"nfs": "True", "extNfsShares": ["/vil/mil"]} - common_configuration_yaml = {'cluster_cidrs': 42, - 'enable_ide': False, - 'enable_nfs': 'True', - 'enable_slurm': False, - 'enable_zabbix': False, + configuration = [{"nfs": "True", "extNfsShares": ["/vil/mil"]}] + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], 'enable_ide': False, + 'enable_nfs': 'True', 'enable_slurm': False, 'enable_zabbix': False, 'ext_nfs_mounts': [{'dst': '/vil/mil', 'src': '/vil/mil'}], - 'local_dns_lookup': False, - 'local_fs': False, - 'nfs_mounts': [{'dst': '/vol/spool', 'src': '/vol/spool'}], - 'use_master_as_compute': True} - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + 'local_dns_lookup': False, 'local_fs': False, + 'nfs_mounts': [{'dst': '//vol/spool', 'src': '//vol/spool'}], 'slurm': True, + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'YryJVnqgg24Ksf8zXQtbct3nuXrMSi9N'}, + 'ssh_user': ssh_user, 'use_master_as_compute': True} + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) def test_generate_common_configuration_ide(self): - cidrs = 42 - configuration = {"ide": "Some1", "ideConf": "Some2"} - common_configuration_yaml = {'cluster_cidrs': 42, - 'enable_ide': "Some1", - 'enable_nfs': False, - 'enable_slurm': False, + configuration = [{"ide": "Some1", "ideConf": {"key1": "Some2"}}] + cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + common_configuration_yaml = {'auto_mount': False, 'cluster_cidrs': cidrs, 'cluster_id': cluster_id, + 'default_user': default_user, 'dns_server_list': ['8.8.8.8'], + 'enable_ide': 'Some1', 'enable_nfs': False, 'enable_slurm': False, 'enable_zabbix': False, - 'ide_conf': 'Some2', - 'local_dns_lookup': False, - 'local_fs': False, - 'use_master_as_compute': True} - self.assertEqual(common_configuration_yaml, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) + 'ide_conf': {'build': False, 'ide': False, 'key1': 'Some2', 'port_end': 8383, + 'port_start': 8181, 'workspace': '${HOME}'}, + 'local_dns_lookup': False, 'local_fs': False, 'slurm': True, + 'slurm_conf': {'db': 'slurm', 'db_password': 'changeme', 'db_user': 'slurm', + 'elastic_scheduling': {'ResumeTimeout': 900, 'SuspendTime': 3600, + 'TreeWidth': 128}, + 'munge_key': 'b7nks3Ur3kanyPAEBxfSC9ypfSHFnWJL'}, + 'ssh_user': ssh_user, 'use_master_as_compute': True} + generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] + self.assertEqual(common_configuration_yaml, generated_common_configuration) @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_roles") - def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_roles): - cidrs = 42 + def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_roles): # TODO + cidrs = "42" ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] mock_ansible_roles.return_value = 21 configuration = {"ansibleRoles": ansible_roles} @@ -139,13 +201,12 @@ def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_rol @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_galaxy_roles") def test_generate_common_configuration_ansible_galaxy_roles(self, mock_galaxy_roles): - cidrs = 42 + cidrs = "42" galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] configuration = {"ansibleGalaxyRoles": galaxy_roles} mock_galaxy_roles.return_value = 21 - self.assertEqual(21, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)[ - "ansible_galaxy_roles"]) + self.assertEqual(21, ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)[ + "ansible_galaxy_roles"]) mock_galaxy_roles.assert_called_with(galaxy_roles) @patch("bibigrid.core.utility.ansibleConfigurator.to_instance_host_dict") @@ -163,16 +224,14 @@ def test_to_instance_host_local(self): ip = 42 ssh_user = 21 local = {"ip": ip, "ansible_connection": "local", - "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, - "ansible_user": ssh_user} + "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} self.assertEqual(local, ansibleConfigurator.to_instance_host_dict(21, 42, True)) def test_to_instance_host_ssh(self): ip = 42 ssh_user = 21 ssh = {"ip": ip, "ansible_connection": "ssh", - "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, - "ansible_user": ssh_user} + "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} self.assertEqual(ssh, ansibleConfigurator.to_instance_host_dict(21, 42, False)) def test_get_cidrs_single(self): @@ -245,9 +304,7 @@ def test_get_ansible_galaxy_roles_mismatch(self, mock_log): mock_log.assert_called() def test_generate_login_file(self): - login_yaml = {"default_user": 99, - "ssh_user": 21, - "munge_key": 32} + login_yaml = {"default_user": 99, "ssh_user": 21, "munge_key": 32} self.assertEqual(login_yaml, ansibleConfigurator.generate_login_file_yaml(21, 32, 99)) def test_generate_worker_specification_file_yaml(self): @@ -285,14 +342,14 @@ def test_write_yaml_alias(self, mock_yaml): @patch("bibigrid.core.utility.ansibleConfigurator.generate_site_file_yaml") @patch("bibigrid.core.utility.ansibleConfigurator.write_yaml") @patch("bibigrid.core.utility.ansibleConfigurator.get_cidrs") - def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, - mock_instances, mock_list, mock_common, mock_login, mock_worker, mock_munge): + def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, mock_instances, + mock_list, mock_common, mock_login, mock_worker, mock_munge): mock_munge.return_value = 420 mock_cidrs.return_value = 421 mock_list.return_value = {2: 422} mock_roles.return_value = 423 provider = MagicMock() - provider.cloud_specification = {"auth": {"username":"Tom"}} + provider.cloud_specification = {"auth": {"username": "Tom"}} ansibleConfigurator.configure_ansible_yaml([provider], [{"sshUser": 42, "ansibleRoles": 21}], 2) mock_munge.assert_called() mock_worker.assert_called_with([{"sshUser": 42, "ansibleRoles": 21}]) @@ -308,6 +365,5 @@ def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_rol call(aRP.COMMONS_LOGIN_FILE, mock_login(), False), call(aRP.COMMONS_CONFIG_FILE, mock_common(), False), call(aRP.COMMONS_INSTANCES_FILE, mock_instances(), False), - call(aRP.HOSTS_CONFIG_FILE, mock_hosts(), False), - call(aRP.SITE_CONFIG_FILE, mock_site(), False)] + call(aRP.HOSTS_CONFIG_FILE, mock_hosts(), False), call(aRP.SITE_CONFIG_FILE, mock_site(), False)] self.assertEqual(expected, mock_yaml.call_args_list) From 774520202d8cac50dbb2a16704238fdfd2495560 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 27 Oct 2023 12:19:12 +0200 Subject: [PATCH 19/34] improved code style --- bibigrid/core/utility/ansible_configurator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bibigrid/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py index cc8f5040..0a5d5630 100644 --- a/bibigrid/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -128,8 +128,7 @@ def write_host_and_group_vars(configurations, providers, cluster_id, log): # py master_dict = {"name": name, "image": master["image"], "network": configuration["network"], "network_cidr": configuration["subnet_cidrs"], "floating_ip": configuration["floating_ip"], "flavor": flavor_dict, "private_v4": configuration["private_v4"], - "cloud_identifier": configuration["cloud_identifier"], - "volumes": configuration["volumes"], + "cloud_identifier": configuration["cloud_identifier"], "volumes": configuration["volumes"], "fallback_on_other_image": configuration.get("fallbackOnOtherImage", False)} if configuration.get("wireguard_peer"): master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")} @@ -165,8 +164,8 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us master_configuration = configurations[0] log.info("Generating common configuration file...") # print(configuration.get("slurmConf", {})) - common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False), - "cluster_id": cluster_id, "cluster_cidrs": cidrs, "default_user": default_user, + common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id, + "cluster_cidrs": cidrs, "default_user": default_user, "local_fs": master_configuration.get("localFS", False), "local_dns_lookup": master_configuration.get("localDNSlookup", False), "use_master_as_compute": master_configuration.get("useMasterAsCompute", True), @@ -266,8 +265,8 @@ def get_cidrs(configurations): """ all_cidrs = [] for configuration in configurations: - subnet = configuration["subnet_cidrs"] - provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"], "provider_cidrs": subnet} + provider_cidrs = {"cloud_identifier": configuration["cloud_identifier"], + "provider_cidrs": configuration["subnet_cidrs"]} all_cidrs.append(provider_cidrs) return all_cidrs @@ -275,8 +274,9 @@ def get_cidrs(configurations): def get_ansible_roles(ansible_roles, log): """ Checks if ansible_roles have all necessary values and returns True if so. - :param ansible_roles: ansible_roles from master configuration (first configuration) - :return: list of valid ansible_roles + @param ansible_roles: ansible_roles from master configuration (first configuration) + @param log: + @return: list of valid ansible_roles """ ansible_roles_yaml = [] for ansible_role in (ansible_roles or []): From e7b6ccd66087d17e404358580353c418e8ce5ea2 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 27 Oct 2023 12:19:31 +0200 Subject: [PATCH 20/34] fixed tests further. One to fix left. --- tests/test_ansible_configurator.py | 221 ++++++++++++++--------------- 1 file changed, 107 insertions(+), 114 deletions(-) diff --git a/tests/test_ansible_configurator.py b/tests/test_ansible_configurator.py index c3ae1851..ff36fbec 100644 --- a/tests/test_ansible_configurator.py +++ b/tests/test_ansible_configurator.py @@ -1,13 +1,12 @@ """ -Tests for ansibleConfigurator +Tests for ansible_configurator """ from unittest import TestCase from unittest.mock import MagicMock, Mock, patch, call, mock_open, ANY -import bibigrid.core.utility.ansible_configurator as ansibleConfigurator import bibigrid.core.utility.paths.ansible_resources_path as aRP -import bibigrid.core.utility.yaml_dumper as yamlDumper from bibigrid.core import startup +from bibigrid.core.utility import ansible_configurator class TestAnsibleConfigurator(TestCase): @@ -26,7 +25,7 @@ def test_generate_site_file_yaml_empty(self): {'become': 'yes', 'hosts': 'workers', 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}], 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml']}] - self.assertEqual(site_yaml, ansibleConfigurator.generate_site_file_yaml([])) + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml([])) def test_generate_site_file_yaml_role(self): custom_roles = [{"file": "file", "hosts": "hosts", "name": "name", "vars": "vars", "vars_file": "varsFile"}] @@ -40,7 +39,7 @@ def test_generate_site_file_yaml_role(self): {'become': 'yes', 'hosts': 'workers', 'roles': [{'role': 'bibigrid', 'tags': ['bibigrid', 'bibigrid-worker']}, 'additional/name'], 'vars_files': ['vars/common_configuration.yml', 'vars/hosts.yml', 'varsFile']}] - self.assertEqual(site_yaml, ansibleConfigurator.generate_site_file_yaml(custom_roles)) + self.assertEqual(site_yaml, ansible_configurator.generate_site_file_yaml(custom_roles)) def test_generate_common_configuration_false(self): cidrs = "42" @@ -57,10 +56,10 @@ def test_generate_common_configuration_false(self): 'TreeWidth': 128}, 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) # munge key is randomly generated common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) @@ -86,10 +85,10 @@ def test_generate_common_configuration_true(self): 'zabbix_conf': {'admin_password': 'bibigrid', 'db': 'zabbix', 'db_password': 'zabbix', 'db_user': 'zabbix', 'server_name': 'bibigrid', 'timezone': 'Europe/Berlin'}} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) @@ -110,10 +109,10 @@ def test_generate_common_configuration_nfs_shares(self): 'TreeWidth': 128}, 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) @@ -133,10 +132,10 @@ def test_generate_common_configuration_nfs(self): 'TreeWidth': 128}, 'munge_key': 'TO_BE_FILLED'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) @@ -157,10 +156,10 @@ def test_generate_common_configuration_ext_nfs_shares(self): 'TreeWidth': 128}, 'munge_key': 'YryJVnqgg24Ksf8zXQtbct3nuXrMSi9N'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) @@ -182,188 +181,182 @@ def test_generate_common_configuration_ide(self): 'TreeWidth': 128}, 'munge_key': 'b7nks3Ur3kanyPAEBxfSC9ypfSHFnWJL'}, 'ssh_user': ssh_user, 'use_master_as_compute': True} - generated_common_configuration = ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration, - cluster_id, ssh_user, - default_user, - startup.LOG) + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) common_configuration_yaml["slurm_conf"]["munge_key"] = generated_common_configuration["slurm_conf"]["munge_key"] self.assertEqual(common_configuration_yaml, generated_common_configuration) - @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_roles") - def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_roles): # TODO + def test_generate_common_configuration_ansible_roles_mock(self): cidrs = "42" ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] - mock_ansible_roles.return_value = 21 - configuration = {"ansibleRoles": ansible_roles} - self.assertEqual(21, - ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)["ansible_roles"]) - mock_ansible_roles.assert_called_with(ansible_roles) - - @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_galaxy_roles") - def test_generate_common_configuration_ansible_galaxy_roles(self, mock_galaxy_roles): + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" + configuration = [{"ansibleRoles": ansible_roles}] + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + self.assertEqual(ansible_roles, generated_common_configuration["ansible_roles"]) + + def test_generate_common_configuration_ansible_galaxy_roles(self): cidrs = "42" + cluster_id = "21" + default_user = "ubuntu" + ssh_user = "test" galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] - configuration = {"ansibleGalaxyRoles": galaxy_roles} - mock_galaxy_roles.return_value = 21 - self.assertEqual(21, ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)[ - "ansible_galaxy_roles"]) - mock_galaxy_roles.assert_called_with(galaxy_roles) - - @patch("bibigrid.core.utility.ansibleConfigurator.to_instance_host_dict") + configuration = [{"ansibleGalaxyRoles": galaxy_roles}] + generated_common_configuration = ansible_configurator.generate_common_configuration_yaml(cidrs, configuration, + cluster_id, ssh_user, + default_user, + startup.LOG) + self.assertEqual(galaxy_roles, generated_common_configuration["ansible_galaxy_roles"]) + + @patch("bibigrid.core.utility.ansible_configurator.to_instance_host_dict") def test_generate_ansible_hosts(self, mock_instance_host_dict): - mock_instance_host_dict.side_effect = [0, 1, 2] - cluster_dict = {"workers": [{"private_v4": 21}], "vpngtws": [{"private_v4": 32}]} - expected = {'master': {'hosts': 0}, 'worker': {'hosts': {21: 1, 32: 2}}} - self.assertEqual(expected, ansibleConfigurator.generate_ansible_hosts_yaml(42, cluster_dict)) + cluster_id = "21" + mock_instance_host_dict.side_effect = [0, 1, 2, 4, 5, {}] + configuration = [{'masterInstance': {'type': 'mini', 'image': 'Ubuntu'}, + 'workerInstances': [{'type': 'tiny', 'image': 'Ubuntu', 'count': 2}, + {'type': 'default', 'image': 'Ubuntu', 'count': 1}]}, + {'vpnInstance': {'type': 'mini', 'image': 'Ubuntu'}, 'workerInstances': [ + {'type': 'tiny', 'image': 'Ubuntu', 'count': 2, 'features': ['holdsinformation']}, + {'type': 'small', 'image': 'Ubuntu', 'count': 2}], 'floating_ip': "42"}] + expected = {'vpn': {'children': {'master': {'hosts': {'localhost': 0}}, + 'vpngtw': {'hosts': {'bibigrid-vpngtw-21-0': {'ansible_host': '42'}}}}, + 'hosts': {}}, 'workers': { + 'children': {'bibigrid_worker_21_0_1': {'hosts': {'bibigrid-worker-21-[0:1]': 1}}, + 'bibigrid_worker_21_2_2': {'hosts': {'bibigrid-worker-21-[2:2]': 2}}, + 'bibigrid_worker_21_3_4': {'hosts': {'bibigrid-worker-21-[3:4]': 4}}, + 'bibigrid_worker_21_5_6': {'hosts': {'bibigrid-worker-21-[5:6]': 5}}}, 'hosts': {}}} + self.assertEqual(expected, + ansible_configurator.generate_ansible_hosts_yaml(42, configuration, cluster_id, startup.LOG)) call_list = mock_instance_host_dict.call_args_list self.assertEqual(call(42), call_list[0]) - self.assertEqual(call(42, ip=21, local=False), call_list[1]) - self.assertEqual(call(42, ip=32, local=False), call_list[2]) + for call_happened in call_list[1:]: + self.assertEqual(call(42, ip=""), call_happened) def test_to_instance_host_local(self): ip = 42 ssh_user = 21 - local = {"ip": ip, "ansible_connection": "local", - "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} - self.assertEqual(local, ansibleConfigurator.to_instance_host_dict(21, 42, True)) + local = {"ip": ip, "ansible_connection": "ssh", + "ansible_python_interpreter": ansible_configurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} + self.assertEqual(local, ansible_configurator.to_instance_host_dict(21, 42)) def test_to_instance_host_ssh(self): ip = 42 ssh_user = 21 ssh = {"ip": ip, "ansible_connection": "ssh", - "ansible_python_interpreter": ansibleConfigurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} - self.assertEqual(ssh, ansibleConfigurator.to_instance_host_dict(21, 42, False)) + "ansible_python_interpreter": ansible_configurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} + self.assertEqual(ssh, ansible_configurator.to_instance_host_dict(21, 42)) - def test_get_cidrs_single(self): + def test_get_cidrs(self): provider = Mock() provider.get_subnet_by_id_or_name.return_value = {"cidr": 42} - configuration = {"subnet": 21} - expected = [{'provider': 'Mock', 'provider_cidrs': [42]}] - self.assertEqual(expected, ansibleConfigurator.get_cidrs([configuration], [provider])) - provider.get_subnet_by_id_or_name.assert_called_with(21) - - def test_get_cidrs_list(self): - provider = Mock() - provider.get_subnet_by_id_or_name.return_value = {"cidr": 42} - configuration = {"subnet": [21, 22]} - expected = [{'provider': 'Mock', 'provider_cidrs': [42, 42]}] - self.assertEqual(expected, ansibleConfigurator.get_cidrs([configuration], [provider])) - call_list = provider.get_subnet_by_id_or_name.call_args_list - self.assertEqual(call(21), call_list[0]) - self.assertEqual(call(22), call_list[1]) + configuration = [{"subnet_cidrs": [21], "cloud_identifier": 13}] + expected = [{'cloud_identifier': 13, 'provider_cidrs': [21]}] + self.assertEqual(expected, ansible_configurator.get_cidrs(configuration)) def test_get_ansible_roles_empty(self): - self.assertEqual([], ansibleConfigurator.get_ansible_roles([])) + self.assertEqual([], ansible_configurator.get_ansible_roles([], startup.LOG)) def test_get_ansible_roles(self): ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] - self.assertEqual(ansible_roles, ansibleConfigurator.get_ansible_roles(ansible_roles)) + self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) def test_get_ansible_roles_add(self): ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] ansible_roles_add = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file", "additional"]}] - self.assertEqual(ansible_roles, ansibleConfigurator.get_ansible_roles(ansible_roles_add)) + self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles_add, startup.LOG)) def test_get_ansible_roles_minus(self): ansible_roles = [{elem: elem for elem in ["file", "hosts"]}] - self.assertEqual(ansible_roles, ansibleConfigurator.get_ansible_roles(ansible_roles)) + self.assertEqual(ansible_roles, ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - @patch("logging.warning") - def test_get_ansible_roles_mismatch_hosts(self, mock_log): + def test_get_ansible_roles_mismatch_hosts(self): ansible_roles = [{"file": "file"}] - self.assertEqual([], ansibleConfigurator.get_ansible_roles(ansible_roles)) - mock_log.assert_called() + self.assertEqual([], ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) - @patch("logging.warning") - def test_get_ansible_roles_mismatch_file(self, mock_log): + def test_get_ansible_roles_mismatch_file(self): ansible_roles = [{"hosts": "hosts"}] - self.assertEqual([], ansibleConfigurator.get_ansible_roles(ansible_roles)) - mock_log.assert_called() + self.assertEqual([], ansible_configurator.get_ansible_roles(ansible_roles, startup.LOG)) def test_get_ansible_galaxy_roles_empty(self): - self.assertEqual([], ansibleConfigurator.get_ansible_galaxy_roles([])) + self.assertEqual([], ansible_configurator.get_ansible_galaxy_roles([], startup.LOG)) def test_get_ansible_galaxy_roles(self): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] - self.assertEqual(galaxy_roles, ansibleConfigurator.get_ansible_galaxy_roles(galaxy_roles)) + self.assertEqual(galaxy_roles, ansible_configurator.get_ansible_galaxy_roles(galaxy_roles, startup.LOG)) def test_get_ansible_galaxy_roles_add(self): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] galaxy_roles_add = [ {elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file", "additional"]}] - self.assertEqual(galaxy_roles, ansibleConfigurator.get_ansible_galaxy_roles(galaxy_roles_add)) + self.assertEqual(galaxy_roles, ansible_configurator.get_ansible_galaxy_roles(galaxy_roles_add, startup.LOG)) def test_get_ansible_galaxy_roles_minus(self): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "vars", "vars_file"]}] - self.assertEqual(galaxy_roles, ansibleConfigurator.get_ansible_galaxy_roles(galaxy_roles)) + self.assertEqual(galaxy_roles, ansible_configurator.get_ansible_galaxy_roles(galaxy_roles, startup.LOG)) @patch("logging.warning") def test_get_ansible_galaxy_roles_mismatch(self, mock_log): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "vars", "vars_file"]}] - self.assertEqual([], ansibleConfigurator.get_ansible_galaxy_roles(galaxy_roles)) + self.assertEqual([], ansible_configurator.get_ansible_galaxy_roles(galaxy_roles, startup.LOG)) mock_log.assert_called() - def test_generate_login_file(self): - login_yaml = {"default_user": 99, "ssh_user": 21, "munge_key": 32} - self.assertEqual(login_yaml, ansibleConfigurator.generate_login_file_yaml(21, 32, 99)) - def test_generate_worker_specification_file_yaml(self): configuration = [{"workerInstances": [{elem: elem for elem in ["type", "image"]}], "network": [32]}] expected = [{'IMAGE': 'image', 'NETWORK': [32], 'TYPE': 'type'}] - self.assertEqual(expected, ansibleConfigurator.generate_worker_specification_file_yaml(configuration)) + self.assertEqual(expected, ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) def test_generate_worker_specification_file_yaml_empty(self): configuration = [{}] expected = [] - self.assertEqual(expected, ansibleConfigurator.generate_worker_specification_file_yaml(configuration)) + self.assertEqual(expected, ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) @patch("yaml.dump") def test_write_yaml_no_alias(self, mock_yaml): with patch('builtins.open', mock_open()) as output_mock: - ansibleConfigurator.write_yaml("here", {"some": "yaml"}, False) + ansible_configurator.write_yaml("here", {"some": "yaml"}, False) output_mock.assert_called_once_with("here", "w+") mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY, Dumper=yamlDumper.NoAliasSafeDumper) @patch("yaml.safe_dump") def test_write_yaml_alias(self, mock_yaml): with patch('builtins.open', mock_open()) as output_mock: - ansibleConfigurator.write_yaml("here", {"some": "yaml"}, True) - output_mock.assert_called_once_with("here", "w+") + ansible_configurator.write_yaml("here", {"some": "yaml"}, startup.LOG, True) + output_mock.assert_called_once_with("here", mode="w+", encoding="UTF-8") mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY) @patch("bibigrid.core.utility.id_generation.generate_munge_key") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_worker_specification_file_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_login_file_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_common_configuration_yaml") + @patch("bibigrid.core.utility.ansible_configurator.generate_worker_specification_file_yaml") + @patch("bibigrid.core.utility.ansible_configurator.generate_common_configuration_yaml") @patch("bibigrid.core.actions.list_clusters.dict_clusters") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_instances_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_ansible_hosts_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_roles") - @patch("bibigrid.core.utility.ansibleConfigurator.generate_site_file_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.write_yaml") - @patch("bibigrid.core.utility.ansibleConfigurator.get_cidrs") - def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, mock_instances, - mock_list, mock_common, mock_login, mock_worker, mock_munge): + @patch("bibigrid.core.utility.ansible_configurator.generate_ansible_hosts_yaml") + @patch("bibigrid.core.utility.ansible_configurator.get_ansible_roles") + @patch("bibigrid.core.utility.ansible_configurator.generate_site_file_yaml") + @patch("bibigrid.core.utility.ansible_configurator.write_yaml") + @patch("bibigrid.core.utility.ansible_configurator.get_cidrs") + def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, + mock_list, mock_common, mock_worker, mock_munge): mock_munge.return_value = 420 mock_cidrs.return_value = 421 mock_list.return_value = {2: 422} mock_roles.return_value = 423 provider = MagicMock() provider.cloud_specification = {"auth": {"username": "Tom"}} - ansibleConfigurator.configure_ansible_yaml([provider], [{"sshUser": 42, "ansibleRoles": 21}], 2) + ansible_configurator.configure_ansible_yaml([provider], [{"sshUser": 42, "ansibleRoles": 21}], 2, startup.LOG) mock_munge.assert_called() mock_worker.assert_called_with([{"sshUser": 42, "ansibleRoles": 21}]) mock_common.assert_called_with(421, configuration={"sshUser": 42, "ansibleRoles": 21}) - mock_login.assert_called_with(ssh_user=42, munge_key=420, default_user="Tom") mock_list.assert_called_with([provider]) - mock_instances.assert_called_with(422) mock_hosts.assert_called_with(42, 422) mock_site.assert_called_with(423) mock_roles.assert_called_with(21) mock_cidrs.assert_called_with([{'sshUser': 42, 'ansibleRoles': 21}], [provider]) expected = [call(aRP.WORKER_SPECIFICATION_FILE, mock_worker(), False), - call(aRP.COMMONS_LOGIN_FILE, mock_login(), False), call(aRP.COMMONS_CONFIG_FILE, mock_common(), False), - call(aRP.COMMONS_INSTANCES_FILE, mock_instances(), False), call(aRP.HOSTS_CONFIG_FILE, mock_hosts(), False), call(aRP.SITE_CONFIG_FILE, mock_site(), False)] self.assertEqual(expected, mock_yaml.call_args_list) From f2015f9985f56a287e10d4edf48d7874f3100dd1 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 27 Oct 2023 13:14:16 +0200 Subject: [PATCH 21/34] fixed additional tests --- tests/test_ansible_configurator.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_ansible_configurator.py b/tests/test_ansible_configurator.py index ff36fbec..43230ff7 100644 --- a/tests/test_ansible_configurator.py +++ b/tests/test_ansible_configurator.py @@ -309,12 +309,14 @@ def test_get_ansible_galaxy_roles_mismatch(self, mock_log): def test_generate_worker_specification_file_yaml(self): configuration = [{"workerInstances": [{elem: elem for elem in ["type", "image"]}], "network": [32]}] expected = [{'IMAGE': 'image', 'NETWORK': [32], 'TYPE': 'type'}] - self.assertEqual(expected, ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) + self.assertEqual(expected, + ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) def test_generate_worker_specification_file_yaml_empty(self): configuration = [{}] expected = [] - self.assertEqual(expected, ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) + self.assertEqual(expected, + ansible_configurator.generate_worker_specification_file_yaml(configuration, startup.LOG)) @patch("yaml.dump") def test_write_yaml_no_alias(self, mock_yaml): @@ -330,7 +332,7 @@ def test_write_yaml_alias(self, mock_yaml): output_mock.assert_called_once_with("here", mode="w+", encoding="UTF-8") mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY) - @patch("bibigrid.core.utility.id_generation.generate_munge_key") + @patch("bibigrid.core.utility.ansible_configurator.write_host_and_group_vars") @patch("bibigrid.core.utility.ansible_configurator.generate_worker_specification_file_yaml") @patch("bibigrid.core.utility.ansible_configurator.generate_common_configuration_yaml") @patch("bibigrid.core.actions.list_clusters.dict_clusters") @@ -339,17 +341,16 @@ def test_write_yaml_alias(self, mock_yaml): @patch("bibigrid.core.utility.ansible_configurator.generate_site_file_yaml") @patch("bibigrid.core.utility.ansible_configurator.write_yaml") @patch("bibigrid.core.utility.ansible_configurator.get_cidrs") - def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, - mock_list, mock_common, mock_worker, mock_munge): - mock_munge.return_value = 420 + def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, mock_list, + mock_common, mock_worker, mock_write): mock_cidrs.return_value = 421 mock_list.return_value = {2: 422} mock_roles.return_value = 423 provider = MagicMock() provider.cloud_specification = {"auth": {"username": "Tom"}} ansible_configurator.configure_ansible_yaml([provider], [{"sshUser": 42, "ansibleRoles": 21}], 2, startup.LOG) - mock_munge.assert_called() - mock_worker.assert_called_with([{"sshUser": 42, "ansibleRoles": 21}]) + print(mock_yaml.call_args_list) + mock_worker.assert_called_with([{"sshUser": 42, "ansibleRoles": 21}], startup.LOG) mock_common.assert_called_with(421, configuration={"sshUser": 42, "ansibleRoles": 21}) mock_list.assert_called_with([provider]) mock_hosts.assert_called_with(42, 422) From 7141a78e8b01635dbf3aed04a0c4a989a546949d Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 27 Oct 2023 15:09:35 +0200 Subject: [PATCH 22/34] fixed all tests for ansible configurator --- tests/test_ansible_configurator.py | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/test_ansible_configurator.py b/tests/test_ansible_configurator.py index 43230ff7..dbaef9a9 100644 --- a/tests/test_ansible_configurator.py +++ b/tests/test_ansible_configurator.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, Mock, patch, call, mock_open, ANY import bibigrid.core.utility.paths.ansible_resources_path as aRP +from bibigrid.core.utility.yaml_dumper import NoAliasSafeDumper from bibigrid.core import startup from bibigrid.core.utility import ansible_configurator @@ -239,16 +240,16 @@ def test_generate_ansible_hosts(self, mock_instance_host_dict): self.assertEqual(call(42, ip=""), call_happened) def test_to_instance_host_local(self): - ip = 42 + ip_address = 42 ssh_user = 21 - local = {"ip": ip, "ansible_connection": "ssh", + local = {"ip": ip_address, "ansible_connection": "ssh", "ansible_python_interpreter": ansible_configurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} self.assertEqual(local, ansible_configurator.to_instance_host_dict(21, 42)) def test_to_instance_host_ssh(self): - ip = 42 + ip_address = 42 ssh_user = 21 - ssh = {"ip": ip, "ansible_connection": "ssh", + ssh = {"ip": ip_address, "ansible_connection": "ssh", "ansible_python_interpreter": ansible_configurator.PYTHON_INTERPRETER, "ansible_user": ssh_user} self.assertEqual(ssh, ansible_configurator.to_instance_host_dict(21, 42)) @@ -300,11 +301,9 @@ def test_get_ansible_galaxy_roles_minus(self): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "vars", "vars_file"]}] self.assertEqual(galaxy_roles, ansible_configurator.get_ansible_galaxy_roles(galaxy_roles, startup.LOG)) - @patch("logging.warning") - def test_get_ansible_galaxy_roles_mismatch(self, mock_log): + def test_get_ansible_galaxy_roles_mismatch(self): galaxy_roles = [{elem: elem for elem in ["hosts", "name", "vars", "vars_file"]}] self.assertEqual([], ansible_configurator.get_ansible_galaxy_roles(galaxy_roles, startup.LOG)) - mock_log.assert_called() def test_generate_worker_specification_file_yaml(self): configuration = [{"workerInstances": [{elem: elem for elem in ["type", "image"]}], "network": [32]}] @@ -321,9 +320,9 @@ def test_generate_worker_specification_file_yaml_empty(self): @patch("yaml.dump") def test_write_yaml_no_alias(self, mock_yaml): with patch('builtins.open', mock_open()) as output_mock: - ansible_configurator.write_yaml("here", {"some": "yaml"}, False) - output_mock.assert_called_once_with("here", "w+") - mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY, Dumper=yamlDumper.NoAliasSafeDumper) + ansible_configurator.write_yaml("here", {"some": "yaml"}, startup.LOG, False) + output_mock.assert_called_once_with("here", mode="w+", encoding="UTF-8") + mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY, Dumper=NoAliasSafeDumper) @patch("yaml.safe_dump") def test_write_yaml_alias(self, mock_yaml): @@ -347,17 +346,20 @@ def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_rol mock_list.return_value = {2: 422} mock_roles.return_value = 423 provider = MagicMock() - provider.cloud_specification = {"auth": {"username": "Tom"}} - ansible_configurator.configure_ansible_yaml([provider], [{"sshUser": 42, "ansibleRoles": 21}], 2, startup.LOG) - print(mock_yaml.call_args_list) - mock_worker.assert_called_with([{"sshUser": 42, "ansibleRoles": 21}], startup.LOG) - mock_common.assert_called_with(421, configuration={"sshUser": 42, "ansibleRoles": 21}) - mock_list.assert_called_with([provider]) - mock_hosts.assert_called_with(42, 422) + provider.cloud_specification = {"auth": {"username": "Default"}} + configuration = [{"sshUser": 42, "ansibleRoles": 21}] + cluster_id = 2 + ansible_configurator.configure_ansible_yaml([provider], configuration, cluster_id, startup.LOG) + mock_worker.assert_called_with(configuration, startup.LOG) + mock_common.assert_called_with(cidrs=421, configurations=configuration, cluster_id=cluster_id, ssh_user=42, + default_user="Default", log=startup.LOG) + mock_hosts.assert_called_with(42, configuration, cluster_id, startup.LOG) mock_site.assert_called_with(423) - mock_roles.assert_called_with(21) - mock_cidrs.assert_called_with([{'sshUser': 42, 'ansibleRoles': 21}], [provider]) - expected = [call(aRP.WORKER_SPECIFICATION_FILE, mock_worker(), False), - call(aRP.COMMONS_CONFIG_FILE, mock_common(), False), - call(aRP.HOSTS_CONFIG_FILE, mock_hosts(), False), call(aRP.SITE_CONFIG_FILE, mock_site(), False)] + mock_roles.assert_called_with(21, startup.LOG) + mock_cidrs.assert_called_with(configuration) + mock_write.assert_called() + expected = [call(aRP.WORKER_SPECIFICATION_FILE, mock_worker(), startup.LOG, False), + call(aRP.COMMONS_CONFIG_FILE, mock_common(), startup.LOG, False), + call(aRP.HOSTS_CONFIG_FILE, mock_hosts(), startup.LOG, False), + call(aRP.SITE_CONFIG_FILE, mock_site(), startup.LOG, False)] self.assertEqual(expected, mock_yaml.call_args_list) From dcf283652c6a03570ba432ec22a102925431ed9a Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 27 Oct 2023 16:28:48 +0200 Subject: [PATCH 23/34] fixed comment --- tests/startup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/startup_tests.py b/tests/startup_tests.py index b3fee8a1..876b6d1f 100644 --- a/tests/startup_tests.py +++ b/tests/startup_tests.py @@ -39,7 +39,7 @@ def suppress_stdout(): unittest.TextTestRunner(verbosity=2).run(suite) # Provider-Test - ## Configuration needs to contain providers and infrastructures + # Configuration needs to contain providers and infrastructures if os.environ.get("OS_KEY_NAME"): suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') with suppress_stdout(): From 400b350585b86bf7823c397a78a565665386dee3 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 31 Oct 2023 13:55:33 +0100 Subject: [PATCH 24/34] fixed multiple tests --- tests/test_create.py | 207 +++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 127 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index e3cd9068..e6d76395 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,119 +1,75 @@ +""" +Module to test create +""" import os from unittest import TestCase from unittest.mock import patch, Mock, MagicMock, mock_open +from bibigrid.core import startup from bibigrid.core.actions import create class TestCreate(TestCase): + """ + Class to test create + """ + # pylint: disable=R0904 - @patch("bibigrid.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") + @patch("bibigrid.core.utility.handler.ssh_handler.get_add_ssh_public_key_commands") @patch("bibigrid.core.utility.id_generation.generate_safe_cluster_id") def test_init(self, mock_id, mock_ssh): - unique_id = 21 + cluster_id = "21" provider = MagicMock() - provider.cloud_specification["auth"]["project_name"] = "name" - key_name = create.KEY_PREFIX + provider.cloud_specification["auth"]["project_name"] \ - + create.SEPARATOR + str(unique_id) - mock_id.return_value = str(unique_id) + provider_dict = {'cloud_specification': {'auth': {'project_name': 'project_name'}}} + provider.__getitem__.side_effect = provider_dict.__getitem__ + key_name = create.KEY_NAME.format(cluster_id=cluster_id) + mock_id.return_value = cluster_id mock_ssh.return_value = [32] - c = create.Create([provider], [{}], "path", False) - self.assertEqual(str(unique_id), c.cluster_id) - self.assertEqual("ubuntu", c.ssh_user) - self.assertEqual([32], c.ssh_add_public_key_commands) - self.assertEqual(c.key_name, key_name) + creator = create.Create([provider], [{}], "path", startup.LOG, False) + self.assertEqual(cluster_id, creator.cluster_id) + self.assertEqual("ubuntu", creator.ssh_user) + self.assertEqual([32], creator.ssh_add_public_key_commands) + self.assertEqual(key_name, creator.key_name) mock_id.assert_called_with([provider]) - @patch("bibigrid.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") + @patch("bibigrid.core.utility.handler.ssh_handler.get_add_ssh_public_key_commands") + @patch("bibigrid.core.utility.id_generation.generate_safe_cluster_id") + def test_init_with_cluster_id(self, mock_id, mock_ssh): + cluster_id = "21" + provider = MagicMock() + provider_dict = {'cloud_specification': {'auth': {'project_name': 'project_name'}}} + provider.__getitem__.side_effect = provider_dict.__getitem__ + key_name = create.KEY_NAME.format(cluster_id=cluster_id) + mock_ssh.return_value = [32] + creator = create.Create([provider], [{}], "path", startup.LOG, False, cluster_id) + self.assertEqual(cluster_id, creator.cluster_id) + self.assertEqual("ubuntu", creator.ssh_user) + self.assertEqual([32], creator.ssh_add_public_key_commands) + self.assertEqual(key_name, creator.key_name) + mock_id.assert_not_called() + + @patch("bibigrid.core.utility.handler.ssh_handler.get_add_ssh_public_key_commands") @patch("bibigrid.core.utility.id_generation.generate_safe_cluster_id") def test_init_username(self, mock_id, mock_ssh): - unique_id = 21 - mock_id.return_value = str(unique_id) + cluster_id = "21" + mock_id.return_value = cluster_id mock_ssh.return_value = [32] - c = create.Create([MagicMock()], [{"sshUser": "ssh"}], "path", False) - self.assertEqual("ssh", c.ssh_user) + creator = create.Create([MagicMock()], [{"sshUser": "ssh"}], "path", startup.LOG, False) + self.assertEqual("ssh", creator.ssh_user) @patch("subprocess.check_output") def test_generate_keypair(self, mock_subprocess): provider = MagicMock() provider.list_servers.return_value = [] - c = create.Create([provider], [{}], "") + creator = create.Create([provider], [{}], "", startup.LOG) public_key = "data" with patch("builtins.open", mock_open(read_data=public_key)): - c.generate_keypair() - provider.create_keypair.assert_called_with(name=c.key_name, public_key=public_key) - mock_subprocess.assert_called_with(f'ssh-keygen -t ecdsa -f {create.KEY_FOLDER}{c.key_name} -P ""') - - def test_start_instance(self): - provider = MagicMock() - provider.list_servers.return_value = [] - provider.create_server.return_value = 42 - provider.add_auto_ip.return_value = {"floating_ip_address": 12} - c = create.Create([provider], [{}], "") - server_type = {"type": "testType", "image": "testImage"} - network = 21 - external_network = "testExternal" - c.start_instance(provider, create.MASTER_IDENTIFIER, server_type, network, worker=False, volumes=2, - external_network=external_network) - provider.create_server.assert_called_with(name=create.MASTER_IDENTIFIER + create.SEPARATOR + c.cluster_id, - flavor=server_type["type"], - key_name=c.key_name, - image=server_type["image"], - network=network, volumes=2) - provider.add_auto_ip.assert_called_with(network=external_network, server=42) - - def test_start_instance_worker(self): - provider = MagicMock() - provider.list_servers.return_value = [] - provider.create_server.return_value = 42 - provider.create_floating_ip.return_value = {"floating_ip_address": 12} - c = create.Create([provider], [{}], "") - server_type = {"type": "testType", "image": "testImage"} - network = 21 - c.start_instance(provider, create.WORKER_IDENTIFIER, server_type, network, worker=True, volumes=None, - external_network=None) - provider.create_server.assert_called_with( - name=create.WORKER_IDENTIFIER.format(0) + create.SEPARATOR + c.cluster_id, - flavor=server_type["type"], - key_name=c.key_name, - image=server_type["image"], - network=network, volumes=None) - provider.create_floating_ip.assert_not_called() - - @patch("bibigrid.models.returnThreading.ReturnThread") - def test_start_instances(self, return_mock): - provider = MagicMock() - provider.list_servers.return_value = [] - external_network = "externalTest" - provider.get_external_netowrk.return_value = external_network - configuration = {"network": 42} - c = create.Create([provider], [configuration], "") - provider.get_external_network.return_value = 32 - with patch.object(c, "prepare_vpn_or_master_args", return_value=(0, 1, 2)) as prepare_mock: - prepare_mock.return_value = (0, 1, 2) - c.start_instances({"network": 42}, provider) - prepare_mock.assert_called_with(configuration, provider) - provider.get_external_network.assert_called_with(configuration["network"]) - return_mock.assert_called_with(target=c.start_instance, - args=[provider, 0, 1, configuration["network"], False, 2, 32]) + creator.generate_keypair() + provider.create_keypair.assert_called_with(name=creator.key_name, public_key=public_key) + mock_subprocess.assert_called_with(f'ssh-keygen -t ecdsa -f {create.KEY_FOLDER}{creator.key_name} -P ""', + shell=True) - @patch("threading.Thread") - @patch("bibigrid.models.returnThreading.ReturnThread") - def test_start_instances_workers(self, return_mock, thread_mock): - provider = MagicMock() - provider.list_servers.return_value = [] - external_network = "externalTest" - provider.get_external_netowrk.return_value = external_network - configuration = {"network": 42, "workerInstances": [{"count": 1}]} - c = create.Create([provider], [configuration], "") - provider.get_external_network.return_value = 32 - with patch.object(c, "prepare_vpn_or_master_args", return_value=(0, 1, 2)) as prepare_mock: - prepare_mock.return_value = (0, 1, 2) - c.start_instances(configuration, provider) - thread_mock.assert_called_with(target=c.start_instance, - args=[provider, create.WORKER_IDENTIFIER, configuration["workerInstances"][0], - configuration["network"], True]) - return_mock.assert_called() + # TODO: Rewrite start instance tests def test_prepare_master_args(self): provider = MagicMock() @@ -121,11 +77,11 @@ def test_prepare_master_args(self): external_network = "externalTest" provider.get_external_netowrk.return_value = external_network configuration = {"network": 42, "masterInstance": "Some"} - c = create.Create([provider], [configuration], "") + creator = create.Create([provider], [configuration], "", startup.LOG) volume_return = [42] - with patch.object(c, "prepare_volumes", return_value=volume_return) as prepare_mock: + with patch.object(creator, "prepare_volumes", return_value=volume_return) as prepare_mock: self.assertEqual((create.MASTER_IDENTIFIER, configuration["masterInstance"], volume_return), - c.prepare_vpn_or_master_args(configuration, provider)) + creator.prepare_vpn_or_master_args(configuration, provider)) prepare_mock.assert_called_with(provider, []) def test_prepare_vpn_args(self): @@ -134,11 +90,11 @@ def test_prepare_vpn_args(self): external_network = "externalTest" provider.get_external_netowrk.return_value = external_network configuration = {"network": 42, "vpnInstance": "Some"} - c = create.Create([provider], [configuration], "") + creator = create.Create([provider], [configuration], "", startup.LOG) volume_return = [42] - with patch.object(c, "prepare_volumes", return_value=volume_return) as prepare_mock: + with patch.object(creator, "prepare_volumes", return_value=volume_return) as prepare_mock: self.assertEqual((create.VPN_WORKER_IDENTIFIER, configuration["vpnInstance"], []), - c.prepare_vpn_or_master_args(configuration, provider)) + creator.prepare_vpn_or_master_args(configuration, provider)) prepare_mock.assert_not_called() def test_prepare_args_keyerror(self): @@ -147,26 +103,24 @@ def test_prepare_args_keyerror(self): external_network = "externalTest" provider.get_external_netowrk.return_value = external_network configuration = {"network": 42} - c = create.Create([provider], [configuration], "") + creator = create.Create([provider], [configuration], "", startup.LOG) volume_return = [42] - with patch.object(c, "prepare_volumes", return_value=volume_return) as prepare_mock: + with patch.object(creator, "prepare_volumes", return_value=volume_return) as prepare_mock: with self.assertRaises(KeyError): self.assertEqual((create.VPN_WORKER_IDENTIFIER, configuration["vpnInstance"], []), - c.prepare_vpn_or_master_args(configuration, provider)) + creator.prepare_vpn_or_master_args(configuration, provider)) prepare_mock.assert_not_called() - @patch("bibigrid.core.utility.handler.sshHandler.ansible_preparation") - def test_setup_reachable_servers_master(self, mock_ansible): + @patch("bibigrid.core.utility.handler.ssh_handler.ansible_preparation") + def test_initialize_instances_master(self, mock_ansible): provider = MagicMock() provider.list_servers.return_value = [] - configuration = {"masterInstance": 42} - c = create.Create([provider], [configuration], "") floating_ip = 21 - c.setup_reachable_servers(configuration, floating_ip) - mock_ansible.assert_called_with(floating_ip=floating_ip, - private_key=create.KEY_FOLDER + c.key_name, - username=c.ssh_user, - commands=[]) + configuration = {"masterInstance": 42, "floating_ip": floating_ip} + creator = create.Create([provider], [configuration], "", startup.LOG) + creator.initialize_instances() + mock_ansible.assert_called_with(floating_ip=floating_ip, private_key=create.KEY_FOLDER + creator.key_name, + username=creator.ssh_user, commands=[], log=startup.LOG, gateway={}) def test_prepare_volumes_none(self): provider = MagicMock() @@ -174,8 +128,8 @@ def test_prepare_volumes_none(self): provider.get_volume_by_id_or_name.return_value = 42 provider.create_volume_from_snapshot = 21 configuration = {"vpnInstance": 42} - c = create.Create([provider], [configuration], "") - self.assertEqual([], c.prepare_volumes(provider, [])) + creator = create.Create([provider], [configuration], "", startup.LOG) + self.assertEqual(set(), creator.prepare_volumes(provider, [])) def test_prepare_volumes_volume(self): provider = MagicMock() @@ -225,36 +179,35 @@ def test_prepare_configurations_no_subnet(self): subnet = ["subnet"] provider.get_subnet_ids_by_network.return_value = subnet configuration = {"network": 42} - c = create.Create([provider], [configuration], "") - c.prepare_configurations() + creator = create.Create([provider], [configuration], "", startup.LOG) + creator.prepare_configurations() provider.get_subnet_ids_by_network.assert_called_with(42) self.assertEqual(subnet, configuration["subnet"]) - self.assertEqual(c.ssh_user, configuration["sshUser"]) + self.assertEqual(creator.ssh_user, configuration["sshUser"]) def test_prepare_configurations_none(self): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "") + creator = create.Create([provider], [configuration], "", startup.LOG) with self.assertRaises(KeyError): - c.prepare_configurations() + creator.prepare_configurations() - @patch("bibigrid.core.utility.ansibleConfigurator.configure_ansible_yaml") - @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") + @patch("bibigrid.core.utility.ansible_configurator.configure_ansible_yaml") + @patch("bibigrid.core.utility.handler.ssh_handler.execute_ssh") def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "") - c.master_ip = 42 - c.upload_data() - mock_configure_ansible.assert_called_with(providers=c.providers, - configurations=c.configurations, - cluster_id=c.cluster_id) - mock_ssh.assert_called_with(floating_ip=c.master_ip, private_key=create.KEY_FOLDER + c.key_name, - username=c.ssh_user, filepaths=[(os.path.expanduser("/Documents/Repos/bibigrid/" - "resources/playbook/"), - "playbook")], + creator = create.Create([provider], [configuration], "", startup.LOG) + creator.master_ip = 42 + creator.upload_data() + mock_configure_ansible.assert_called_with(providers=creator.providers, configurations=creator.configurations, + cluster_id=creator.cluster_id) + mock_ssh.assert_called_with(floating_ip=creator.master_ip, private_key=create.KEY_FOLDER + creator.key_name, + username=creator.ssh_user, + filepaths=[(os.path.expanduser("/Documents/Repos/bibigrid/" + "resources/playbook/"), "playbook")], commands=['echo ansible_start']) @patch("threading.Thread") From 65d5da7c440faf8b1315203cbe1fa3618eb0ef5e Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Fri, 10 Nov 2023 17:03:05 +0100 Subject: [PATCH 25/34] fixed a few tests --- tests/test_create.py | 73 ++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index e6d76395..db96e319 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,7 +3,7 @@ """ import os from unittest import TestCase -from unittest.mock import patch, Mock, MagicMock, mock_open +from unittest.mock import patch, MagicMock, mock_open from bibigrid.core import startup from bibigrid.core.actions import create @@ -134,32 +134,30 @@ def test_prepare_volumes_none(self): def test_prepare_volumes_volume(self): provider = MagicMock() provider.list_servers.return_value = [] - provider.get_volume_by_id_or_name.return_value = 42 + provider.get_volume_by_id_or_name.return_value = {"id": 42} provider.create_volume_from_snapshot = 21 configuration = {"vpnInstance": 42} - c = create.Create([provider], [configuration], "") - self.assertEqual([42], c.prepare_volumes(provider, ["Test"])) + creator = create.Create([provider], [configuration], "", startup.LOG) + self.assertEqual({42}, creator.prepare_volumes(provider, ["Test"])) def test_prepare_volumes_snapshot(self): provider = MagicMock() provider.list_servers.return_value = [] - provider.get_volume_by_id_or_name.return_value = None + provider.get_volume_by_id_or_name.return_value = {"id": None} provider.create_volume_from_snapshot.return_value = 21 configuration = {"vpnInstance": 42} - c = create.Create([provider], [configuration], "") - self.assertEqual([21], c.prepare_volumes(provider, ["Test"])) + creator = create.Create([provider], [configuration], "", startup.LOG) + self.assertEqual({21}, creator.prepare_volumes(provider, ["Test"])) - @patch("logging.warning") - def test_prepare_volumes_mismatch(self, mock_log): + def test_prepare_volumes_mismatch(self): provider = MagicMock() provider.list_servers.return_value = [] - provider.get_volume_by_id_or_name.return_value = None + provider.get_volume_by_id_or_name.return_value = {"id": None} provider.create_volume_from_snapshot.return_value = None configuration = {"vpnInstance": 42} - c = create.Create([provider], [configuration], "") + creator = create.Create([provider], [configuration], "", startup.LOG) mount = "Test" - self.assertEqual([], c.prepare_volumes(provider, [mount])) - mock_log.assert_called_with(f"Mount {mount} is neither a snapshot nor a volume.") + self.assertEqual(set(), creator.prepare_volumes(provider, [mount])) def test_prepare_configurations_no_network(self): provider = MagicMock() @@ -167,11 +165,11 @@ def test_prepare_configurations_no_network(self): network = "network" provider.get_network_id_by_subnet.return_value = network configuration = {"subnet": 42} - c = create.Create([provider], [configuration], "") - c.prepare_configurations() + creator = create.Create([provider], [configuration], "", startup.LOG) + creator.prepare_configurations() provider.get_network_id_by_subnet.assert_called_with(42) self.assertEqual(network, configuration["network"]) - self.assertEqual(c.ssh_user, configuration["sshUser"]) + self.assertEqual(creator.ssh_user, configuration["sshUser"]) def test_prepare_configurations_no_subnet(self): provider = MagicMock() @@ -210,46 +208,33 @@ def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): "resources/playbook/"), "playbook")], commands=['echo ansible_start']) - @patch("threading.Thread") - def test_start_start_instances_thread(self, mock_thread): - provider = MagicMock() - provider.list_servers.return_value = [] - configuration = {} - c = create.Create([provider], [configuration], "") - start_instances_mock_thread = Mock() - mock_thread.return_value = start_instances_mock_thread - c.start_start_instances_threads() - mock_thread.assert_called_with(target=c.start_instances, args=[configuration, provider]) - start_instances_mock_thread.start.assert_called() - start_instances_mock_thread.join.assert_called() - @patch.object(create.Create, "generate_keypair") @patch.object(create.Create, "prepare_configurations") - @patch.object(create.Create, "start_start_instances_threads") + @patch.object(create.Create, "start_start_instance_threads") @patch.object(create.Create, "upload_data") - @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") + @patch.object(create.Create, "log_cluster_start_info") + @patch("bibigrid.core.actions.terminate.terminate") def test_create_non_debug(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "", False) - self.assertEqual(0, c.create()) + creator = create.Create([provider], [configuration], "", startup.LOG, False) + self.assertEqual(0, creator.create()) for mock in [mock_info, mock_up, mock_start, mock_conf, mock_key]: mock.assert_called() mock_terminate.assert_not_called() @patch.object(create.Create, "generate_keypair") @patch.object(create.Create, "prepare_configurations") - @patch.object(create.Create, "start_start_instances_threads") + @patch.object(create.Create, "start_start_instance_threads") @patch.object(create.Create, "upload_data") - @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") + @patch.object(create.Create, "log_cluster_start_info") + @patch("bibigrid.core.actions.terminate.terminate") def test_create_non_debug_upload_raise(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "", False) + c = create.Create([provider], [configuration], "", startup.LOG,False) mock_up.side_effect = [ConnectionError()] self.assertEqual(1, c.create()) for mock in [mock_start, mock_conf, mock_key, mock_up]: @@ -260,16 +245,16 @@ def test_create_non_debug_upload_raise(self, mock_terminate, mock_info, mock_up, @patch.object(create.Create, "generate_keypair") @patch.object(create.Create, "prepare_configurations") - @patch.object(create.Create, "start_start_instances_threads") + @patch.object(create.Create, "start_start_instance_threads") @patch.object(create.Create, "upload_data") - @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") + @patch.object(create.Create, "log_cluster_start_info") + @patch("bibigrid.core.actions.terminate.terminate") def test_create_debug(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "", True) - self.assertEqual(0, c.create()) + creator = create.Create([provider], [configuration], "", startup.LOG,True) + self.assertEqual(0, creator.create()) for mock in [mock_info, mock_up, mock_start, mock_conf, mock_key]: mock.assert_called() - mock_terminate.assert_called_with(cluster_id=c.cluster_id, providers=[provider], debug=True) + mock_terminate.assert_called_with(cluster_id=creator.cluster_id, providers=[provider], log=startup.LOG, debug=True) From c989ad3ea7aaf9c8293854a0fc75705869c030c9 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 14 Nov 2023 12:36:22 +0100 Subject: [PATCH 26/34] Fixed create --- tests/test_create.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index db96e319..339a90db 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,12 +1,12 @@ """ Module to test create """ -import os from unittest import TestCase from unittest.mock import patch, MagicMock, mock_open from bibigrid.core import startup from bibigrid.core.actions import create +from bibigrid.core.utility.handler import ssh_handler class TestCreate(TestCase): @@ -192,8 +192,9 @@ def test_prepare_configurations_none(self): creator.prepare_configurations() @patch("bibigrid.core.utility.ansible_configurator.configure_ansible_yaml") + @patch("bibigrid.core.utility.handler.ssh_handler.get_ac_command") @patch("bibigrid.core.utility.handler.ssh_handler.execute_ssh") - def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): + def test_upload_playbooks(self, mock_execute_ssh, mock_ac_ssh, mock_configure_ansible): provider = MagicMock() provider.list_servers.return_value = [] configuration = {} @@ -201,12 +202,12 @@ def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): creator.master_ip = 42 creator.upload_data() mock_configure_ansible.assert_called_with(providers=creator.providers, configurations=creator.configurations, - cluster_id=creator.cluster_id) - mock_ssh.assert_called_with(floating_ip=creator.master_ip, private_key=create.KEY_FOLDER + creator.key_name, - username=creator.ssh_user, - filepaths=[(os.path.expanduser("/Documents/Repos/bibigrid/" - "resources/playbook/"), "playbook")], - commands=['echo ansible_start']) + cluster_id=creator.cluster_id, log=startup.LOG) + mock_execute_ssh.assert_called_with(floating_ip=creator.master_ip, + private_key=create.KEY_FOLDER + creator.key_name, username=creator.ssh_user, + filepaths=create.FILEPATHS, + commands=[mock_ac_ssh()] + ssh_handler.ANSIBLE_START, log=startup.LOG, + gateway={}) @patch.object(create.Create, "generate_keypair") @patch.object(create.Create, "prepare_configurations") @@ -234,14 +235,15 @@ def test_create_non_debug_upload_raise(self, mock_terminate, mock_info, mock_up, provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - c = create.Create([provider], [configuration], "", startup.LOG,False) + creator = create.Create([provider], [configuration], "", startup.LOG, False) mock_up.side_effect = [ConnectionError()] - self.assertEqual(1, c.create()) + self.assertEqual(1, creator.create()) for mock in [mock_start, mock_conf, mock_key, mock_up]: mock.assert_called() for mock in [mock_info]: mock.assert_not_called() - mock_terminate.assert_called_with(cluster_id=c.cluster_id, providers=[provider], debug=False) + mock_terminate.assert_called_with(cluster_id=creator.cluster_id, providers=[provider], log=startup.LOG, + debug=False) @patch.object(create.Create, "generate_keypair") @patch.object(create.Create, "prepare_configurations") @@ -253,8 +255,9 @@ def test_create_debug(self, mock_terminate, mock_info, mock_up, mock_start, mock provider = MagicMock() provider.list_servers.return_value = [] configuration = {} - creator = create.Create([provider], [configuration], "", startup.LOG,True) + creator = create.Create([provider], [configuration], "", startup.LOG, True) self.assertEqual(0, creator.create()) for mock in [mock_info, mock_up, mock_start, mock_conf, mock_key]: mock.assert_called() - mock_terminate.assert_called_with(cluster_id=creator.cluster_id, providers=[provider], log=startup.LOG, debug=True) + mock_terminate.assert_called_with(cluster_id=creator.cluster_id, providers=[provider], log=startup.LOG, + debug=True) From 4442c1bd4ffe3e466028a5331199874e4b5f4b37 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 14 Nov 2023 14:05:50 +0100 Subject: [PATCH 27/34] fixed some issues regarding --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f6c40ad0..a84bed8b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ resources/playbook/ansible_hosts resources/playbook/vars/ resources/playbook/host_vars/ resources/playbook/group_vars/ +tests/resources/* +!test/resources/test_configuration.yml # any log files *.log From f5532e7c7fb3f8e6be38d0681ca0c9c548c462f4 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 14 Nov 2023 14:06:44 +0100 Subject: [PATCH 28/34] fixing test_provider.py --- tests/provider/test_provider.py | 8 +++++--- tests/startup_tests.py | 13 ++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/provider/test_provider.py b/tests/provider/test_provider.py index ef19b69d..32a472a6 100644 --- a/tests/provider/test_provider.py +++ b/tests/provider/test_provider.py @@ -4,6 +4,7 @@ import os import unittest +import logging import bibigrid.core.utility.handler.configuration_handler as configurationHandler import bibigrid.core.utility.handler.provider_handler as providerHandler @@ -61,9 +62,9 @@ "MFbUTTukAiDf4jAgvJkg7ayE0MPapGpI/OhSK2gyN45VAzs2m7uykun87B491JagZ57qr16vt8vxGYpFCEe8QqAcrUszUPqyPrb0auA8bz" \ "jO8S41Kx8FfG+7eTu4dQ0= user" -CONFIGURATIONS = configurationHandler.read_configuration( +CONFIGURATIONS = configurationHandler.read_configuration(logging, os.path.join(bP.ROOT_PATH, "tests/resources/infrastructure_cloud.yml")) -PROVIDERS = providerHandler.get_providers(CONFIGURATIONS) +PROVIDERS = providerHandler.get_providers(CONFIGURATIONS, logging) class ProviderServer: @@ -95,6 +96,7 @@ def test_get_free_resources(self): with self.subTest(provider.NAME): free_dict = provider.get_free_resources() self.assertEqual(FREE_RESOURCES_KEYS, set(free_dict.keys())) + print(free_dict) for value in free_dict.values(): self.assertLessEqual(0, value) @@ -209,7 +211,7 @@ def test_get_image_mismatch(self): with self.subTest(provider.NAME): self.assertIsNone(provider.get_image_by_id_or_name("NONE")) - if os.environ.get("OS_SNAPSHOT"): + if CONFIGURATIONS[0].get("snapshot_image"): def test_get_snapshot(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): with self.subTest(provider.NAME): diff --git a/tests/startup_tests.py b/tests/startup_tests.py index 876b6d1f..467e9b7e 100644 --- a/tests/startup_tests.py +++ b/tests/startup_tests.py @@ -34,13 +34,12 @@ def suppress_stdout(): logging.basicConfig(level=logging.ERROR) if __name__ == '__main__': # Unittests - suite = unittest.TestLoader().discover("./", pattern='test_*.py') - with suppress_stdout(): - unittest.TextTestRunner(verbosity=2).run(suite) + #suite = unittest.TestLoader().discover("./", pattern='test_*.py') + #with suppress_stdout(): + # unittest.TextTestRunner(verbosity=2).run(suite) # Provider-Test # Configuration needs to contain providers and infrastructures - if os.environ.get("OS_KEY_NAME"): - suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') - with suppress_stdout(): - unittest.TextTestRunner(verbosity=2).run(suite) + suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') + with suppress_stdout(): + unittest.TextTestRunner(verbosity=2).run(suite) From 6bbc3495689233d978e6b7374ba6267c157bcc37 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 14 Nov 2023 16:31:20 +0100 Subject: [PATCH 29/34] removed infrastructure_cloud.yml --- tests/resources/infrastructure_cloud.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 tests/resources/infrastructure_cloud.yml diff --git a/tests/resources/infrastructure_cloud.yml b/tests/resources/infrastructure_cloud.yml deleted file mode 100644 index 028c0247..00000000 --- a/tests/resources/infrastructure_cloud.yml +++ /dev/null @@ -1,10 +0,0 @@ - # See https://cloud.denbi.de/wiki/Tutorials/BiBiGrid/ (after update) - -- infrastructure: openstack # former mode. - cloud: openstack # name of clouds.yaml entry - image: 2e61eb1b-dbd2-4ed8-b62b-5ee9fe0510e6 - flavor: de.NBI tiny - network: network0 - snapshot_image: test - -#- [next configurations] \ No newline at end of file From 9b0629af5422bc17b2b1a8f50e128009a14d5670 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Tue, 14 Nov 2023 17:08:21 +0100 Subject: [PATCH 30/34] minor fixes --- tests/provider/test_provider.py | 27 ++++++++++++++------------- tests/startup_tests.py | 4 ++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/provider/test_provider.py b/tests/provider/test_provider.py index 32a472a6..545ee7d6 100644 --- a/tests/provider/test_provider.py +++ b/tests/provider/test_provider.py @@ -2,13 +2,14 @@ Module containing integration and unit tests regarding the provider """ +import logging import os import unittest -import logging -import bibigrid.core.utility.handler.configuration_handler as configurationHandler -import bibigrid.core.utility.handler.provider_handler as providerHandler import bibigrid.core.utility.paths.basic_path as bP +from bibigrid.core.utility.handler import configuration_handler +from bibigrid.core.utility.handler import provider_handler +from bibigrid.models.exceptions import ExecutionException SERVER_KEYS = {'id', 'name', 'flavor', 'image', 'block_device_mapping', 'location', 'volumes', 'has_config_drive', 'host_id', 'progress', 'disk_config', 'power_state', 'task_state', 'vm_state', 'launched_at', @@ -62,9 +63,9 @@ "MFbUTTukAiDf4jAgvJkg7ayE0MPapGpI/OhSK2gyN45VAzs2m7uykun87B491JagZ57qr16vt8vxGYpFCEe8QqAcrUszUPqyPrb0auA8bz" \ "jO8S41Kx8FfG+7eTu4dQ0= user" -CONFIGURATIONS = configurationHandler.read_configuration(logging, - os.path.join(bP.ROOT_PATH, "tests/resources/infrastructure_cloud.yml")) -PROVIDERS = providerHandler.get_providers(CONFIGURATIONS, logging) +CONFIGURATIONS = configuration_handler.read_configuration(logging, os.path.join(bP.ROOT_PATH, + "tests/resources/infrastructure_cloud.yml")) +PROVIDERS = provider_handler.get_providers(CONFIGURATIONS, logging) class ProviderServer: @@ -96,9 +97,9 @@ def test_get_free_resources(self): with self.subTest(provider.NAME): free_dict = provider.get_free_resources() self.assertEqual(FREE_RESOURCES_KEYS, set(free_dict.keys())) - print(free_dict) - for value in free_dict.values(): - self.assertLessEqual(0, value) + for key, value in free_dict.items(): + if key != "floating_ips": + self.assertLessEqual(0, value) def test_server_start_type_error(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): @@ -117,16 +118,16 @@ def test_server_start_type_error(self): def test_server_start_attribute_error(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): with self.subTest(provider.NAME): - with self.assertRaises(AttributeError): + with self.assertRaises(ExecutionException): provider.create_server(name="name", image="ERROR", flavor=configuration["flavor"], network=configuration["network"]) - with self.assertRaises(AttributeError): + with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor="ERROR", image=configuration["image"], network=configuration["network"]) - with self.assertRaises(AttributeError): + with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"], network="ERROR") - with self.assertRaises(AttributeError): + with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"], network=configuration["network"], key_name="ERROR") diff --git a/tests/startup_tests.py b/tests/startup_tests.py index 467e9b7e..2ac2823e 100644 --- a/tests/startup_tests.py +++ b/tests/startup_tests.py @@ -41,5 +41,5 @@ def suppress_stdout(): # Provider-Test # Configuration needs to contain providers and infrastructures suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') - with suppress_stdout(): - unittest.TextTestRunner(verbosity=2).run(suite) + # with suppress_stdout(): + unittest.TextTestRunner(verbosity=2).run(suite) From 0f83ceeb80ae3c56ea54bc4fd245ec77feeb930a Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 16 Nov 2023 15:52:54 +0100 Subject: [PATCH 31/34] fixed all tests --- bibigrid/openstack/openstack_provider.py | 2 ++ tests/provider/test_provider.py | 38 ++++++++++++++---------- tests/startup_tests.py | 10 +++---- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/bibigrid/openstack/openstack_provider.py b/bibigrid/openstack/openstack_provider.py index 15ec3ef9..ae97a2ec 100644 --- a/bibigrid/openstack/openstack_provider.py +++ b/bibigrid/openstack/openstack_provider.py @@ -121,6 +121,8 @@ def create_server(self, name, flavor, image, network, key_name=None, wait=True, except openstack.exceptions.BadRequestException as exc: if "is not active" in str(exc): raise ImageDeactivatedException() from exc + if "Invalid key_name provided" in str(exc): + raise ExecutionException() from exc raise ConnectionError() from exc except openstack.exceptions.SDKException as exc: raise ExecutionException() from exc diff --git a/tests/provider/test_provider.py b/tests/provider/test_provider.py index 545ee7d6..45b0de7e 100644 --- a/tests/provider/test_provider.py +++ b/tests/provider/test_provider.py @@ -20,7 +20,12 @@ 'updated', 'user_id', 'tags', 'interface_ip', 'properties', 'hostId', 'config_drive', 'project_id', 'tenant_id', 'region', 'cloud', 'az', 'OS-DCF:diskConfig', 'OS-EXT-AZ:availability_zone', 'OS-SRV-USG:launched_at', 'OS-SRV-USG:terminated_at', 'OS-EXT-STS:task_state', 'OS-EXT-STS:vm_state', - 'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached'} + 'OS-EXT-STS:power_state', 'os-extended-volumes:volumes_attached', 'OS-EXT-SRV-ATTR:ramdisk_id', + 'max_count', 'trusted_image_certificates', 'OS-SCH-HNT:scheduler_hints', + 'OS-EXT-SRV-ATTR:reservation_id', 'OS-EXT-SRV-ATTR:host', 'locked', 'host_status', + 'OS-EXT-SRV-ATTR:hypervisor_hostname', 'OS-EXT-SRV-ATTR:launch_index', + 'OS-EXT-SRV-ATTR:root_device_name', 'OS-EXT-SRV-ATTR:instance_name', 'OS-EXT-SRV-ATTR:user_data', + 'min_count', 'OS-EXT-SRV-ATTR:kernel_id', 'OS-EXT-SRV-ATTR:hostname'} FLOATING_IP_KEYS = {'attached', 'fixed_ip_address', 'floating_ip_address', 'id', 'location', 'network', 'port', 'router', 'status', 'created_at', 'updated_at', 'description', 'revision_number', 'properties', 'port_id', 'router_id', 'project_id', 'tenant_id', 'floating_network_id', 'port_details', @@ -41,7 +46,7 @@ 'is_protected', 'locations', 'properties', 'is_public', 'visibility', 'description', 'owner_specified.openstack.md5', 'owner_specified.openstack.object', 'owner_specified.openstack.sha256', 'os_hidden', 'os_hash_algo', 'os_hash_value', 'os_distro', 'os_version', 'schema', 'protected', - 'metadata', 'created', 'updated', 'minDisk', 'minRam'} + 'metadata', 'created', 'updated', 'minDisk', 'minRam', 'stores'} SNAPSHOT_KEYS = {'id', 'created_at', 'updated_at', 'name', 'description', 'volume_id', 'status', 'size', 'metadata', 'os-extended-snapshot-attributes:project_id', 'os-extended-snapshot-attributes:progress'} @@ -63,8 +68,9 @@ "MFbUTTukAiDf4jAgvJkg7ayE0MPapGpI/OhSK2gyN45VAzs2m7uykun87B491JagZ57qr16vt8vxGYpFCEe8QqAcrUszUPqyPrb0auA8bz" \ "jO8S41Kx8FfG+7eTu4dQ0= user" -CONFIGURATIONS = configuration_handler.read_configuration(logging, os.path.join(bP.ROOT_PATH, - "tests/resources/infrastructure_cloud.yml")) +CONFIGURATIONS = configuration_handler.read_configuration(logging, + os.path.join(bP.ROOT_PATH, + "tests/resources/infrastructure_cloud.yml")) PROVIDERS = provider_handler.get_providers(CONFIGURATIONS, logging) @@ -78,7 +84,7 @@ def __init__(self, provider, name, configuration, key_name=None): self.name = name self.server_dict = provider.create_server(name=self.name, flavor=configuration["flavor"], image=configuration["image"], network=configuration["network"], - key_name=key_name) + key_name=key_name, security_groups=[]) def __enter__(self): return self.server_dict @@ -106,30 +112,32 @@ def test_server_start_type_error(self): with self.subTest(provider.NAME): with self.assertRaises(TypeError): provider.create_server(name="name", flavor=configuration["flavor"], - network=configuration["network"]) + network=configuration["network"], security_groups=[]) with self.assertRaises(TypeError): - provider.create_server(name="name", image=configuration["image"], network=configuration["network"]) + provider.create_server(name="name", image=configuration["image"], network=configuration["network"], + security_groups=[]) with self.assertRaises(TypeError): provider.create_server(flavor=configuration["flavor"], image=configuration["image"], - network=configuration["network"]) + security_groups=[], network=configuration["network"]) with self.assertRaises(TypeError): - provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"]) + provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"], + security_groups=[]) def test_server_start_attribute_error(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): with self.subTest(provider.NAME): with self.assertRaises(ExecutionException): provider.create_server(name="name", image="ERROR", flavor=configuration["flavor"], - network=configuration["network"]) + network=configuration["network"], security_groups=[]) with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor="ERROR", image=configuration["image"], - network=configuration["network"]) + network=configuration["network"], security_groups=[]) with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"], - network="ERROR") + network="ERROR", security_groups=[]) with self.assertRaises(ExecutionException): provider.create_server(name="name", flavor=configuration["flavor"], image=configuration["image"], - network=configuration["network"], key_name="ERROR") + network=configuration["network"], key_name="ERROR", security_groups=[]) def test_create_keypair_create_delete_false_delete(self): for provider in PROVIDERS: @@ -144,8 +152,8 @@ def test_active_server_methods(self): with self.subTest(provider.NAME): with ProviderServer(provider, "bibigrid_test_server", configuration, "bibigrid_test_keypair") as provider_server: - floating_ip = provider.create_floating_ip(provider.get_external_network(configuration["network"]), - provider_server) + floating_ip = provider.attach_available_floating_ip( + provider.get_external_network(configuration["network"]), provider_server) server_list = provider.list_servers() self.assertEqual(SERVER_KEYS, set(provider_server.keys())) self.assertEqual("bibigrid_test_keypair", provider_server["key_name"]) diff --git a/tests/startup_tests.py b/tests/startup_tests.py index 2ac2823e..4a719ef2 100644 --- a/tests/startup_tests.py +++ b/tests/startup_tests.py @@ -34,12 +34,12 @@ def suppress_stdout(): logging.basicConfig(level=logging.ERROR) if __name__ == '__main__': # Unittests - #suite = unittest.TestLoader().discover("./", pattern='test_*.py') - #with suppress_stdout(): - # unittest.TextTestRunner(verbosity=2).run(suite) + suite = unittest.TestLoader().discover("./", pattern='test_*.py') + with suppress_stdout(): + unittest.TextTestRunner(verbosity=2).run(suite) # Provider-Test # Configuration needs to contain providers and infrastructures suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') - # with suppress_stdout(): - unittest.TextTestRunner(verbosity=2).run(suite) + with suppress_stdout(): + unittest.TextTestRunner(verbosity=2).run(suite) From 422eda580c909be86bf9611bb2eee2d7f0db57c5 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 16 Nov 2023 16:52:13 +0100 Subject: [PATCH 32/34] removed print --- bibigrid/core/actions/ide.py | 3 +-- bibigrid/core/startup.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bibigrid/core/actions/ide.py b/bibigrid/core/actions/ide.py index 840d970b..1ad2c871 100644 --- a/bibigrid/core/actions/ide.py +++ b/bibigrid/core/actions/ide.py @@ -23,7 +23,6 @@ LOCALHOST = "127.0.0.1" - def sigint_handler(caught_signal, frame): # pylint: disable=unused-argument """ Is called when SIGINT is thrown and terminates the program @@ -82,7 +81,7 @@ def ide(cluster_id, master_provider, master_configuration, log): ssh_pkey=used_private_key, local_bind_address=(LOCALHOST, used_local_bind_address), remote_bind_address=(LOCALHOST, REMOTE_BIND_ADDRESS)) as server: - print("CTRL+C to close port forwarding when you are done.") + log.log(42, "CTRL+C to close port forwarding when you are done.") with server: # opens in existing window if any default program exists webbrowser.open(f"http://localhost:{used_local_bind_address}", new=2) diff --git a/bibigrid/core/startup.py b/bibigrid/core/startup.py index 4676f44e..3a073f27 100755 --- a/bibigrid/core/startup.py +++ b/bibigrid/core/startup.py @@ -70,7 +70,6 @@ def run_action(args, configurations, config_path): exit_state = 0 try: providers = provider_handler.get_providers(configurations, LOG) - print(args) if providers: if args.list: LOG.info("Action list selected") From 975f2e54ec5090ded70bafe37491a0a34b4498b6 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 23 Nov 2023 16:10:51 +0100 Subject: [PATCH 33/34] changed prints to log --- bibigrid/core/actions/list_clusters.py | 6 +++--- bibigrid/core/utility/validate_configuration.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bibigrid/core/actions/list_clusters.py b/bibigrid/core/actions/list_clusters.py index 99e1b6bc..554b7221 100644 --- a/bibigrid/core/actions/list_clusters.py +++ b/bibigrid/core/actions/list_clusters.py @@ -24,8 +24,8 @@ def dict_clusters(providers, log): servers = provider.list_servers() for server in servers: result = SERVER_REGEX.match(server["name"]) - print(server["name"]) - print(result) + log.log(42, server["name"]) + log.log(42, result) if result: identifier = result.group(4) or result.group(2) cluster_id = result.group(5) or result.group(3) @@ -71,7 +71,7 @@ def log_list(cluster_id, providers, log): master_count, worker_count, vpn_count = get_size_overview(cluster_dict[cluster_id], log) log.log(42, f"\tCluster has {master_count} master, {vpn_count} vpngtw and {worker_count} regular workers. " f"The cluster is spread over {vpn_count + master_count} reachable provider(s).") - pprint.pprint(cluster_dict[cluster_id]) + log.log(42, pprint.pformat(cluster_dict[cluster_id])) else: log.info("Cluster with cluster-id {cluster_id} not found.") log.log(42, f"Cluster with cluster-id {cluster_id} not found.") diff --git a/bibigrid/core/utility/validate_configuration.py b/bibigrid/core/utility/validate_configuration.py index 72deb806..a4232d2e 100644 --- a/bibigrid/core/utility/validate_configuration.py +++ b/bibigrid/core/utility/validate_configuration.py @@ -303,8 +303,8 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p flavor = provider.get_flavor(instance_type) if not flavor: self.log.warning("Flavor %s does not exist.", instance_type) - print("Available flavors:") - print("\n".join(provider.get_active_flavors())) + self.log.log(42, "Available flavors:") + self.log.log(42, "\n".join(provider.get_active_flavors())) return False type_max_disk_space = flavor["disk"] type_max_ram = flavor["ram"] From 4b47b5103e1d7eefc64596279a0592adbd849cb0 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier Date: Thu, 23 Nov 2023 16:13:44 +0100 Subject: [PATCH 34/34] removed log --- bibigrid/core/actions/list_clusters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bibigrid/core/actions/list_clusters.py b/bibigrid/core/actions/list_clusters.py index 554b7221..90e7aa5e 100644 --- a/bibigrid/core/actions/list_clusters.py +++ b/bibigrid/core/actions/list_clusters.py @@ -24,8 +24,6 @@ def dict_clusters(providers, log): servers = provider.list_servers() for server in servers: result = SERVER_REGEX.match(server["name"]) - log.log(42, server["name"]) - log.log(42, result) if result: identifier = result.group(4) or result.group(2) cluster_id = result.group(5) or result.group(3)