From 549841f5d290416b536b7938a2c11c40489cbaa1 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 21 Feb 2024 15:35:02 -0800 Subject: [PATCH] Update integ test runner to record each configuration separately (#4438) Signed-off-by: Zelin Hao --- .../integ_test/integ_test_runner.py | 2 +- .../integ_test/integ_test_suite_opensearch.py | 18 ++--- .../integ_test_suite_opensearch_dashboards.py | 18 ++--- .../test_recorder/test_recorder.py | 3 + .../test_integ_test_runner_opensearch.py | 2 +- ...integ_test_runner_opensearch_dashboards.py | 2 +- .../test_integ_test_suite_opensearch.py | 73 ++++++++++++++++--- .../test_recorder/test_test_recorder.py | 3 + 8 files changed, 91 insertions(+), 30 deletions(-) diff --git a/src/test_workflow/integ_test/integ_test_runner.py b/src/test_workflow/integ_test/integ_test_runner.py index c9f04050fa..e2959a5c31 100644 --- a/src/test_workflow/integ_test/integ_test_runner.py +++ b/src/test_workflow/integ_test/integ_test_runner.py @@ -45,7 +45,7 @@ def run(self) -> TestSuiteResults: if test_config.integ_test: test_suite = self.__create_test_suite__(component, test_config, work_dir.path) test_results = test_suite.execute_tests() - [self.test_recorder.test_results_logs.save_test_result_data(result_data) for result_data in test_suite.result_data] + [self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data] all_results.append(component.name, test_results) else: logging.info(f"Skipping integ-tests for {component.name}, as it is currently not supported") diff --git a/src/test_workflow/integ_test/integ_test_suite_opensearch.py b/src/test_workflow/integ_test/integ_test_suite_opensearch.py index dfbf2bd7da..42c6b391ae 100644 --- a/src/test_workflow/integ_test/integ_test_suite_opensearch.py +++ b/src/test_workflow/integ_test/integ_test_suite_opensearch.py @@ -121,16 +121,16 @@ def custom_node_endpoint_encoder(node_endpoint: NodeEndpoint) -> dict: self.repo_work_dir = os.path.join( self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False) - self.test_result_data.append( - TestResultData( - self.component.name, - test_config, - status, - stdout, - stderr, - self.test_artifact_files - ) + test_result_data_local = TestResultData( + self.component.name, + test_config, + status, + stdout, + stderr, + self.test_artifact_files ) + self.save_logs.save_test_result_data(test_result_data_local) + self.test_result_data.append(test_result_data_local) if stderr: logging.info("Stderr reported for component: " + self.component.name) logging.info(stderr) diff --git a/src/test_workflow/integ_test/integ_test_suite_opensearch_dashboards.py b/src/test_workflow/integ_test/integ_test_suite_opensearch_dashboards.py index a6bbd9dd7f..6edc8500e2 100644 --- a/src/test_workflow/integ_test/integ_test_suite_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/integ_test_suite_opensearch_dashboards.py @@ -114,16 +114,16 @@ def custom_node_endpoint_encoder(node_endpoint: NodeEndpoint) -> dict: self.repo_work_dir = os.path.join( self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False) - self.test_result_data.append( - TestResultData( - self.component.name, - test_config, - status, - stdout, - stderr, - self.test_artifact_files - ) + test_result_data_local = TestResultData( + self.component.name, + test_config, + status, + stdout, + stderr, + self.test_artifact_files ) + self.save_logs.save_test_result_data(test_result_data_local) + self.test_result_data.append(test_result_data_local) if stderr: logging.info("Stderr reported for component: " + self.component.name) logging.info(stderr) diff --git a/src/test_workflow/test_recorder/test_recorder.py b/src/test_workflow/test_recorder/test_recorder.py index 84921fc7a1..4edad516df 100644 --- a/src/test_workflow/test_recorder/test_recorder.py +++ b/src/test_workflow/test_recorder/test_recorder.py @@ -144,6 +144,9 @@ def save_test_result_data(self, test_result_data: TestResultData) -> None: logging.info(f"Recording component test results for {test_result_data.component_name} at " f"{os.path.realpath(dest_directory)}") self.parent_class._generate_std_files(test_result_data.stdout, test_result_data.stderr, dest_directory) self.parent_class._copy_log_files(test_result_data.log_files, dest_directory) + + def generate_component_yml(self, test_result_data: TestResultData) -> None: + dest_directory = self.parent_class._create_base_folder_structure(test_result_data.component_name, test_result_data.component_test_config) self.parent_class._generate_yml(test_result_data, dest_directory) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py index 8375a25f5c..2d86aac3f5 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py @@ -69,7 +69,7 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s self.assertEqual(results["sql"], mock_test_results) mock_suite_object.result_data.__iter__.assert_called() - mock_test_recorder_object.test_results_logs.save_test_result_data.assert_called() + mock_test_recorder_object.test_results_logs.generate_component_yml.assert_called() mock_suite.assert_called_once_with( mock_properties_object.dependency_installer, diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py index 103c6231b7..7540a9e29f 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py @@ -76,7 +76,7 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s self.assertEqual(results["sql"], mock_test_results) mock_suite_object.result_data.__iter__.assert_called() - mock_test_recorder_object.test_results_logs.save_test_result_data.assert_called() + mock_test_recorder_object.test_results_logs.generate_component_yml.assert_called() mock_suite.assert_called_once_with( mock_properties_dependency_object.dependency_installer, diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_suite_opensearch.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_suite_opensearch.py index da8c51b819..3e27504c61 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_suite_opensearch.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_suite_opensearch.py @@ -35,9 +35,10 @@ def setUp(self) -> None: self.work_dir = Path("test_dir") @patch("os.path.exists", return_value=True) + @patch("test_workflow.integ_test.integ_test_suite_opensearch.IntegTestSuiteOpenSearch.multi_execute_integtest_sh") @patch("test_workflow.integ_test.integ_test_suite_opensearch.Topology") @patch("test_workflow.test_recorder.test_recorder.TestRecorder") - def test_execute_with_multiple_test_configs(self, mock_test_recorder: Mock, mock_topology: Mock, *mock: Any) -> None: + def test_execute_with_multiple_test_configs(self, mock_test_recorder: Mock, mock_topology: Mock, mock_multi_execute_integtest_sh: Mock, *mock: Any) -> None: test_config, component = self.__get_test_config_and_bundle_component("job-scheduler") dependency_installer = MagicMock() integ_test_suite = IntegTestSuiteOpenSearch( @@ -50,8 +51,6 @@ def test_execute_with_multiple_test_configs(self, mock_test_recorder: Mock, mock mock_test_recorder ) mock_topology.create().__enter__.return_value = [{"cluster_name": "cluster1", "data_nodes": [{"endpoint": "localhost", "port": 9200, "transport": 9300}], "cluster_manager_nodes": []}] - mock_multi_execute_integtest_sh = MagicMock() - IntegTestSuiteOpenSearch.multi_execute_integtest_sh = mock_multi_execute_integtest_sh # type: ignore mock_multi_execute_integtest_sh.return_value = "success" test_results = integ_test_suite.execute_tests() @@ -63,9 +62,10 @@ def test_execute_with_multiple_test_configs(self, mock_test_recorder: Mock, mock call([{"cluster_name": "cluster1", "data_nodes": [{"endpoint": "localhost", "port": 9200, "transport": 9300}], "cluster_manager_nodes": []}], False, "without-security") ]) + @patch("test_workflow.integ_test.integ_test_suite_opensearch.IntegTestSuiteOpenSearch.multi_execute_integtest_sh") @patch("test_workflow.integ_test.integ_test_suite_opensearch.Topology") @patch("test_workflow.test_recorder.test_recorder.TestRecorder") - def test_execute_with_build_dependencies(self, mock_test_recorder: Mock, mock_topology: Mock, *mock: Any) -> None: + def test_execute_with_build_dependencies(self, mock_test_recorder: Mock, mock_topology: Mock, mock_multi_execute_integtest_sh: Mock, *mock: Any) -> None: dependency_installer = MagicMock() test_config, component = self.__get_test_config_and_bundle_component("index-management") integ_test_suite = IntegTestSuiteOpenSearch( @@ -80,8 +80,6 @@ def test_execute_with_build_dependencies(self, mock_test_recorder: Mock, mock_to mock_topology.create().__enter__.return_value = [{"cluster_name": "cluster1", "data_nodes": [{"endpoint": "localhost", "port": 9200, "transport": 9300}], "cluster_manager_nodes": []}] - mock_multi_execute_integtest_sh = MagicMock() - IntegTestSuiteOpenSearch.multi_execute_integtest_sh = mock_multi_execute_integtest_sh # type: ignore mock_multi_execute_integtest_sh.return_value = "success" integ_test_suite.execute_tests() @@ -93,6 +91,7 @@ def test_execute_with_build_dependencies(self, mock_test_recorder: Mock, mock_to [{"cluster_name": "cluster1", "data_nodes": [{"endpoint": "localhost", "port": 9200, "transport": 9300}], "cluster_manager_nodes": []}], False, "without-security")] ) + @patch("test_workflow.integ_test.integ_test_suite_opensearch.IntegTestSuiteOpenSearch.multi_execute_integtest_sh") @patch("test_workflow.test_recorder.test_recorder.TestRecorder") @patch("test_workflow.integ_test.integ_test_suite_opensearch.execute") def test_execute_without_build_dependencies(self, mock_execute: Mock, *mock: Any) -> None: @@ -160,9 +159,10 @@ def __get_test_config_and_bundle_component(self, component_name: str) -> Tuple[T @patch("os.path.exists", return_value=True) @patch.object(ScriptFinder, "find_integ_test_script") + @patch("test_workflow.integ_test.integ_test_suite_opensearch.IntegTestSuiteOpenSearch.multi_execute_integtest_sh") @patch("test_workflow.integ_test.integ_test_suite_opensearch.Topology") @patch("test_workflow.test_recorder.test_recorder.TestRecorder") - def test_execute_with_working_directory(self, mock_test_recorder: Mock, mock_topology: Mock, mock_script_finder: Mock, *mock: Any) -> None: + def test_execute_with_working_directory(self, mock_test_recorder: Mock, mock_topology: Mock, mock_multi_execute_integtest_sh: Mock, mock_script_finder: Mock, *mock: Any) -> None: test_config, component = self.__get_test_config_and_bundle_component("dashboards-reports") dependency_installer = MagicMock() integ_test_suite = IntegTestSuiteOpenSearch( @@ -178,8 +178,6 @@ def test_execute_with_working_directory(self, mock_test_recorder: Mock, mock_top mock_topology.create().__enter__.return_value = [{"cluster_name": "cluster1", "data_nodes": [{"endpoint": "localhost", "port": 9200, "transport": 9300}], "cluster_manager_nodes": []}] mock_script_finder.return_value = "integtest.sh" - mock_multi_execute_integtest_sh = MagicMock() - IntegTestSuiteOpenSearch.multi_execute_integtest_sh = mock_multi_execute_integtest_sh # type: ignore mock_multi_execute_integtest_sh.return_value = "success" integ_test_suite.execute_tests() # type: ignore @@ -189,3 +187,60 @@ def test_execute_with_working_directory(self, mock_test_recorder: Mock, mock_top True, "with-security" ) + + @patch("os.path.exists") + @patch("os.makedirs") + @patch("test_workflow.test_recorder.test_recorder.TestRecorder") + @patch("test_workflow.integ_test.integ_test_suite_opensearch.TestResultData") + @patch("test_workflow.integ_test.integ_test_suite_opensearch.GitRepository.__checkout__") + @patch("test_workflow.integ_test.integ_test_suite_opensearch.execute", return_value=True) + def test_multi_execute_integtest_sh(self, mock_execute: Mock, mock_git: Mock, mock_test_result_data: Mock, + mock_test_recorder: Mock, mock_makedirs: Mock, mock_path_exists: Mock, *mock: Any) -> None: + mock_find = MagicMock() + mock_find.return_value = "./integtest.sh" + + ScriptFinder.find_integ_test_script = mock_find # type: ignore + + mock_execute.return_value = ("test_status", "test_stdout", "") + + mock_test_result_data_object = MagicMock() + mock_test_result_data.return_value = mock_test_result_data_object + mock_path_exists.return_value = True + + test_config, component = self.__get_test_config_and_bundle_component("job-scheduler") + dependency_installer = MagicMock() + integ_test_suite = IntegTestSuiteOpenSearch( + dependency_installer, + component, + test_config, + self.bundle_manifest, + self.build_manifest, + self.work_dir, + mock_test_recorder + ) + + self.assertEqual(integ_test_suite.repo.url, "https://github.com/opensearch-project/job-scheduler.git") + self.assertEqual(integ_test_suite.repo.ref, "4504dabfc67dd5628c1451e91e9a1c3c4ca71525") + integ_test_suite.repo.dir = "dir" + + # call the test target + mock_endpoint = MagicMock() + status = integ_test_suite.multi_execute_integtest_sh([mock_endpoint], True, "with-security") + + mock_find.assert_called() + self.assertEqual(status, "test_status") + mock_execute.assert_called() + + mock_test_result_data.assert_called_once_with( + "job-scheduler", + "with-security", + "test_status", + "test_stdout", + "", + { + "opensearch-integ-test": os.path.join("dir", "build", "reports", "tests", "integTest") + } + ) + + assert(mock_test_result_data.return_value in integ_test_suite.result_data) + self.assertEqual(integ_test_suite.additional_cluster_config, None) diff --git a/tests/tests_test_workflow/test_recorder/test_test_recorder.py b/tests/tests_test_workflow/test_recorder/test_test_recorder.py index ea09e8d365..5b17f78b42 100644 --- a/tests/tests_test_workflow/test_recorder/test_test_recorder.py +++ b/tests/tests_test_workflow/test_recorder/test_test_recorder.py @@ -191,4 +191,7 @@ def test(self) -> None: logs.save_test_result_data(mock_test_result_data) mock_parent_class._copy_log_files.assert_called_once_with(mock_test_result_data.log_files, dest_directory) + + logs.generate_component_yml(mock_test_result_data) + mock_parent_class._generate_yml.assert_called_once_with(mock_test_result_data, dest_directory)