From 74b94780f646458c8dbfa80e53b83be6d3e485f9 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Tue, 12 Dec 2023 18:35:58 +0000 Subject: [PATCH 1/4] Handle commissioning error on python test suite --- .../sdk_tests/support/python_testing/models/test_suite.py | 7 +++++++ .../support/tests/python_tests/test_python_test_suite.py | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 9ca85534..80554582 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -32,6 +32,10 @@ class SuiteType(Enum): AUTOMATED = 1 +class DUTCommissioningError(Exception): + pass + + # Custom Type variable used to annotate the factory methods of classmethod. T = TypeVar("T", bound="PythonTestSuite") @@ -103,3 +107,6 @@ def commission_device(self) -> None: ) handle_logs(cast(Generator, exec_result.output), logger) + + if exec_result.exit_code != 0: + raise DUTCommissioningError("Failed to commission DUT") diff --git a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py index a335ec94..f0492a92 100644 --- a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py +++ b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py @@ -25,6 +25,9 @@ from app.test_engine.logger import test_engine_logger from app.tests.utils.test_pics_data import create_random_pics from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipTool +from test_collections.sdk_tests.support.chip_tool.exec_run_in_container import ( + ExecResultExtended, +) from test_collections.sdk_tests.support.python_testing.models.test_suite import ( PythonTestSuite, SuiteType, @@ -167,6 +170,7 @@ async def test_commission_device() -> None: command_args = ["arg1", "arg2", "arg3"] expected_command = [f"{RUNNER_CLASS_PATH} commission"] expected_command.extend(command_args) + mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) suite = PythonTestSuite(TestSuiteExecution()) @@ -174,7 +178,7 @@ async def test_commission_device() -> None: target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".PythonTestSuite.config" ), mock.patch.object( - target=chip_tool, attribute="send_command" + target=chip_tool, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".generate_command_arguments", From 9936bb43ce70dd954f917b9764cc06e52e24dc07 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Tue, 12 Dec 2023 18:36:38 +0000 Subject: [PATCH 2/4] Fix steps state after a failure in python tests --- .../support/python_testing/models/test_case.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 9222f2ef..ae0c8cf7 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -93,15 +93,16 @@ def step_start(self, name: str) -> None: pass def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: - # TODO Handle Logs properly self.step_over() def step_failure( self, logger: Any, logs: str, duration: int, request: Any, received: Any ) -> None: - # TODO Handle Logs properly self.mark_step_failure("Python test step failure") - self.step_over() + + # Python tests stop when there's a failure. We need to skip the next steps + # and execute only the last one, which shows the logs + self.skip_to_last_step() def step_unknown(self) -> None: self.__runned += 1 @@ -212,6 +213,11 @@ async def execute(self) -> None: finally: pass + def skip_to_last_step(self) -> None: + self.current_test_step.mark_as_completed() + self.current_test_step_index = len(self.test_steps) - 1 + self.current_test_step.mark_as_executing() + def __handle_update(self, update: SDKPythonTestResultBase) -> None: self.__call_function_from_name(update.type.value, update.params_dict()) From c0cff4b011f3c221170b008d541df41a84463295 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Tue, 12 Dec 2023 20:23:06 +0000 Subject: [PATCH 3/4] Fix logic for commissioning result validation --- .../sdk_tests/support/chip_tool/chip_tool.py | 13 +++++++++++++ .../support/python_testing/models/test_suite.py | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test_collections/sdk_tests/support/chip_tool/chip_tool.py b/test_collections/sdk_tests/support/chip_tool/chip_tool.py index eba63e6d..04895107 100644 --- a/test_collections/sdk_tests/support/chip_tool/chip_tool.py +++ b/test_collections/sdk_tests/support/chip_tool/chip_tool.py @@ -494,6 +494,19 @@ def last_command_exit_code(self) -> Optional[int]: exit_code = exec_data.get("ExitCode") return exit_code + def exec_exit_code(self, exec_id: str) -> Optional[int]: + if self.__chip_tool_container is None: + self.logger.info("No SDK container, cannot get execution exit code") + return None + + exec_data = self.__chip_tool_container.client.api.exec_inspect(exec_id) + + if exec_data is None: + self.logger.error("Docker didn't return any execution metadata") + return None + + return exec_data.get("ExitCode") + async def send_websocket_command(self, cmd: str) -> Union[str, bytes, bytearray]: await self.start_runner() response = await self.__test_harness_runner.execute(cmd) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 80554582..bc655065 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -108,5 +108,7 @@ def commission_device(self) -> None: handle_logs(cast(Generator, exec_result.output), logger) - if exec_result.exit_code != 0: + exit_code = self.chip_tool.exec_exit_code(exec_result.exec_id) + + if exit_code: raise DUTCommissioningError("Failed to commission DUT") From e590edda9487b2daa4558efefbb2e89ea7886484 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Wed, 13 Dec 2023 11:24:12 +0000 Subject: [PATCH 4/4] Fix step state for test cases the don't follow the template --- .../support/python_testing/models/test_case.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index ae0c8cf7..c3f569f3 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -100,9 +100,15 @@ def step_failure( ) -> None: self.mark_step_failure("Python test step failure") - # Python tests stop when there's a failure. We need to skip the next steps - # and execute only the last one, which shows the logs - self.skip_to_last_step() + # Python tests with only 2 steps are the ones that don't follow the template. + # In the case of a test file with multiple test cases, more than one of these + # tests can fail and so this method will be called for each of them. These + # failures should be reported in the first step and moving to the logs step + # should only happen after all test cases are executed. + if len(self.test_steps) > 2: + # Python tests stop when there's a failure. We need to skip the next steps + # and execute only the last one, which shows the logs + self.skip_to_last_step() def step_unknown(self) -> None: self.__runned += 1