Fix python tests error handling

test_collections/sdk_tests/support/chip_tool/chip_tool.py

@@ -494,6 +494,19 @@ def last_command_exit_code(self) -> Optional[int]:
         exit_code = exec_data.get("ExitCode")
         return exit_code
 
+    def exec_exit_code(self, exec_id: str) -> Optional[int]:
+        if self.__chip_tool_container is None:
+            self.logger.info("No SDK container, cannot get execution exit code")
+            return None
+
+        exec_data = self.__chip_tool_container.client.api.exec_inspect(exec_id)
+
+        if exec_data is None:
+            self.logger.error("Docker didn't return any execution metadata")
+            return None
+
+        return exec_data.get("ExitCode")
+
     async def send_websocket_command(self, cmd: str) -> Union[str, bytes, bytearray]:
         await self.start_runner()
         response = await self.__test_harness_runner.execute(cmd)

test_collections/sdk_tests/support/python_testing/models/test_case.py

@@ -93,15 +93,22 @@ def step_start(self, name: str) -> None:
         pass
 
     def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None:
-        # TODO Handle Logs properly
         self.step_over()
 
     def step_failure(
         self, logger: Any, logs: str, duration: int, request: Any, received: Any
     ) -> None:
-        # TODO Handle Logs properly
         self.mark_step_failure("Python test step failure")
-        self.step_over()
+
+        # Python tests with only 2 steps are the ones that don't follow the template.
+        # In the case of a test file with multiple test cases, more than one of these
+        # tests can fail and so this method will be called for each of them. These
+        # failures should be reported in the first step and moving to the logs step
+        # should only happen after all test cases are executed.
+        if len(self.test_steps) > 2:
+            # Python tests stop when there's a failure. We need to skip the next steps
+            # and execute only the last one, which shows the logs
+            self.skip_to_last_step()
 
     def step_unknown(self) -> None:
         self.__runned += 1
@@ -212,6 +219,11 @@ async def execute(self) -> None:
         finally:
             pass
 
+    def skip_to_last_step(self) -> None:
+        self.current_test_step.mark_as_completed()
+        self.current_test_step_index = len(self.test_steps) - 1
+        self.current_test_step.mark_as_executing()
+
     def __handle_update(self, update: SDKPythonTestResultBase) -> None:
         self.__call_function_from_name(update.type.value, update.params_dict())
 

test_collections/sdk_tests/support/python_testing/models/test_suite.py

@@ -32,6 +32,10 @@ class SuiteType(Enum):
     AUTOMATED = 1
 
 
+class DUTCommissioningError(Exception):
+    pass
+
+
 # Custom Type variable used to annotate the factory methods of classmethod.
 T = TypeVar("T", bound="PythonTestSuite")
 
@@ -103,3 +107,8 @@ def commission_device(self) -> None:
         )
 
         handle_logs(cast(Generator, exec_result.output), logger)
+
+        exit_code = self.chip_tool.exec_exit_code(exec_result.exec_id)
+
+        if exit_code:
+            raise DUTCommissioningError("Failed to commission DUT")

test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py

@@ -25,6 +25,9 @@
 from app.test_engine.logger import test_engine_logger
 from app.tests.utils.test_pics_data import create_random_pics
 from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.exec_run_in_container import (
+    ExecResultExtended,
+)
 from test_collections.sdk_tests.support.python_testing.models.test_suite import (
     PythonTestSuite,
     SuiteType,
@@ -167,14 +170,15 @@ async def test_commission_device() -> None:
     command_args = ["arg1", "arg2", "arg3"]
     expected_command = [f"{RUNNER_CLASS_PATH} commission"]
     expected_command.extend(command_args)
+    mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock())
 
     suite = PythonTestSuite(TestSuiteExecution())
 
     with mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch(
         target="test_collections.sdk_tests.support.python_testing.models.test_suite"
         ".PythonTestSuite.config"
     ), mock.patch.object(
-        target=chip_tool, attribute="send_command"
+        target=chip_tool, attribute="send_command", return_value=mock_result
     ) as mock_send_command, mock.patch(
         target="test_collections.sdk_tests.support.python_testing.models.test_suite"
         ".generate_command_arguments",