From aed2b6ef7fa683fe7ea8186966f5c19c6b81e5c5 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Mon, 11 Dec 2023 17:39:22 +0000 Subject: [PATCH 1/4] Show python logs at the end of the test execution --- .../python_testing/models/test_case.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 52921d7f..c08bb5e5 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -16,7 +16,7 @@ import re from asyncio import sleep from multiprocessing.managers import BaseManager -from typing import Any, Type, TypeVar +from typing import Any, Generator, Type, TypeVar, cast from app.models import TestCaseExecution from app.test_engine.logger import test_engine_logger as logger @@ -31,7 +31,12 @@ SDKPythonTestResultBase, SDKPythonTestRunnerHooks, ) -from .utils import EXECUTABLE, RUNNER_CLASS_PATH, generate_command_arguments +from .utils import ( + EXECUTABLE, + RUNNER_CLASS_PATH, + generate_command_arguments, + handle_logs, +) # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") @@ -172,13 +177,12 @@ async def execute(self) -> None: if self.chip_tool.pics_file_created: command.append(f" --PICS {PICS_FILE_PATH}") - # TODO Ignoring stream from docker execution - self.chip_tool.send_command( + exec_result = self.chip_tool.send_command( command, prefix=EXECUTABLE, is_stream=True, is_socket=False, - ).output + ) while ((update := test_runner_hooks.update_test()) is not None) or ( not test_runner_hooks.is_finished() @@ -189,6 +193,11 @@ async def execute(self) -> None: self.__handle_update(update) + # Step: Show test logs + self.next_step() + logger.info("---- Start of Python test logs ----") + handle_logs(cast(Generator, exec_result.output), logger) + logger.info("---- End of Python test logs ----") finally: pass @@ -208,3 +217,4 @@ def create_test_steps(self) -> None: for step in self.python_test.steps: python_test_step = TestStep(step.label) self.test_steps.append(python_test_step) + self.test_steps.append(TestStep("Show test logs")) From b852a01a3a9902b53151fd6a6834a8eb19cd3fac Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Mon, 11 Dec 2023 18:11:46 +0000 Subject: [PATCH 2/4] Fix next_step logic in python test --- .../support/python_testing/models/test_case.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index c08bb5e5..3e997ec9 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -62,10 +62,10 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: self.test_stop_called = False def next_step(self) -> None: - # Python tests that don't follow the template only have the default step "Start - # Python test", but inside the file there can be more than one test case, so the - # hooks steps methods will continue to be called - if len(self.test_steps) == 1: + # Python tests that don't follow the template only have the default steps "Start + # Python test" and "Show test logs", but inside the file there can be more than + # one test case, so the hooks steps methods will continue to be called + if len(self.test_steps) == 2: return super().next_step() @@ -194,7 +194,8 @@ async def execute(self) -> None: self.__handle_update(update) # Step: Show test logs - self.next_step() + # Skip the override because here it should always go to the next step + super().next_step() logger.info("---- Start of Python test logs ----") handle_logs(cast(Generator, exec_result.output), logger) logger.info("---- End of Python test logs ----") From abe42154475039710b7bf64df700aebc43b09f30 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Mon, 11 Dec 2023 18:19:12 +0000 Subject: [PATCH 3/4] New fix for next_step logic --- .../python_testing/models/test_case.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 3e997ec9..c9a935c7 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -61,14 +61,16 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: self.__runned = 0 self.test_stop_called = False - def next_step(self) -> None: + # Move to the next step if the test case has additional steps appart from the 2 + # deafult ones + def step_over(self) -> None: # Python tests that don't follow the template only have the default steps "Start # Python test" and "Show test logs", but inside the file there can be more than - # one test case, so the hooks steps methods will continue to be called + # one test case, so the hooks' step methods will continue to be called if len(self.test_steps) == 2: return - super().next_step() + self.next_step() def start(self, count: int) -> None: pass @@ -78,29 +80,28 @@ def stop(self, duration: int) -> None: self.current_test_step.mark_as_completed() def test_start(self, filename: str, name: str, count: int) -> None: - self.next_step() + self.step_over() def test_stop(self, exception: Exception, duration: int) -> None: self.test_stop_called = True - self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: self.current_test_step.mark_as_not_applicable("Test step skipped") - self.next_step() + self.step_over() def step_start(self, name: str) -> None: pass def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: # TODO Handle Logs properly - self.next_step() + self.step_over() def step_failure( self, logger: Any, logs: str, duration: int, request: Any, received: Any ) -> None: # TODO Handle Logs properly self.mark_step_failure("Python test step failure") - self.next_step() + self.step_over() def step_unknown(self) -> None: self.__runned += 1 @@ -194,11 +195,20 @@ async def execute(self) -> None: self.__handle_update(update) # Step: Show test logs - # Skip the override because here it should always go to the next step - super().next_step() + + # Python tests that don't follow the template only have the 2 default steps + # and, at this point, will still be in the first step because of the + # step_over method. So we have to explicitly move on to the next step here. + # The tests that do follow the template will have additional steps and will + # have already been moved to the correct step by the hooks' step methods. + if len(self.test_steps) == 2: + self.next_step() + logger.info("---- Start of Python test logs ----") handle_logs(cast(Generator, exec_result.output), logger) logger.info("---- End of Python test logs ----") + + self.current_test_step.mark_as_completed() finally: pass From 7dc0f2d375e206ed585e772ebfd01b2e864d38f9 Mon Sep 17 00:00:00 2001 From: Carolina Lopes Date: Mon, 11 Dec 2023 18:57:37 +0000 Subject: [PATCH 4/4] Fix typo --- .../sdk_tests/support/python_testing/models/test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index c9a935c7..9222f2ef 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -61,7 +61,7 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: self.__runned = 0 self.test_stop_called = False - # Move to the next step if the test case has additional steps appart from the 2 + # Move to the next step if the test case has additional steps apart from the 2 # deafult ones def step_over(self) -> None: # Python tests that don't follow the template only have the default steps "Start