From 69c5983313b4bcb9f29bdfd54c25626d7d4476cf Mon Sep 17 00:00:00 2001 From: "Antonio Melo Jr." Date: Thu, 5 Sep 2024 20:51:52 +0000 Subject: [PATCH 1/3] Updating the Step Start Hook methods to receive the endpoint info All the related code were updated along with the necessary logging. Also, the UI observable were updated to signalize the frontend websocket with the endpoint information for every step. --- app/test_engine/models/manual_test_case.py | 2 +- app/test_engine/models/test_case.py | 1 + app/test_engine/models/test_step.py | 15 ++++++-- app/test_engine/test_ui_observer.py | 1 + .../support/models/matter_test_models.py | 4 +++ .../models/python_test_parser.py | 14 ++++---- .../models/python_testing_hooks_proxy.py | 34 +++++++++++-------- .../python_testing/models/test_case.py | 3 +- .../support/yaml_tests/models/chip_test.py | 10 ++++-- .../support/yaml_tests/models/test_case.py | 3 +- 10 files changed, 58 insertions(+), 29 deletions(-) diff --git a/app/test_engine/models/manual_test_case.py b/app/test_engine/models/manual_test_case.py index 91a4b467..10c4f57e 100644 --- a/app/test_engine/models/manual_test_case.py +++ b/app/test_engine/models/manual_test_case.py @@ -193,7 +193,7 @@ async def execute(self) -> None: if isinstance(step, ManualVerificationTestStep): await step.prompt_verification_step() else: - logger.error(f"Unsupported test test {step.__class__}") + logger.error(f"Unsupported test step {step.__class__}") self.next_step() if isinstance(self.current_test_step, ManualLogUploadStep): diff --git a/app/test_engine/models/test_case.py b/app/test_engine/models/test_case.py index c45a7c6d..95005290 100644 --- a/app/test_engine/models/test_case.py +++ b/app/test_engine/models/test_case.py @@ -293,6 +293,7 @@ def next_step(self) -> None: # update current step self.current_test_step_index += 1 + self.current_test_step.mark_as_executing() def __print_log_separator(self) -> None: diff --git a/app/test_engine/models/test_step.py b/app/test_engine/models/test_step.py index d666d4c2..71847295 100644 --- a/app/test_engine/models/test_step.py +++ b/app/test_engine/models/test_step.py @@ -29,11 +29,17 @@ class TestStep(TestObservable): __test__ = False # Needed to indicate to PyTest that this is not a "test" - def __init__(self, name: str, state: TestStateEnum = TestStateEnum.PENDING) -> None: + def __init__( + self, + name: str, + endpoint: int | None = None, + state: TestStateEnum = TestStateEnum.PENDING, + ) -> None: super().__init__() self.errors: List[str] = [] self.failures: List[str] = [] self.name = name + self.endpoint = endpoint self.__state = state self.test_step_execution: Optional[TestStepExecution] = None @@ -90,7 +96,12 @@ def mark_as_completed(self) -> None: else: self.state = TestStateEnum.PASSED - logger.info(f"Test Step Completed [{self.state.name}]: {self.name}") + endpoint_info = ( + f" for endpoint {self.endpoint}" if self.endpoint is not None else "" + ) + logger.info( + f"Test Step Completed [{self.state.name}]: {self.name}{endpoint_info}" + ) self.__print_log_separator() def __print_log_separator(self) -> None: diff --git a/app/test_engine/test_ui_observer.py b/app/test_engine/test_ui_observer.py index b49ccb13..ec5b44dd 100644 --- a/app/test_engine/test_ui_observer.py +++ b/app/test_engine/test_ui_observer.py @@ -119,6 +119,7 @@ def __onTestStepUpdate(self, observable: TestStep) -> None: "test_suite_execution_index": test_suite_execution.execution_index, "test_case_execution_index": test_case_execution.execution_index, "test_step_execution_index": test_step_execution.execution_index, + "endpoint": observable.endpoint, "state": observable.state, "errors": observable.errors, "failures": observable.failures, diff --git a/test_collections/matter/sdk_tests/support/models/matter_test_models.py b/test_collections/matter/sdk_tests/support/models/matter_test_models.py index ef604def..53b91bc2 100644 --- a/test_collections/matter/sdk_tests/support/models/matter_test_models.py +++ b/test_collections/matter/sdk_tests/support/models/matter_test_models.py @@ -33,6 +33,10 @@ class MatterTestType(Enum): class MatterTestStep(BaseModel): label: str + # Pydantic will fail parsing YAML files since they are using endpoints as variables + # instead of numbers only. + # So endpoint has to be a String as well to pass the file parsing. + endpoint: Optional[int | str] = None PICS: Optional[str] = None verification: Optional[str] = None command: Optional[str] diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index bee3e364..07b89f26 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -288,18 +288,20 @@ def __retrieve_steps(method: FunctionDefType) -> List[MatterTestStep]: step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value parsed_step_name = step_name + python_steps.append( + MatterTestStep( + label=parsed_step_name, + command=None, + arguments=None, + is_commissioning=arg_is_commissioning, + ) + ) except Exception as e: logger.warning( f"Failed parsing step name from {method.name}, Error:{str(e)}" ) parsed_step_name = "UNABLE TO PARSE TEST STEP NAME" - python_steps.append( - MatterTestStep( - label=parsed_step_name, is_commissioning=arg_is_commissioning - ) - ) - return python_steps diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index b74dd7a2..9e95be46 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -44,17 +44,17 @@ def params_dict(self) -> dict: class SDKPythonTestResultStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.START count: int class SDKPythonTestResultStop(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STOP + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STOP duration: int class SDKPythonTestResultTestStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_START filename: Optional[str] name: Optional[str] count: Optional[int] @@ -62,30 +62,31 @@ class SDKPythonTestResultTestStart(SDKPythonTestResultBase): class SDKPythonTestResultTestStop(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_STOP + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_STOP duration: Optional[int] exception: Any class SDKPythonTestResultTestSkipped(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_SKIPPED + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_SKIPPED filename: Optional[str] name: Optional[str] class SDKPythonTestResultStepSkipped(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_SKIPPED + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_SKIPPED name: Optional[str] expression: Optional[str] class SDKPythonTestResultStepStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_START name: Optional[str] + endpoint: Optional[int] class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_SUCCESS + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_SUCCESS logger: Any logs: Any duration: int @@ -93,7 +94,7 @@ class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_FAILURE + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_FAILURE logger: Any logs: Any duration: int @@ -102,15 +103,15 @@ class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): class SDKPythonTestResultStepUnknown(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_UNKNOWN + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_UNKNOWN class SDKPythonTestResultStepManual(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_MANUAL + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_MANUAL class SDKPythonTestResultShowPrompt(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.SHOW_PROMPT + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.SHOW_PROMPT msg: str placeholder: Optional[str] default_value: Optional[str] @@ -159,10 +160,12 @@ def test_skipped(self, filename: str, name: str) -> None: self.results.put(SDKPythonTestResultTestSkipped(filename=filename, name=name)) def step_skipped(self, name: str, expression: str) -> None: - self.results.put(SDKPythonTestResultStepSkipped(expression=expression)) + self.results.put( + SDKPythonTestResultStepSkipped(name=None, expression=expression) + ) - def step_start(self, name: str) -> None: - self.results.put(SDKPythonTestResultStepStart(name=name)) + def step_start(self, name: str, endpoint: Optional[int] = None) -> None: + self.results.put(SDKPythonTestResultStepStart(name=name, endpoint=endpoint)) def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: self.results.put( @@ -198,6 +201,7 @@ def show_prompt( msg: str, placeholder: Optional[str] = None, default_value: Optional[str] = None, + endpoint_id: Optional[int] = None, ) -> None: self.results.put( SDKPythonTestResultShowPrompt( diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py index ee2d3f91..c9a2bf37 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py @@ -118,8 +118,9 @@ def step_skipped(self, name: str, expression: str) -> None: else: self.current_test_step.mark_as_not_applicable(skiped_msg) - def step_start(self, name: str) -> None: + def step_start(self, name: str, endpoint: int | None = None) -> None: self.step_over() + self.current_test_step.endpoint = endpoint def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: pass diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py index 4eb6f4ea..d80ee1fb 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py @@ -112,7 +112,7 @@ def test_start( # since there is step execute outside runner context self.next_step() - def test_stop(self, duration: int) -> None: + def test_stop(self, exception: Exception, duration: int) -> None: self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: @@ -127,7 +127,7 @@ def step_skipped(self, name: str, expression: str) -> None: self.__skipped += 1 self.next_step() - def step_start(self, request: TestStep) -> None: + def step_start(self, request: TestStep, endpoint: int | None = None) -> None: if ( self.server_type == ChipServerType.CHIP_APP and @@ -140,6 +140,9 @@ def step_start(self, request: TestStep) -> None: self.__prompt_user_for_controller_action(prompt), loop=loop ) self.__index += 1 + # The TestStep request object already has the endpoint information for YAML + # tests and it will be stored in the current test step observable below. + self.current_test_step.endpoint = request.endpoint def step_success( self, logger: Any, logs: Any, duration: int, request: TestStep @@ -180,7 +183,7 @@ async def step_manual(self) -> None: self.__prompt_user_manual_step(step), OUTCOME_TIMEOUT_S ) except asyncio.TimeoutError: - self.current_test_step.append_failure("Prompt timed out.") + step.append_failure("Prompt timed out.") self.next_step() def show_prompt( @@ -188,6 +191,7 @@ def show_prompt( msg: str, placeholder: Optional[str] = None, default_value: Optional[str] = None, + endpoint_id: Optional[int] = None, ) -> None: pass diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py index 01ed130c..a56949f1 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py @@ -170,7 +170,8 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: ) return - step = TestStep(yaml_step.label) + endpoint = yaml_step.endpoint if isinstance(yaml_step.endpoint, int) else None + step = TestStep(yaml_step.label, endpoint=endpoint) if yaml_step.command == "UserPrompt": step = ManualVerificationTestStep( name=yaml_step.label, From 22cdb98c80551ed5fec4e611c287d327f651412c Mon Sep 17 00:00:00 2001 From: "Antonio Melo Jr." Date: Wed, 18 Sep 2024 21:05:39 +0000 Subject: [PATCH 2/3] Removing unnecessary blank line --- app/test_engine/models/test_case.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/test_engine/models/test_case.py b/app/test_engine/models/test_case.py index 95005290..c45a7c6d 100644 --- a/app/test_engine/models/test_case.py +++ b/app/test_engine/models/test_case.py @@ -293,7 +293,6 @@ def next_step(self) -> None: # update current step self.current_test_step_index += 1 - self.current_test_step.mark_as_executing() def __print_log_separator(self) -> None: From f52a2a644a76844b4c4df4d7069b8584e430da13 Mon Sep 17 00:00:00 2001 From: "Antonio Melo Jr." Date: Thu, 19 Sep 2024 16:39:35 +0000 Subject: [PATCH 3/3] Updating Test Ui Observable Unit Test with the expected endpoint information --- app/tests/test_engine/test_ui_observer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tests/test_engine/test_ui_observer.py b/app/tests/test_engine/test_ui_observer.py index bd2c1fb2..7303f08d 100644 --- a/app/tests/test_engine/test_ui_observer.py +++ b/app/tests/test_engine/test_ui_observer.py @@ -147,7 +147,7 @@ def __expected_test_case_dict(index: int, suite_index: int) -> Dict[str, Any]: def __expected_test_step_dict( - index: int, case_index: int, suite_index: int + index: int, case_index: int, suite_index: int, endpoint: int | None = None ) -> Dict[str, Any]: return { MessageKeysEnum.TYPE: MessageTypeEnum.TEST_UPDATE, @@ -157,6 +157,7 @@ def __expected_test_step_dict( "test_suite_execution_index": suite_index, "test_case_execution_index": case_index, "test_step_execution_index": index, + "endpoint": endpoint, "state": TestStateEnum.EXECUTING, "errors": [], "failures": [],