diff --git a/app/test_engine/models/manual_test_case.py b/app/test_engine/models/manual_test_case.py index 91a4b467..10c4f57e 100644 --- a/app/test_engine/models/manual_test_case.py +++ b/app/test_engine/models/manual_test_case.py @@ -193,7 +193,7 @@ async def execute(self) -> None: if isinstance(step, ManualVerificationTestStep): await step.prompt_verification_step() else: - logger.error(f"Unsupported test test {step.__class__}") + logger.error(f"Unsupported test step {step.__class__}") self.next_step() if isinstance(self.current_test_step, ManualLogUploadStep): diff --git a/app/test_engine/models/test_case.py b/app/test_engine/models/test_case.py index 208885ae..ef536df9 100644 --- a/app/test_engine/models/test_case.py +++ b/app/test_engine/models/test_case.py @@ -292,6 +292,7 @@ def next_step(self) -> None: # update current step self.current_test_step_index += 1 + self.current_test_step.mark_as_executing() def __print_log_separator(self) -> None: diff --git a/app/test_engine/models/test_step.py b/app/test_engine/models/test_step.py index d666d4c2..71847295 100644 --- a/app/test_engine/models/test_step.py +++ b/app/test_engine/models/test_step.py @@ -29,11 +29,17 @@ class TestStep(TestObservable): __test__ = False # Needed to indicate to PyTest that this is not a "test" - def __init__(self, name: str, state: TestStateEnum = TestStateEnum.PENDING) -> None: + def __init__( + self, + name: str, + endpoint: int | None = None, + state: TestStateEnum = TestStateEnum.PENDING, + ) -> None: super().__init__() self.errors: List[str] = [] self.failures: List[str] = [] self.name = name + self.endpoint = endpoint self.__state = state self.test_step_execution: Optional[TestStepExecution] = None @@ -90,7 +96,12 @@ def mark_as_completed(self) -> None: else: self.state = TestStateEnum.PASSED - logger.info(f"Test Step Completed [{self.state.name}]: {self.name}") + endpoint_info = ( + f" for endpoint {self.endpoint}" if self.endpoint is not None else "" + ) + logger.info( + f"Test Step Completed [{self.state.name}]: {self.name}{endpoint_info}" + ) self.__print_log_separator() def __print_log_separator(self) -> None: diff --git a/app/test_engine/test_ui_observer.py b/app/test_engine/test_ui_observer.py index 14f464dc..f2e41214 100644 --- a/app/test_engine/test_ui_observer.py +++ b/app/test_engine/test_ui_observer.py @@ -118,6 +118,7 @@ def __onTestStepUpdate(self, observable: TestStep) -> None: "test_suite_execution_index": test_suite_execution.execution_index, "test_case_execution_index": test_case_execution.execution_index, "test_step_execution_index": test_step_execution.execution_index, + "endpoint": observable.endpoint, "state": observable.state, "errors": observable.errors, "failures": observable.failures, diff --git a/test_collections/matter/sdk_tests/support/models/matter_test_models.py b/test_collections/matter/sdk_tests/support/models/matter_test_models.py index ef604def..53b91bc2 100644 --- a/test_collections/matter/sdk_tests/support/models/matter_test_models.py +++ b/test_collections/matter/sdk_tests/support/models/matter_test_models.py @@ -33,6 +33,10 @@ class MatterTestType(Enum): class MatterTestStep(BaseModel): label: str + # Pydantic will fail parsing YAML files since they are using endpoints as variables + # instead of numbers only. + # So endpoint has to be a String as well to pass the file parsing. + endpoint: Optional[int | str] = None PICS: Optional[str] = None verification: Optional[str] = None command: Optional[str] diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index b9f1fbf6..57cdeb8d 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -236,16 +236,18 @@ def __retrieve_steps(method: FunctionDefType) -> List[MatterTestStep]: step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value parsed_step_name = step_name + python_steps.append( + MatterTestStep( + label=parsed_step_name, + command=None, + arguments=None, + is_commissioning=arg_is_commissioning, + ) + ) except Exception as e: logger.error(f"Error while parsing step from {method.name}, Error:{str(e)}") parsed_step_name = "UNABLE TO PARSE TEST STEP NAME" - python_steps.append( - MatterTestStep( - label=parsed_step_name, is_commissioning=arg_is_commissioning - ) - ) - return python_steps diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index b74dd7a2..9e95be46 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -44,17 +44,17 @@ def params_dict(self) -> dict: class SDKPythonTestResultStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.START count: int class SDKPythonTestResultStop(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STOP + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STOP duration: int class SDKPythonTestResultTestStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_START filename: Optional[str] name: Optional[str] count: Optional[int] @@ -62,30 +62,31 @@ class SDKPythonTestResultTestStart(SDKPythonTestResultBase): class SDKPythonTestResultTestStop(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_STOP + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_STOP duration: Optional[int] exception: Any class SDKPythonTestResultTestSkipped(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.TEST_SKIPPED + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.TEST_SKIPPED filename: Optional[str] name: Optional[str] class SDKPythonTestResultStepSkipped(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_SKIPPED + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_SKIPPED name: Optional[str] expression: Optional[str] class SDKPythonTestResultStepStart(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_START + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_START name: Optional[str] + endpoint: Optional[int] class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_SUCCESS + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_SUCCESS logger: Any logs: Any duration: int @@ -93,7 +94,7 @@ class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_FAILURE + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_FAILURE logger: Any logs: Any duration: int @@ -102,15 +103,15 @@ class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): class SDKPythonTestResultStepUnknown(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_UNKNOWN + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_UNKNOWN class SDKPythonTestResultStepManual(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.STEP_MANUAL + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.STEP_MANUAL class SDKPythonTestResultShowPrompt(SDKPythonTestResultBase): - type = SDKPythonTestResultEnum.SHOW_PROMPT + type: SDKPythonTestResultEnum = SDKPythonTestResultEnum.SHOW_PROMPT msg: str placeholder: Optional[str] default_value: Optional[str] @@ -159,10 +160,12 @@ def test_skipped(self, filename: str, name: str) -> None: self.results.put(SDKPythonTestResultTestSkipped(filename=filename, name=name)) def step_skipped(self, name: str, expression: str) -> None: - self.results.put(SDKPythonTestResultStepSkipped(expression=expression)) + self.results.put( + SDKPythonTestResultStepSkipped(name=None, expression=expression) + ) - def step_start(self, name: str) -> None: - self.results.put(SDKPythonTestResultStepStart(name=name)) + def step_start(self, name: str, endpoint: Optional[int] = None) -> None: + self.results.put(SDKPythonTestResultStepStart(name=name, endpoint=endpoint)) def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: self.results.put( @@ -198,6 +201,7 @@ def show_prompt( msg: str, placeholder: Optional[str] = None, default_value: Optional[str] = None, + endpoint_id: Optional[int] = None, ) -> None: self.results.put( SDKPythonTestResultShowPrompt( diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py index 0863c074..c7098122 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py @@ -117,8 +117,9 @@ def test_skipped(self, filename: str, name: str) -> None: def step_skipped(self, name: str, expression: str) -> None: self.current_test_step.mark_as_not_applicable("Test step skipped") - def step_start(self, name: str) -> None: + def step_start(self, name: str, endpoint: int | None = None) -> None: self.step_over() + self.current_test_step.endpoint = endpoint def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: pass diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py index 4eb6f4ea..d80ee1fb 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py @@ -112,7 +112,7 @@ def test_start( # since there is step execute outside runner context self.next_step() - def test_stop(self, duration: int) -> None: + def test_stop(self, exception: Exception, duration: int) -> None: self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: @@ -127,7 +127,7 @@ def step_skipped(self, name: str, expression: str) -> None: self.__skipped += 1 self.next_step() - def step_start(self, request: TestStep) -> None: + def step_start(self, request: TestStep, endpoint: int | None = None) -> None: if ( self.server_type == ChipServerType.CHIP_APP and @@ -140,6 +140,9 @@ def step_start(self, request: TestStep) -> None: self.__prompt_user_for_controller_action(prompt), loop=loop ) self.__index += 1 + # The TestStep request object already has the endpoint information for YAML + # tests and it will be stored in the current test step observable below. + self.current_test_step.endpoint = request.endpoint def step_success( self, logger: Any, logs: Any, duration: int, request: TestStep @@ -180,7 +183,7 @@ async def step_manual(self) -> None: self.__prompt_user_manual_step(step), OUTCOME_TIMEOUT_S ) except asyncio.TimeoutError: - self.current_test_step.append_failure("Prompt timed out.") + step.append_failure("Prompt timed out.") self.next_step() def show_prompt( @@ -188,6 +191,7 @@ def show_prompt( msg: str, placeholder: Optional[str] = None, default_value: Optional[str] = None, + endpoint_id: Optional[int] = None, ) -> None: pass diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py index 14496155..e309901c 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py @@ -168,7 +168,8 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: ) return - step = TestStep(yaml_step.label) + endpoint = yaml_step.endpoint if isinstance(yaml_step.endpoint, int) else None + step = TestStep(yaml_step.label, endpoint=endpoint) if yaml_step.command == "UserPrompt": step = ManualVerificationTestStep( name=yaml_step.label,