From 06a725e8f74ef35d368f94f127ca80e009b27a15 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <116586593+rquidute@users.noreply.github.com> Date: Thu, 7 Dec 2023 18:26:50 -0300 Subject: [PATCH] Python test use project config to retrieve test arguments (#38) * Get python test parameters from project config * Fix test name parameter --- .../python_testing/models/test_case.py | 67 +++++- .../python_tests/test_python_test_case.py | 197 ++++++++++++++++++ 2 files changed, 257 insertions(+), 7 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index e0008d7..81e35b2 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -19,6 +19,7 @@ from typing import Any, Type, TypeVar from app.models import TestCaseExecution +from app.schemas.test_environment_config import DutPairingModeEnum from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestCase, TestStep from test_collections.sdk_tests.support.chip_tool.chip_tool import ( @@ -145,6 +146,58 @@ async def setup(self) -> None: async def cleanup(self) -> None: logger.info("Test Cleanup") + def __generate_command_arguments(self) -> list: + # All valid arguments for python test + valid_args = [ + "ble_interface_id", + "commissioning_method", + "controller_node_id", + "discriminator", + "endpoint", + "logs_path", + "PICS", + "paa_trust_store_path", + "timeout", + "trace_to", + "int_arg", + "float_arg", + "string_arg", + "json_arg", + "hex_arg", + "bool_arg", + "storage_path", + "passcode", + "dut-node-id", + ] + + dut_config = self.project.config.dut_config + test_parameters = self.project.config.test_parameters + + pairing_mode = ( + "on-network" + if dut_config.pairing_mode == DutPairingModeEnum.ON_NETWORK.value + else dut_config.pairing_mode + ) + + arguments = [] + # Retrieve arguments from dut_config + arguments.append(f"--discriminator {dut_config.discriminator}") + arguments.append(f"--passcode {dut_config.setup_code}") + arguments.append(f"--commissioning-method {pairing_mode}") + + # Retrieve arguments from test_parameters + if test_parameters: + for name, value in test_parameters.items(): + if name in valid_args: + if str(value) != "": + arguments.append(f"--{name.replace('_','-')} {str(value)}") + else: + arguments.append(f"--{name.replace('_','-')} " "") + else: + logger.warning(f"Argument {name} is not valid") + + return arguments + async def execute(self) -> None: try: logger.info("Running Python Test: " + self.python_test.name) @@ -155,15 +208,15 @@ async def execute(self) -> None: test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS - command = ( - f"{runner_class} {self.python_test.name}" - " --commissioning-method on-network --discriminator 3840 --passcode" - " 20202021 --storage-path /root/admin_storage.json" - " --paa-trust-store-path /paa-root-certs" - ) + command = [f"{runner_class} {self.python_test.name}"] + + # Generate the command argument by getting the test_parameters from + # project configuration + command_arguments = self.__generate_command_arguments() + command.extend(command_arguments) if self.chip_tool.pics_file_created: - command += f" --PICS {PICS_FILE_PATH}" + command.append(f" --PICS {PICS_FILE_PATH}") # TODO Ignoring stream from docker execution self.chip_tool.send_command( diff --git a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py index 6fb8803..0fbb15f 100644 --- a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py +++ b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py @@ -18,10 +18,14 @@ from pathlib import Path from typing import Any, Optional, Type from unittest import mock +from unittest.mock import PropertyMock import pytest +from app.default_environment_config import default_environment_config +from app.models import Project from app.models.test_case_execution import TestCaseExecution +from app.schemas.test_environment_config import DutConfig, DutPairingModeEnum from app.test_engine.logger import test_engine_logger from test_collections.sdk_tests.support.chip_tool.test_case import TestError from test_collections.sdk_tests.support.models.matter_test_models import ( @@ -202,3 +206,196 @@ async def test_setup_super_error_handling() -> None: test.python_test_version = "some version" # Assert this doesn't raise an exception await test.setup() + + +@pytest.mark.asyncio +async def test_generate_command_arguments_on_network() -> None: + # Mock config + mock_config = default_environment_config.copy(deep=True) + + mock_config.test_parameters = { + "paa_trust_store_path": "/paa-root-certs", + "storage_path": "/root/admin_storage.json", + } + + mock_dut_config = DutConfig( + discriminator="123", + setup_code="1234", + pairing_mode=DutPairingModeEnum.ON_NETWORK, + ) + + mock_config.dut_config = mock_dut_config + + mock_project = Project() + mock_project.config = mock_config + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ), mock.patch( + "app.test_engine.models.test_case.TestCase.project", + new_callable=PropertyMock, + return_value=mock_project, + ): + test = PythonTestCase(TestCaseExecution()) + arguments = test._PythonTestCase__generate_command_arguments() # type: ignore + + assert [ + "--discriminator 123", + "--passcode 1234", + "--commissioning-method on-network", + "--paa-trust-store-path /paa-root-certs", + "--storage-path /root/admin_storage.json", + ] == arguments + + +@pytest.mark.asyncio +async def test_generate_command_arguments_ble_wifi() -> None: + # Mock config + mock_config = default_environment_config.copy(deep=True) + + mock_config.test_parameters = { + "paa_trust_store_path": "/paa-root-certs", + "storage_path": "/root/admin_storage.json", + } + + mock_dut_config = DutConfig( + discriminator="147", + setup_code="357", + pairing_mode=DutPairingModeEnum.BLE_WIFI, + ) + + mock_config.dut_config = mock_dut_config + + mock_project = Project() + mock_project.config = mock_config + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ), mock.patch( + "app.test_engine.models.test_case.TestCase.project", + new_callable=PropertyMock, + return_value=mock_project, + ): + test = PythonTestCase(TestCaseExecution()) + arguments = test._PythonTestCase__generate_command_arguments() # type: ignore + + assert [ + "--discriminator 147", + "--passcode 357", + "--commissioning-method ble-wifi", + "--paa-trust-store-path /paa-root-certs", + "--storage-path /root/admin_storage.json", + ] == arguments + + +@pytest.mark.asyncio +async def test_generate_command_arguments_ble_thread() -> None: + # Mock config + mock_config = default_environment_config.copy(deep=True) + + mock_config.test_parameters = { + "paa_trust_store_path": "/paa-root-certs", + "storage_path": "/root/admin_storage.json", + } + + mock_dut_config = DutConfig( + discriminator="456", + setup_code="8765", + pairing_mode=DutPairingModeEnum.BLE_THREAD, + ) + + mock_config.dut_config = mock_dut_config + + mock_project = Project() + mock_project.config = mock_config + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ), mock.patch( + "app.test_engine.models.test_case.TestCase.project", + new_callable=PropertyMock, + return_value=mock_project, + ): + test = PythonTestCase(TestCaseExecution()) + arguments = test._PythonTestCase__generate_command_arguments() # type: ignore + + assert [ + "--discriminator 456", + "--passcode 8765", + "--commissioning-method ble-thread", + "--paa-trust-store-path /paa-root-certs", + "--storage-path /root/admin_storage.json", + ] == arguments + + +@pytest.mark.asyncio +async def test_generate_command_arguments_no_test_parameter_informed() -> None: + # Mock config + mock_config = default_environment_config.copy(deep=True) + + mock_config.test_parameters = None + + mock_dut_config = DutConfig( + discriminator="456", + setup_code="8765", + pairing_mode=DutPairingModeEnum.BLE_THREAD, + ) + + mock_config.dut_config = mock_dut_config + + mock_project = Project() + mock_project.config = mock_config + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ), mock.patch( + "app.test_engine.models.test_case.TestCase.project", + new_callable=PropertyMock, + return_value=mock_project, + ): + test = PythonTestCase(TestCaseExecution()) + arguments = test._PythonTestCase__generate_command_arguments() # type: ignore + + assert [ + "--discriminator 456", + "--passcode 8765", + "--commissioning-method ble-thread", + ] == arguments + + +@pytest.mark.asyncio +async def test_generate_command_arguments_invalid_arg() -> None: + # Mock config + mock_config = default_environment_config.copy(deep=True) + + mock_config.test_parameters = { + "my_arg1": "/paa-root-certs", + "my_arg2": "/root/admin_storage.json", + } + + mock_dut_config = DutConfig( + discriminator="456", + setup_code="8765", + pairing_mode=DutPairingModeEnum.BLE_THREAD, + ) + + mock_config.dut_config = mock_dut_config + + mock_project = Project() + mock_project.config = mock_config + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ), mock.patch( + "app.test_engine.models.test_case.TestCase.project", + new_callable=PropertyMock, + return_value=mock_project, + ): + test = PythonTestCase(TestCaseExecution()) + arguments = test._PythonTestCase__generate_command_arguments() # type: ignore + + assert [ + "--discriminator 456", + "--passcode 8765", + "--commissioning-method ble-thread", + ] == arguments