diff --git a/app/core/config.py b/app/core/config.py index 0d9e91a4..85e08590 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -109,7 +109,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: # Test Engine Config CHIP_TOOL_TRACE: bool = True - CHIP_TOOL_CONTAINER_NAME: str = "th-chip-tool" + SDK_CONTAINER_NAME: str = "th-sdk" # SDK Docker Image SDK_DOCKER_IMAGE: str = "connectedhomeip/chip-cert-bins" diff --git a/test_collections/python_tests b/test_collections/python_tests index f2e5ab57..5367de32 160000 --- a/test_collections/python_tests +++ b/test_collections/python_tests @@ -1 +1 @@ -Subproject commit f2e5ab5706ae4ab6e7c8c058081f1a4e93a50f1b +Subproject commit 5367de326ceefc4cf1b4f60a34ee4610e11eb336 diff --git a/test_collections/sdk_tests/support/chip/chip_tool.py b/test_collections/sdk_tests/support/chip/chip_tool.py index 26b0dd1e..82b4d325 100644 --- a/test_collections/sdk_tests/support/chip/chip_tool.py +++ b/test_collections/sdk_tests/support/chip/chip_tool.py @@ -24,7 +24,6 @@ from typing import Any, Generator, Optional, Union, cast import loguru -from docker.models.containers import Container from matter_chip_tool_adapter import adapter as ChipToolAdapter from matter_chip_tool_adapter.decoder import MatterLog from matter_placeholder_adapter import adapter as ChipAppAdapter @@ -38,7 +37,6 @@ from matter_yamltests.runner import TestRunnerConfig, TestRunnerOptions from matter_yamltests.websocket_runner import WebSocketRunner, WebSocketRunnerConfig -from app.container_manager import container_manager from app.container_manager.backend_container import backend_container from app.core.config import settings from app.schemas.pics import PICS, PICSError @@ -46,8 +44,12 @@ from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL from app.test_engine.logger import test_engine_logger as logger from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH - -from .exec_run_in_container import ExecResultExtended, exec_run_in_container +from test_collections.sdk_tests.support.pics import PICS_FILE_PATH, set_pics_command +from test_collections.sdk_tests.support.sdk_container import ( + DOCKER_LOGS_PATH, + DOCKER_PAA_CERTS_PATH, + SDKContainer, +) # Chip Tool Parameters CHIP_TOOL_EXE = "./chip-tool" @@ -76,49 +78,11 @@ CHIP_APP_DEFAULT_PORT = 5540 CHIP_APP_TEST_CMD_ARG = "--command" -# PICS parameters -SHELL_PATH = "/bin/sh" -SHELL_OPTION = "-c" -PICS_FILE_PATH = "/var/tmp/pics" -ECHO_COMMAND = "echo" -# List of default PICS which needs to set specifically in TH are added here. -# These PICS are applicable for CI / Chip tool testing purposes only. -# These PICS are unknown / not visible to external users. -DEFAULT_PICS = ["PICS_SDK_CI_ONLY=0", "PICS_SKIP_SAMPLE_APP=1", "PICS_USER_PROMPT=1"] - -# Trace mount -LOCAL_LOGS_PATH = Path("/var/tmp") -DOCKER_LOGS_PATH = "/logs" - -# PAA Cert mount -LOCAL_PAA_CERTS_PATH = Path("/var/paa-root-certs") -DOCKER_PAA_CERTS_PATH = "/paa-root-certs" - -# Credentials Development mount -LOCAL_CREDENTIALS_DEVELOPMENT_PATH = Path("/var/credentials/development") -DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development" - # Websocket runner YAML_TESTS_PATH_BASE = SDK_CHECKOUT_PATH / Path("yaml_tests/") YAML_TESTS_PATH = YAML_TESTS_PATH_BASE / Path("yaml/sdk") XML_SPEC_DEFINITION_PATH = SDK_CHECKOUT_PATH / Path("sdk_runner/specifications/chip/") -# Python Testing Folder -LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections" -LOCAL_PYTHON_TESTING_PATH = Path( - LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing" -) -DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" - -# RPC Client Running on SDK Container -LOCAL_RPC_PYTHON_TESTING_PATH = Path( - LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/support/python_testing/models/rpc_client/" - "test_harness_client.py" -) -DOCKER_RPC_PYTHON_TESTING_PATH = ( - "/root/python_testing/scripts/sdk/test_harness_client.py" -) - # Docker Network DOCKER_NETWORK_SETTINGS_KEY = "NetworkSettings" @@ -154,43 +118,7 @@ class ChipTool(metaclass=Singleton): calling start_device and when done cleanup by calling destroy_device """ - container_name = settings.CHIP_TOOL_CONTAINER_NAME - image_tag = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}" - run_parameters = { - "privileged": True, - "detach": True, - "network": "host", - "name": container_name, - "command": "tail -f /dev/null", # while true - "volumes": { - "/var/run/dbus/system_bus_socket": { - "bind": "/var/run/dbus/system_bus_socket", - "mode": "rw", - }, - LOCAL_LOGS_PATH: { - "bind": DOCKER_LOGS_PATH, - "mode": "rw", - }, - LOCAL_PAA_CERTS_PATH: { - "bind": DOCKER_PAA_CERTS_PATH, - "mode": "ro", - }, - LOCAL_CREDENTIALS_DEVELOPMENT_PATH: { - "bind": DOCKER_CREDENTIALS_DEVELOPMENT_PATH, - "mode": "ro", - }, - LOCAL_PYTHON_TESTING_PATH: { - "bind": DOCKER_PYTHON_TESTING_PATH, - "mode": "rw", - }, - LOCAL_RPC_PYTHON_TESTING_PATH: { - "bind": DOCKER_RPC_PYTHON_TESTING_PATH, - "mode": "rw", - }, - }, - } - - __node_id: Optional[int] # will be reset every time the container is started + __node_id: Optional[int] = None __pics_file_created: bool # Flag that is set if PICS needs to be passed to chiptool def __init__( @@ -203,14 +131,11 @@ def __init__( logger (Logger, optional): Optional logger injection. Defaults to standard self.logger. """ - self.__chip_tool_container: Optional[Container] = None - # Last execution id is updated every time a command is executed - # This is used to retrieve the exit code of a command when streaming the logs. - self.__last_exec_id: Optional[str] = None - self.__pics_file_created = False self.logger = logger - self.__chip_tool_server_id: Optional[str] = None + self.sdk_container: SDKContainer = SDKContainer(logger) + self.__pics_file_created = False + self.__chip_server_id: Optional[str] = None self.__server_started = False self.__server_logs: Union[Generator, bytes, tuple] self.__use_paa_certs = False @@ -243,42 +168,29 @@ def __reset_node_id(self) -> int: """Resets node_id to a random uint64.""" max_uint_64 = (1 << 64) - 1 self.__node_id = randrange(max_uint_64) + self.logger.info(f"New Node Id generated: {hex(self.__node_id)}") return self.__node_id - def __destroy_existing_container(self) -> None: - """This will kill and remove any existing container using the same name.""" - existing_container = container_manager.get_container(self.container_name) - if existing_container is not None: - logger.info( - f'Existing container named "{self.container_name}" found. Destroying.' - ) - container_manager.destroy(existing_container) - - def is_running(self) -> bool: - if self.__chip_tool_container is None: - return False - else: - return container_manager.is_running(self.__chip_tool_container) - async def __wait_for_server_start(self, log_generator: Generator) -> bool: for chunk in log_generator: decoded_log = chunk.decode().strip() log_lines = decoded_log.splitlines() for line in log_lines: if "LWS_CALLBACK_PROTOCOL_INIT" in line: - logger.log(CHIPTOOL_LEVEL, line) + self.logger.log(CHIPTOOL_LEVEL, line) return True - logger.log(CHIPTOOL_LEVEL, line) + self.logger.log(CHIPTOOL_LEVEL, line) else: return False async def start_chip_server( self, test_type: ChipTestType, use_paa_certs: bool = False ) -> Generator: - # Start ChipTool Interactive Server + # Start chip interactive server self.__use_paa_certs = use_paa_certs self.__test_type = test_type - self.logger.info("Starting Chip Tool Server") + + self.logger.info("Starting chip server") if self.__server_started: return cast(Generator, self.__server_logs) @@ -299,13 +211,15 @@ async def start_chip_server( paa_cert_params = f"{CHIP_TOOL_ARG_PAA_CERTS_PATH} {DOCKER_PAA_CERTS_PATH}" command.append(paa_cert_params) - self.__server_logs = self.send_command( + exec_result = self.sdk_container.send_command( command, prefix=prefix, is_stream=True, is_socket=False, - ).output - self.__chip_tool_server_id = self.__last_exec_id + ) + self.__server_logs = exec_result.output + self.__chip_server_id = exec_result.exec_id + wait_result = await self.__wait_for_server_start( cast(Generator, self.__server_logs) ) @@ -315,42 +229,27 @@ async def start_chip_server( return cast(Generator, self.__server_logs) def __wait_for_server_exit(self) -> Optional[int]: - exit_code = None - if self.__chip_tool_container is None: + if self.__chip_server_id is None: self.logger.info( - "No chip-tool container, cannot return last command exit code." + "Server execution id not found, cannot wait for server exit." ) return None - if self.__chip_tool_server_id is None: - self.logger.info( - "Last execution id not found, cannot return last command exit code." - ) - return None - - while True: - exec_data = self.__chip_tool_container.client.api.exec_inspect( - self.__chip_tool_server_id - ) - if exec_data is None: - self.logger.error( - "Docker didn't return any execution metadata," - " cannot return last command exit code." - ) - return None - exit_code = exec_data.get("ExitCode") - if exit_code is not None: - break + exit_code = self.sdk_container.exec_exit_code(self.__chip_server_id) + while not exit_code: + exit_code = self.sdk_container.exec_exit_code(self.__chip_server_id) return exit_code - async def stop_chip_tool_server(self) -> None: - if self.__server_started: - await self.start_runner() - await self.__test_harness_runner._client.send("quit()") - self.__wait_for_server_exit() - await self.stop_runner() - self.__server_started = False + async def stop_chip_server(self) -> None: + if not self.__server_started: + return + + await self.start_runner() + await self.__test_harness_runner._client.send("quit()") + self.__wait_for_server_exit() + await self.stop_runner() + self.__server_started = False def __get_gateway_ip(self) -> str: """ @@ -370,59 +269,22 @@ def __get_gateway_ip(self) -> str: .get(DOCKER_GATEWAY_KEY, "") ) - async def start_container(self) -> None: - """ - Creates the chip-tool container without any server running - (ChipTool or ChipApp). - """ - if self.is_running(): - self.logger.info( - "chip-tool container already running, no need to start a new container" - ) - return - - # Ensure there's no existing container running using the same name. - self.__destroy_existing_container() - # Async return when the container is running - self.__chip_tool_container = await container_manager.create_container( - self.image_tag, self.run_parameters - ) + async def start_server( + self, test_type: ChipTestType, use_paa_certs: bool = False + ) -> None: # Reset any previous states - self.__last_exec_id = None self.__pics_file_created = False # Generate new random node id for the DUT self.__reset_node_id() - self.logger.info(f"New Node Id generated: {hex(self.node_id)}") - self.logger.info( - f""" - chip-tool started: {self.container_name} - with configuration: {self.run_parameters} - """ - ) # Server started is false after spinning up a new container. self.__server_started = False - async def start_server( - self, test_type: ChipTestType, use_paa_certs: bool = False - ) -> None: - """Creates the chip-tool container. - - Returns only when the container is created and all chip-tool services start. - """ - await self.start_container() - web_socket_config = WebSocketRunnerConfig() web_socket_config.server_address = self.__get_gateway_ip() self.__test_harness_runner = WebSocketRunner(config=web_socket_config) self.__chip_tool_log = await self.start_chip_server(test_type, use_paa_certs) - async def destroy_device(self) -> None: - """Destroy the device container.""" - if self.__chip_tool_container is not None: - container_manager.destroy(self.__chip_tool_container) - self.__chip_tool_container = None - async def start_runner(self) -> None: if not self.__test_harness_runner.is_connected: await self.__test_harness_runner.start() @@ -430,85 +292,6 @@ async def start_runner(self) -> None: async def stop_runner(self) -> None: await self.__test_harness_runner.stop() - def send_command( - self, - command: Union[str, list], - prefix: str, - is_stream: bool = False, - is_socket: bool = False, - ) -> ExecResultExtended: - if self.__chip_tool_container is None: - raise ChipToolNotRunning() - - full_cmd = [prefix] - if isinstance(command, list): - full_cmd += command - else: - full_cmd.append(str(command)) - - self.logger.info("Sending command to chip-tool: " + " ".join(full_cmd)) - - result = exec_run_in_container( - self.__chip_tool_container, - " ".join(full_cmd), - socket=is_socket, - stream=is_stream, - stdin=True, - ) - - # When streaming logs, the exit code is not directly available. - # By storing the execution id, the exit code can be fetched from docker later. - self.__last_exec_id = result.exec_id - - return result - - def last_command_exit_code(self) -> Optional[int]: - """Get the exit code of the last run command. - - When streaming logs from chip-tool in docker, the exit code is not directly - available. Using the id of the execution, the exit code is fetched from docker. - - Returns: - Optional[int]: exit code of the last run command - """ - if self.__last_exec_id is None: - self.logger.info( - "Last execution id not found, cannot return last command exit code." - ) - return None - - if self.__chip_tool_container is None: - self.logger.info( - "No chip-tool container, cannot return last command exit code." - ) - return None - - exec_data = self.__chip_tool_container.client.api.exec_inspect( - self.__last_exec_id - ) - if exec_data is None: - self.logger.error( - "Docker didn't return any execution metadata," - " cannot return last command exit code." - ) - return None - - exit_code = exec_data.get("ExitCode") - return exit_code - - def exec_exit_code(self, exec_id: str) -> Optional[int]: - if self.__chip_tool_container is None: - self.logger.info("No SDK container, cannot get execution exit code") - return None - - exec_data = self.__chip_tool_container.client.api.exec_inspect(exec_id) - - if exec_data is None: - self.logger.error("Docker didn't return any execution metadata") - return None - - return exec_data.get("ExitCode") - async def send_websocket_command(self, cmd: str) -> Union[str, bytes, bytearray]: await self.start_runner() response = await self.__test_harness_runner.execute(cmd) @@ -698,14 +481,11 @@ def __trace_file_params(self, topic: str) -> str: path = Path(DOCKER_LOGS_PATH) / filename return f'--trace_file "{path}" --trace_decode 1' - def set_pics(self, pics: PICS, in_container: bool) -> None: + def set_pics(self, pics: PICS) -> None: """Sends command to create pics file. Args: pics (PICS): PICS that contains all the pics codes - in_container (bool): Whether the file should be created in the SDK container - or not. YAML tests run directly in the backend and - python tests run in the SDK container. Raises: PICSError: If creating PICS file inside the container fails. @@ -714,21 +494,13 @@ def set_pics(self, pics: PICS, in_container: bool) -> None: # These PICS are applicable for CI / Chip tool testing purposes only. # These PICS are unknown / not visible to external users. - pics_codes = self.__pics_file_content(pics) + "\n".join(DEFAULT_PICS) - - prefix = f"{SHELL_PATH} {SHELL_OPTION}" - cmd = f"\"{ECHO_COMMAND} '{pics_codes}' > {PICS_FILE_PATH}\"" + prefix, cmd = set_pics_command(pics) - if in_container: - exec_result = self.send_command(cmd, prefix=prefix) - success = exec_result.exit_code == 0 - else: - full_cmd = f"{prefix} {cmd}" - self.logger.info(f"Sending command: {full_cmd}") - result = subprocess.run(full_cmd, shell=True) - success = result.returncode == 0 + full_cmd = f"{prefix} {cmd}" + self.logger.info(f"Sending command: {full_cmd}") + result = subprocess.run(full_cmd, shell=True) - if not success: + if result.returncode != 0: raise PICSError("Creating PICS file failed") self.__pics_file_created = True @@ -736,74 +508,8 @@ def set_pics(self, pics: PICS, in_container: bool) -> None: def reset_pics_state(self) -> None: self.__pics_file_created = False - def __pics_file_content(self, pics: PICS) -> str: - """Generates PICS file content in the below format: - PICS_CODE1=1 - PICS_CODE2=1 - PICS_CODE3=0 - ..... - - Args: - pics (PICS): PICS that contains all the pics codes - - Returns: - str: Returns a string in this format PICS_CODE1=1\nPICS_CODE1=2\n" - """ - pics_str: str = "" - - for cluster in pics.clusters.values(): - for pi in cluster.items.values(): - if pi.enabled: - pics_str += pi.number + "=1" + "\n" - else: - pics_str += pi.number + "=0" + "\n" - - return pics_str - async def restart_server(self) -> None: - await self.stop_chip_tool_server() + await self.stop_chip_server() self.__chip_tool_log = await self.start_chip_server( self.__test_type, self.__use_paa_certs ) - - # TODO(#490): Need to be refactored to support real PIXIT format - def __test_parameters_arguments( - self, test_parameters: Optional[dict[str, Any]] - ) -> list: - """Generate cli arguments for chip-tool based on test_parameters. - - Note: - - Currently `nodeID` is managed by this class, and should not be overridden - by test_parameters. - - chip-tool allows users to configure `cluster` via cli, but this should not - be used. - - This method is ignoring `nodeID` and `cluster` from input argument - `test_parameters`. - - chip-tool is expecting test_parameter arguments as - `-- value` - - Args: - test_parameters (Optional[dict[str, Any]]): dictionary of test parameters - - Returns: - list: list of chip-tool arguments. - """ - if test_parameters is None: - return [] - - arguments = [] - for name, value in test_parameters.items(): - # skip nodeId, as it is passed separately - # skip cluster, as we don't allow to override this - if name in ["nodeId", "cluster"]: - continue - - if str(value) != "": - # TODO: does this work for all formats, string, number etc? - arguments.append(f"--{name} {str(value)}") - else: - arguments.append(f'--{name} ""') - - return arguments diff --git a/test_collections/sdk_tests/support/chip/exec_run_in_container.py b/test_collections/sdk_tests/support/exec_run_in_container.py similarity index 100% rename from test_collections/sdk_tests/support/chip/exec_run_in_container.py rename to test_collections/sdk_tests/support/exec_run_in_container.py diff --git a/test_collections/sdk_tests/support/pics.py b/test_collections/sdk_tests/support/pics.py new file mode 100644 index 00000000..31e3f3b0 --- /dev/null +++ b/test_collections/sdk_tests/support/pics.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from app.schemas.pics import PICS + +# PICS parameters +SHELL_PATH = "/bin/sh" +SHELL_OPTION = "-c" +PICS_FILE_PATH = "/var/tmp/pics" +ECHO_COMMAND = "echo" +# List of default PICS which needs to set specifically in TH are added here. +# These PICS are applicable for CI / Chip tool testing purposes only. +# These PICS are unknown / not visible to external users. +DEFAULT_PICS = ["PICS_SDK_CI_ONLY=0", "PICS_SKIP_SAMPLE_APP=1", "PICS_USER_PROMPT=1"] + + +def pics_file_content(pics: PICS) -> str: + """Generates PICS file content in the below format: + PICS_CODE1=1 + PICS_CODE2=1 + PICS_CODE3=0 + ..... + + Args: + pics (PICS): PICS that contains all the pics codes + + Returns: + str: Returns a string in this format PICS_CODE1=1\nPICS_CODE1=2\n" + """ + pics_str: str = "" + + for cluster in pics.clusters.values(): + for pi in cluster.items.values(): + if pi.enabled: + pics_str += pi.number + "=1" + "\n" + else: + pics_str += pi.number + "=0" + "\n" + + return pics_str + + +def set_pics_command(pics: PICS) -> tuple[str, str]: + pics_codes = pics_file_content(pics) + "\n".join(DEFAULT_PICS) + + prefix = f"{SHELL_PATH} {SHELL_OPTION}" + cmd = f"\"{ECHO_COMMAND} '{pics_codes}' > {PICS_FILE_PATH}\"" + + return (prefix, cmd) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 41800089..29e9dc54 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -25,7 +25,8 @@ from app.test_engine.models.test_case import CUSTOM_TEST_IDENTIFIER from app.user_prompt_support.prompt_request import OptionsSelectPromptRequest from app.user_prompt_support.user_prompt_support import UserPromptSupport -from test_collections.sdk_tests.support.chip.chip_tool import PICS_FILE_PATH, ChipTool +from test_collections.sdk_tests.support.chip.chip_tool import PICS_FILE_PATH +from test_collections.sdk_tests.support.sdk_container import SDKContainer from test_collections.sdk_tests.support.utils import prompt_for_commissioning_mode from .python_test_models import PythonTest, PythonTestType @@ -65,12 +66,12 @@ class PythonTestCase(TestCase, UserPromptSupport): in all instances of such subclass. """ + sdk_container: SDKContainer = SDKContainer(logger) python_test: PythonTest python_test_version: str def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) - self.chip_tool: ChipTool = ChipTool(logger) self.__runned = 0 self.test_stop_called = False @@ -221,10 +222,10 @@ async def execute(self) -> None: ) command.extend(command_arguments) - if self.chip_tool.pics_file_created: + if self.sdk_container.pics_file_created: command.append(f" --PICS {PICS_FILE_PATH}") - exec_result = self.chip_tool.send_command( + exec_result = self.sdk_container.send_command( command, prefix=EXECUTABLE, is_stream=True, diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 38dba23f..0e603897 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -19,10 +19,10 @@ from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestSuite from app.user_prompt_support.user_prompt_support import UserPromptSupport -from test_collections.sdk_tests.support.chip import ChipTool from test_collections.sdk_tests.support.python_testing.models.utils import ( commission_device, ) +from test_collections.sdk_tests.support.sdk_container import SDKContainer from test_collections.sdk_tests.support.utils import prompt_for_commissioning_mode @@ -45,7 +45,7 @@ class PythonTestSuite(TestSuite): python_test_version: str suite_name: str - chip_tool: ChipTool = ChipTool(logger) + sdk_container: SDKContainer = SDKContainer(logger) @classmethod def class_factory( @@ -89,20 +89,20 @@ async def setup(self) -> None: logger.info("Suite Setup") logger.info(f"Python Test Version: {self.python_test_version}") - logger.info("Starting SDK container") - await self.chip_tool.start_container() + logger.info("Setting up SDK container") + await self.sdk_container.start() if len(self.pics.clusters) > 0: logger.info("Create PICS file for DUT") - self.chip_tool.set_pics(pics=self.pics, in_container=True) + self.sdk_container.set_pics(pics=self.pics) else: - self.chip_tool.reset_pics_state() + self.sdk_container.reset_pics_state() async def cleanup(self) -> None: logger.info("Suite Cleanup") logger.info("Stopping SDK container") - await self.chip_tool.destroy_device() + self.sdk_container.destroy() class CommissioningPythonTestSuite(PythonTestSuite, UserPromptSupport): diff --git a/test_collections/sdk_tests/support/python_testing/models/utils.py b/test_collections/sdk_tests/support/python_testing/models/utils.py index 7391b5e5..abf13068 100644 --- a/test_collections/sdk_tests/support/python_testing/models/utils.py +++ b/test_collections/sdk_tests/support/python_testing/models/utils.py @@ -22,7 +22,7 @@ from app.schemas.test_environment_config import TestEnvironmentConfig from app.test_engine.logger import PYTHON_TEST_LEVEL -from test_collections.sdk_tests.support.chip import ChipTool +from test_collections.sdk_tests.support.sdk_container import SDKContainer # Command line params RUNNER_CLASS_PATH = "/root/python_testing/scripts/sdk/test_harness_client.py" @@ -73,13 +73,13 @@ def commission_device( config: TestEnvironmentConfig, logger: loguru.Logger, ) -> None: - chip_tool = ChipTool() + sdk_container = SDKContainer(logger) command = [f"{RUNNER_CLASS_PATH} commission"] command_arguments = generate_command_arguments(config) command.extend(command_arguments) - exec_result = chip_tool.send_command( + exec_result = sdk_container.send_command( command, prefix=EXECUTABLE, is_stream=True, @@ -88,7 +88,7 @@ def commission_device( handle_logs(cast(Generator, exec_result.output), logger) - exit_code = chip_tool.exec_exit_code(exec_result.exec_id) + exit_code = sdk_container.exec_exit_code(exec_result.exec_id) if exit_code: raise DUTCommissioningError("Failed to commission DUT") diff --git a/test_collections/sdk_tests/support/sdk_container.py b/test_collections/sdk_tests/support/sdk_container.py new file mode 100644 index 00000000..b4c8c6fa --- /dev/null +++ b/test_collections/sdk_tests/support/sdk_container.py @@ -0,0 +1,245 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Union + +import loguru +from docker.models.containers import Container + +from app.container_manager import container_manager +from app.core.config import settings +from app.schemas.pics import PICS, PICSError +from app.singleton import Singleton +from app.test_engine.logger import test_engine_logger as logger +from test_collections.sdk_tests.support.pics import set_pics_command + +from .exec_run_in_container import ExecResultExtended, exec_run_in_container + +# Trace mount +LOCAL_LOGS_PATH = Path("/var/tmp") +DOCKER_LOGS_PATH = "/logs" + +# PAA Cert mount +LOCAL_PAA_CERTS_PATH = Path("/var/paa-root-certs") +DOCKER_PAA_CERTS_PATH = "/paa-root-certs" + +# Credentials Development mount +LOCAL_CREDENTIALS_DEVELOPMENT_PATH = Path("/var/credentials/development") +DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development" + +# Python Testing Folder +LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections" +LOCAL_PYTHON_TESTING_PATH = Path( + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing" +) +DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" + +# RPC Client Running on SDK Container +LOCAL_RPC_PYTHON_TESTING_PATH = Path( + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/support/python_testing/models/rpc_client/" + "test_harness_client.py" +) +DOCKER_RPC_PYTHON_TESTING_PATH = ( + "/root/python_testing/scripts/sdk/test_harness_client.py" +) + + +class SDKContainerNotRunning(Exception): + """Raised when we attempt to use the docker container, but it is not running""" + + +class SDKContainerRetrieveExitCodeError(Exception): + """Raised when there's an error in the attempt to retrieve an execution's exit + code""" + + +class SDKContainer(metaclass=Singleton): + """ + Base class for the SDK container to be setup and managed. + + Usage: + Create an instance by calling initializer. When ready to use, ... + """ + + container_name = settings.SDK_CONTAINER_NAME + image_tag = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}" + run_parameters = { + "privileged": True, + "detach": True, + "network": "host", + "name": container_name, + "command": "tail -f /dev/null", # while true + "volumes": { + "/var/run/dbus/system_bus_socket": { + "bind": "/var/run/dbus/system_bus_socket", + "mode": "rw", + }, + LOCAL_LOGS_PATH: { + "bind": DOCKER_LOGS_PATH, + "mode": "rw", + }, + LOCAL_PAA_CERTS_PATH: { + "bind": DOCKER_PAA_CERTS_PATH, + "mode": "ro", + }, + LOCAL_CREDENTIALS_DEVELOPMENT_PATH: { + "bind": DOCKER_CREDENTIALS_DEVELOPMENT_PATH, + "mode": "ro", + }, + LOCAL_PYTHON_TESTING_PATH: { + "bind": DOCKER_PYTHON_TESTING_PATH, + "mode": "rw", + }, + LOCAL_RPC_PYTHON_TESTING_PATH: { + "bind": DOCKER_RPC_PYTHON_TESTING_PATH, + "mode": "rw", + }, + }, + } + + def __init__( + self, + logger: loguru.Logger = logger, + ) -> None: + """ + Args: + logger (Logger, optional): Optional logger injection. Defaults to standard + self.logger. + """ + self.__container: Optional[Container] = None + + self.__pics_file_created = False + self.logger = logger + + @property + def pics_file_created(self) -> bool: + return self.__pics_file_created + + def __destroy_existing_container(self) -> None: + """This will kill and remove any existing container using the same name.""" + existing_container = container_manager.get_container(self.container_name) + if existing_container is not None: + logger.info( + f'Existing container named "{self.container_name}" found. Destroying.' + ) + container_manager.destroy(existing_container) + + def is_running(self) -> bool: + if self.__container is None: + return False + else: + return container_manager.is_running(self.__container) + + async def start(self) -> None: + """Creates the SDK container. + + Returns only when the container is created. + """ + + if self.is_running(): + self.logger.info( + "SDK container already running, no need to start a new container" + ) + return + + # Ensure there's no existing container running using the same name. + self.__destroy_existing_container() + + # Async return when the container is running + self.__container = await container_manager.create_container( + self.image_tag, self.run_parameters + ) + + self.logger.info( + f"{self.container_name} container started" + f" with configuration: {self.run_parameters}" + ) + + def destroy(self) -> None: + """Destroy the container.""" + if self.__container is not None: + container_manager.destroy(self.__container) + self.__container = None + + def send_command( + self, + command: Union[str, list], + prefix: str, + is_stream: bool = False, + is_socket: bool = False, + ) -> ExecResultExtended: + if self.__container is None: + raise SDKContainerNotRunning() + + full_cmd = [prefix] + if isinstance(command, list): + full_cmd += command + else: + full_cmd.append(str(command)) + + self.logger.info("Sending command to SDK container: " + " ".join(full_cmd)) + + result = exec_run_in_container( + self.__container, + " ".join(full_cmd), + socket=is_socket, + stream=is_stream, + stdin=True, + ) + + return result + + def exec_exit_code(self, exec_id: str) -> Optional[int]: + if self.__container is None: + raise SDKContainerRetrieveExitCodeError( + "No SDK container, cannot get execution exit code" + ) + + exec_data = self.__container.client.api.exec_inspect(exec_id) + + if exec_data is None: + raise SDKContainerRetrieveExitCodeError( + f"Docker didn't return any execution metadata for exec_id {exec_id}" + ) + + return exec_data.get("ExitCode") + + def set_pics(self, pics: PICS) -> None: + """Sends command to create pics file. + + Args: + pics (PICS): PICS that contains all the pics codes + + Raises: + PICSError: If creating PICS file inside the container fails. + """ + # List of default PICS which needs to set specifically in TH are added here. + # These PICS are applicable for CI / Chip tool testing purposes only. + # These PICS are unknown / not visible to external users. + + prefix, cmd = set_pics_command(pics) + + exec_result = self.send_command(cmd, prefix=prefix) + + if exec_result.exit_code != 0: + raise PICSError("Creating PICS file failed") + + self.__pics_file_created = True + + def reset_pics_state(self) -> None: + self.__pics_file_created = False diff --git a/test_collections/sdk_tests/support/tests/chip/test_chip_tool.py b/test_collections/sdk_tests/support/tests/chip/test_chip_tool.py index 4d16cb78..d87bde7e 100644 --- a/test_collections/sdk_tests/support/tests/chip/test_chip_tool.py +++ b/test_collections/sdk_tests/support/tests/chip/test_chip_tool.py @@ -24,11 +24,10 @@ from matter_yamltests.hooks import TestParserHooks, TestRunnerHooks from matter_yamltests.parser_builder import TestParserBuilderConfig from matter_yamltests.runner import TestRunnerConfig +from matter_yamltests.websocket_runner import WebSocketRunner, WebSocketRunnerConfig -from app.container_manager import container_manager from app.core.config import settings from app.schemas.pics import PICSError -from app.tests.utils.docker import make_fake_container from app.tests.utils.test_pics_data import create_random_pics from test_collections.sdk_tests.support.chip import ChipTool from test_collections.sdk_tests.support.chip.chip_tool import ( @@ -38,105 +37,30 @@ CHIP_TOOL_EXE, CHIP_TOOL_TEST_DEFAULT_TIMEOUT_IN_SEC, DOCKER_PAA_CERTS_PATH, - PICS_FILE_PATH, - SHELL_OPTION, - SHELL_PATH, TEST_STEP_DELAY_VALUE, ChipTestType, - ChipToolNotRunning, ChipToolStartingError, ChipToolUnknownTestType, ) -from test_collections.sdk_tests.support.chip.exec_run_in_container import ( - ExecResultExtended, +from test_collections.sdk_tests.support.exec_run_in_container import ExecResultExtended +from test_collections.sdk_tests.support.pics import ( + PICS_FILE_PATH, + SHELL_OPTION, + SHELL_PATH, ) - - -@pytest.mark.asyncio -async def test_start_container() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - - # Values to verify - docker_image = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}" - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container", return_value=None - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool.backend_container" - ), mock.patch.object( - target=container_manager, attribute="create_container" - ) as mock_create_container, mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ) as mock_start_chip_server: - await chip_tool.start_server(test_type) - - mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) - mock_start_chip_server.assert_awaited_once_with(test_type, False) - assert chip_tool._ChipTool__chip_tool_container is not None - - # clean up: - chip_tool._ChipTool__chip_tool_container = None - - -@pytest.mark.asyncio -async def test_start_container_using_paa_certs() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - - # Values to verify - docker_image = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}" - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container", return_value=None - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool.backend_container" - ), mock.patch.object( - target=container_manager, attribute="create_container" - ) as mock_create_container, mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ) as mock_start_chip_server: - await chip_tool.start_server(test_type, use_paa_certs=True) - - mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) - mock_start_chip_server.assert_awaited_once_with(test_type, True) - assert chip_tool._ChipTool__chip_tool_container is not None - - # clean up: - chip_tool._ChipTool__chip_tool_container = None - - -@pytest.mark.asyncio -async def test_not_start_container_when_running() -> None: - chip_tool = ChipTool() - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=True - ), mock.patch.object( - target=container_manager, attribute="create_container" - ) as mock_create_container, mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ) as mock_start_chip_server: - await chip_tool.start_container() - - mock_create_container.assert_not_called() - mock_start_chip_server.assert_not_called() - assert chip_tool._ChipTool__chip_tool_container is None +from test_collections.sdk_tests.support.sdk_container import SDKContainer @pytest.mark.asyncio async def test_start_chip_server_already_started() -> None: chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_TOOL chip_tool._ChipTool__server_logs = "log output".encode() chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="send_command" + target=sdk_container, attribute="send_command" ) as mock_send_command: await chip_tool.start_chip_server(test_type) @@ -150,10 +74,11 @@ async def test_start_chip_server_already_started() -> None: @pytest.mark.asyncio async def test_start_chip_server_unsupported_test_type() -> None: chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = "unsupported" with mock.patch.object( - target=chip_tool, attribute="send_command" + target=sdk_container, attribute="send_command" ) as mock_send_command, pytest.raises(ChipToolUnknownTestType): await chip_tool.start_chip_server(test_type, use_paa_certs=False) @@ -167,6 +92,7 @@ async def test_start_chip_server_waiting_failure() -> None: settings.CHIP_TOOL_TRACE = False chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_TOOL mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) @@ -174,7 +100,7 @@ async def test_start_chip_server_waiting_failure() -> None: expected_prefix = CHIP_TOOL_EXE with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch.object( target=chip_tool, attribute="_ChipTool__wait_for_server_start", @@ -192,7 +118,7 @@ async def test_start_chip_server_waiting_failure() -> None: # clean up: chip_tool._ChipTool__server_logs = None - chip_tool._ChipTool__chip_tool_server_id = None + chip_tool._ChipTool__chip_server_id = None settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -203,6 +129,7 @@ async def test_start_chip_server_chip_tool() -> None: settings.CHIP_TOOL_TRACE = False chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_TOOL mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) @@ -210,7 +137,7 @@ async def test_start_chip_server_chip_tool() -> None: expected_prefix = CHIP_TOOL_EXE with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch.object( target=chip_tool, attribute="_ChipTool__wait_for_server_start", @@ -226,7 +153,7 @@ async def test_start_chip_server_chip_tool() -> None: # clean up: chip_tool._ChipTool__server_logs = None - chip_tool._ChipTool__chip_tool_server_id = None + chip_tool._ChipTool__chip_server_id = None chip_tool._ChipTool__server_started = False settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -238,6 +165,7 @@ async def test_start_chip_server_chip_tool_using_paa_certs() -> None: settings.CHIP_TOOL_TRACE = False chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_TOOL mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) @@ -249,7 +177,7 @@ async def test_start_chip_server_chip_tool_using_paa_certs() -> None: expected_prefix = CHIP_TOOL_EXE with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch.object( target=chip_tool, attribute="_ChipTool__wait_for_server_start", @@ -265,7 +193,7 @@ async def test_start_chip_server_chip_tool_using_paa_certs() -> None: # clean up: chip_tool._ChipTool__server_logs = None - chip_tool._ChipTool__chip_tool_server_id = None + chip_tool._ChipTool__chip_server_id = None chip_tool._ChipTool__server_started = False settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -277,6 +205,7 @@ async def test_start_chip_server_chip_app() -> None: settings.CHIP_TOOL_TRACE = False chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_APP mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) @@ -284,7 +213,7 @@ async def test_start_chip_server_chip_app() -> None: expected_prefix = CHIP_APP_EXE with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch.object( target=chip_tool, attribute="_ChipTool__wait_for_server_start", @@ -300,7 +229,7 @@ async def test_start_chip_server_chip_app() -> None: # clean up: chip_tool._ChipTool__server_logs = None - chip_tool._ChipTool__chip_tool_server_id = None + chip_tool._ChipTool__chip_server_id = None chip_tool._ChipTool__server_started = False settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -312,6 +241,7 @@ async def test_start_chip_server_chip_app_using_paa_certs() -> None: settings.CHIP_TOOL_TRACE = False chip_tool = ChipTool() + sdk_container: SDKContainer = SDKContainer() test_type = ChipTestType.CHIP_APP mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) @@ -323,7 +253,7 @@ async def test_start_chip_server_chip_app_using_paa_certs() -> None: expected_prefix = CHIP_APP_EXE with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch.object( target=chip_tool, attribute="_ChipTool__wait_for_server_start", @@ -339,91 +269,14 @@ async def test_start_chip_server_chip_app_using_paa_certs() -> None: # clean up: chip_tool._ChipTool__server_logs = None - chip_tool._ChipTool__chip_tool_server_id = None + chip_tool._ChipTool__chip_server_id = None chip_tool._ChipTool__server_started = False settings.CHIP_TOOL_TRACE = original_trace_setting_value -@pytest.mark.asyncio -async def test_destroy_container_running() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ) as mock_destroy, mock.patch.object( - target=container_manager, attribute="create_container" - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ): - await chip_tool.start_server(test_type) - await chip_tool.start_server(test_type) - - assert chip_tool._ChipTool__chip_tool_container is not None - - await chip_tool.destroy_device() - - mock_destroy.assert_called() - assert chip_tool._ChipTool__chip_tool_container is None - - -@pytest.mark.asyncio -async def test_destroy_container_not_running() -> None: - chip_tool = ChipTool() - - with mock.patch.object( - target=container_manager, attribute="destroy" - ) as mock_destroy: - await chip_tool.destroy_device() - - mock_destroy.assert_not_called() - assert chip_tool._ChipTool__chip_tool_container is None - - -@pytest.mark.asyncio -async def test_destroy_container_once() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container", return_value=None - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool.backend_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ) as mock_destroy, mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=make_fake_container(), - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ): - await chip_tool.start_server(test_type) - - await chip_tool.destroy_device() - await chip_tool.destroy_device() - - mock_destroy.assert_called_once() - assert chip_tool._ChipTool__chip_tool_container is None - - -def test_send_command_without_starting() -> None: - chip_tool = ChipTool() - - with pytest.raises(ChipToolNotRunning): - chip_tool.send_command("--help", prefix=CHIP_TOOL_EXE) - - @pytest.mark.asyncio async def test_set_pics() -> None: chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL pics = create_random_pics() # expected PICS = PICS from create_random_pics() + \n + DEFAULT PICS @@ -437,33 +290,17 @@ async def test_set_pics() -> None: f'> {PICS_FILE_PATH}"' ) - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=make_fake_container(), - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( + with mock.patch( target="test_collections.sdk_tests.support.chip.chip_tool.subprocess.run", return_value=CompletedProcess(expected_command, 0), ) as mock_run: - await chip_tool.start_server(test_type) - - chip_tool.set_pics(pics, in_container=False) + chip_tool.set_pics(pics) mock_run.assert_called_once_with(expected_command, shell=True) assert chip_tool._ChipTool__pics_file_created is True # clean up: chip_tool._ChipTool__pics_file_created = False - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None def test_set_pics_with_error() -> None: @@ -474,99 +311,9 @@ def test_set_pics_with_error() -> None: target="test_collections.sdk_tests.support.chip.chip_tool.subprocess.run", return_value=CompletedProcess("", 1), ), pytest.raises(PICSError): - chip_tool.set_pics(pics, in_container=False) - assert chip_tool._ChipTool__pics_file_created is False + chip_tool.set_pics(pics) - # clean up: - chip_tool._ChipTool__last_exec_id = None - - -@pytest.mark.asyncio -async def test_send_command_default_prefix() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - fake_container = make_fake_container() - cmd = "--help" - chip_tool_prefix = CHIP_TOOL_EXE - mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool." - "exec_run_in_container", - return_value=mock_result, - ) as mock_exec_run: - await chip_tool.start_server(test_type) - - result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) - - mock_exec_run.assert_called_once_with( - fake_container, - f"{chip_tool_prefix} {cmd}", - socket=False, - stream=False, - stdin=True, - ) - assert result == mock_result - - # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None - - -@pytest.mark.asyncio -async def test_send_command_custom_prefix() -> None: - chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - fake_container = make_fake_container() - cmd = "--help" - chip_tool_prefix = "cat" - mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) - - with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool." - "exec_run_in_container", - return_value=mock_result, - ) as mock_exec_run: - await chip_tool.start_server(test_type) - - result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) - - mock_exec_run.assert_called_once_with( - fake_container, - f"{chip_tool_prefix} {cmd}", - socket=False, - stream=False, - stdin=True, - ) - assert result == mock_result - - # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + assert chip_tool._ChipTool__pics_file_created is False @pytest.mark.asyncio @@ -574,31 +321,17 @@ async def test_run_test_default_config() -> None: chip_tool = ChipTool() test_type = ChipTestType.CHIP_TOOL test_id = "TC_TEST_ID" - fake_container = make_fake_container() + runner = WebSocketRunner(WebSocketRunnerConfig()) + chip_tool._ChipTool__test_harness_runner = runner + chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" + target=chip_tool, attribute="start_runner" ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.start", - return_value=True, - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.run", + target=runner, + attribute="run", return_value=True, ) as mock_run: - await chip_tool.start_server(test_type) - await chip_tool.run_test( test_step_interface=TestRunnerHooks(), test_parser_hooks=TestParserHooks(), @@ -623,8 +356,8 @@ async def test_run_test_default_config() -> None: assert runner_options.delay_in_ms == TEST_STEP_DELAY_VALUE # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + chip_tool._ChipTool__test_harness_runner = None + chip_tool._ChipTool__server_started = False @pytest.mark.asyncio @@ -633,31 +366,17 @@ async def test_run_test_custom_timeout() -> None: test_type = ChipTestType.CHIP_TOOL test_id = "TC_TEST_ID" test_timeout = "900" - fake_container = make_fake_container() + runner = WebSocketRunner(WebSocketRunnerConfig()) + chip_tool._ChipTool__test_harness_runner = runner + chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False + target=chip_tool, attribute="start_runner" ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.start", - return_value=True, - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.run", + target=runner, + attribute="run", return_value=True, ) as mock_run: - await chip_tool.start_server(test_type) - await chip_tool.run_test( test_step_interface=TestRunnerHooks(), test_parser_hooks=TestParserHooks(), @@ -675,8 +394,8 @@ async def test_run_test_custom_timeout() -> None: assert parser_options["timeout"] == f"{test_timeout}" # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + chip_tool._ChipTool__test_harness_runner = None + chip_tool._ChipTool__server_started = False @pytest.mark.asyncio @@ -686,31 +405,17 @@ async def test_run_test_with_custom_parameter() -> None: test_id = "TC_TEST_ID" test_param_name = "param1" test_param_value = "value" - fake_container = make_fake_container() + runner = WebSocketRunner(WebSocketRunnerConfig()) + chip_tool._ChipTool__test_harness_runner = runner + chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" + target=chip_tool, attribute="start_runner" ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.start", - return_value=True, - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.run", + target=runner, + attribute="run", return_value=True, ) as mock_run: - await chip_tool.start_server(test_type) - await chip_tool.run_test( test_step_interface=TestRunnerHooks(), test_parser_hooks=TestParserHooks(), @@ -729,8 +434,8 @@ async def test_run_test_with_custom_parameter() -> None: assert parser_options.get(test_param_name) == test_param_value # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + chip_tool._ChipTool__test_harness_runner = None + chip_tool._ChipTool__server_started = False @pytest.mark.asyncio @@ -740,31 +445,17 @@ async def test_run_test_with_endpoint_parameter() -> None: test_id = "TC_TEST_ID" test_param_name = "endpoint" test_param_value = 1 - fake_container = make_fake_container() + runner = WebSocketRunner(WebSocketRunnerConfig()) + chip_tool._ChipTool__test_harness_runner = runner + chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" + target=chip_tool, attribute="start_runner" ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.start", - return_value=True, - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.run", + target=runner, + attribute="run", return_value=True, ) as mock_run: - await chip_tool.start_server(test_type) - await chip_tool.run_test( test_step_interface=TestRunnerHooks(), test_parser_hooks=TestParserHooks(), @@ -782,8 +473,8 @@ async def test_run_test_with_endpoint_parameter() -> None: assert parser_options[test_param_name] == test_param_value # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + chip_tool._ChipTool__test_harness_runner = None + chip_tool._ChipTool__server_started = False @pytest.mark.asyncio @@ -793,31 +484,17 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None: test_id = "TC_TEST_ID" test_param_name = "endpoint" test_param_value = 1 - fake_container = make_fake_container() + runner = WebSocketRunner(WebSocketRunnerConfig()) + chip_tool._ChipTool__test_harness_runner = runner + chip_tool._ChipTool__server_started = True with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" + target=chip_tool, attribute="start_runner" ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.start", - return_value=True, - ), mock.patch( - target="test_collections.sdk_tests.support.chip.chip_tool" - ".WebSocketRunner.run", + target=runner, + attribute="run", return_value=True, ) as mock_run: - await chip_tool.start_server(test_type) - await chip_tool.run_test( test_step_interface=TestRunnerHooks(), test_parser_hooks=TestParserHooks(), @@ -841,8 +518,8 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None: assert parser_options.get("cluster") != "custom" # clean up: - chip_tool._ChipTool__last_exec_id = None - chip_tool._ChipTool__chip_tool_container = None + chip_tool._ChipTool__test_harness_runner = None + chip_tool._ChipTool__server_started = False @pytest.mark.asyncio @@ -853,31 +530,14 @@ async def test_pairing_on_network_command_params() -> None: # Attributes chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - fake_container = make_fake_container() discriminator = "1234" setup_code = "0123456" with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch.object( target=chip_tool, attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_server(test_type) - - # Send on-network pairing command result = await chip_tool.pairing_on_network( setup_code=setup_code, discriminator=discriminator, @@ -890,7 +550,6 @@ async def test_pairing_on_network_command_params() -> None: mock_send_websocket_command.assert_awaited_once_with(expected_command) # clean up: - chip_tool._ChipTool__chip_tool_container = None settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -902,33 +561,16 @@ async def test_pairing_ble_wifi_command_params() -> None: # Attributes chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - fake_container = make_fake_container() discriminator = "1234" setup_code = "0123456" ssid = "WifiIsGood" password = "WifiIsGoodAndSecret" with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch.object( target=chip_tool, attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_server(test_type) - - # Send BLE-WIFI pairing command result = await chip_tool.pairing_ble_wifi( ssid=ssid, password=password, @@ -945,7 +587,6 @@ async def test_pairing_ble_wifi_command_params() -> None: mock_send_websocket_command.assert_awaited_once_with(expected_command) # clean up: - chip_tool._ChipTool__chip_tool_container = None settings.CHIP_TOOL_TRACE = original_trace_setting_value @@ -957,33 +598,16 @@ async def test_pairing_ble_thread_command_params() -> None: # Attributes chip_tool = ChipTool() - test_type = ChipTestType.CHIP_TOOL - fake_container = make_fake_container() discriminator = "1234" hex_dataset = "c0ffee" setup_code = "0123456" with mock.patch.object( - target=chip_tool, attribute="is_running", return_value=False - ), mock.patch.object( - target=container_manager, attribute="get_container" - ), mock.patch.object( - target=container_manager, attribute="destroy" - ), mock.patch.object( - target=container_manager, - attribute="create_container", - return_value=fake_container, - ), mock.patch.object( - target=chip_tool, attribute="start_chip_server" - ), mock.patch.object( target=chip_tool, attribute="send_websocket_command", return_value='{"results": []}', # '{ "results": [{ "error": "FAILURE" }] ) as mock_send_websocket_command: - await chip_tool.start_server(test_type) - - # Send BLE-THREAD pairing command result = await chip_tool.pairing_ble_thread( hex_dataset=hex_dataset, setup_code=setup_code, @@ -999,5 +623,4 @@ async def test_pairing_ble_thread_command_params() -> None: mock_send_websocket_command.assert_awaited_once_with(expected_command) # clean up: - chip_tool._ChipTool__chip_tool_container = None settings.CHIP_TOOL_TRACE = original_trace_setting_value diff --git a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py index becd4d03..cbe6ef99 100644 --- a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py +++ b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py @@ -23,7 +23,6 @@ from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger -from test_collections.sdk_tests.support.chip.test_case import TestError from test_collections.sdk_tests.support.models.matter_test_models import ( MatterTestStep, MatterTestType, diff --git a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py index af335554..974a9ee1 100644 --- a/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py +++ b/test_collections/sdk_tests/support/tests/python_tests/test_python_test_suite.py @@ -25,12 +25,12 @@ from app.schemas import PICS from app.test_engine.logger import test_engine_logger from app.tests.utils.test_pics_data import create_random_pics -from test_collections.sdk_tests.support.chip.chip_tool import ChipTool from test_collections.sdk_tests.support.python_testing.models.test_suite import ( CommissioningPythonTestSuite, PythonTestSuite, SuiteType, ) +from test_collections.sdk_tests.support.sdk_container import SDKContainer def test_python_suite_class_factory_name() -> None: @@ -75,7 +75,7 @@ def test_commissioning_suite_subclass() -> None: @pytest.mark.asyncio async def test_suite_setup_log_python_version() -> None: """Test that test suite python version is logged to test engine logger in setup.""" - chip_tool: ChipTool = ChipTool() + sdk_container: SDKContainer = SDKContainer() for type in list(SuiteType): python_test_version = "best_version" @@ -89,7 +89,7 @@ async def test_suite_setup_log_python_version() -> None: with mock.patch.object( target=test_engine_logger, attribute="info" ) as logger_info, mock.patch.object( - target=chip_tool, attribute="start_container" + target=sdk_container, attribute="start" ), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".PythonTestSuite.pics", @@ -114,7 +114,7 @@ async def test_suite_setup_log_python_version() -> None: @pytest.mark.asyncio async def test_suite_setup_without_pics() -> None: - chip_tool: ChipTool = ChipTool() + sdk_container: SDKContainer = SDKContainer() for type in list(SuiteType): python_test_version = "best_version" @@ -125,16 +125,14 @@ async def test_suite_setup_without_pics() -> None: suite_instance = suite_class(TestSuiteExecution()) - with mock.patch( - "test_collections.sdk_tests.support.chip.test_suite.ChipSuite.setup" - ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + with mock.patch.object(target=sdk_container, attribute="start"), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".PythonTestSuite.pics", new_callable=PICS, ), mock.patch.object( - target=chip_tool, attribute="set_pics" + target=sdk_container, attribute="set_pics" ) as mock_set_pics, mock.patch.object( - target=chip_tool, attribute="reset_pics_state" + target=sdk_container, attribute="reset_pics_state" ) as mock_reset_pics_state, mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".prompt_for_commissioning_mode", @@ -155,7 +153,7 @@ async def test_suite_setup_without_pics() -> None: @pytest.mark.asyncio async def test_suite_setup_with_pics() -> None: - chip_tool: ChipTool = ChipTool() + sdk_container: SDKContainer = SDKContainer() for type in list(SuiteType): python_test_version = "best_version" @@ -166,16 +164,14 @@ async def test_suite_setup_with_pics() -> None: suite_instance = suite_class(TestSuiteExecution()) - with mock.patch( - "test_collections.sdk_tests.support.chip.test_suite.ChipSuite.setup" - ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + with mock.patch.object(target=sdk_container, attribute="start"), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".PythonTestSuite.pics", new_callable=create_random_pics, ), mock.patch.object( - target=chip_tool, attribute="set_pics" + target=sdk_container, attribute="set_pics" ) as mock_set_pics, mock.patch.object( - target=chip_tool, attribute="reset_pics_state" + target=sdk_container, attribute="reset_pics_state" ) as mock_reset_pics_state, mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".prompt_for_commissioning_mode", @@ -196,7 +192,7 @@ async def test_suite_setup_with_pics() -> None: @pytest.mark.asyncio async def test_commissioning_suite_setup_with_pics() -> None: - chip_tool: ChipTool = ChipTool() + sdk_container: SDKContainer = SDKContainer() python_test_version = "best_version" # Create a subclass of PythonTestSuite @@ -208,15 +204,11 @@ async def test_commissioning_suite_setup_with_pics() -> None: suite_instance = suite_class(TestSuiteExecution()) - with mock.patch( - "test_collections.sdk_tests.support.chip.test_suite.ChipSuite.setup" - ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + with mock.patch.object(target=sdk_container, attribute="start"), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".PythonTestSuite.pics", new_callable=PICS, - ), mock.patch.object( - target=chip_tool, attribute="reset_pics_state" - ), mock.patch( + ), mock.patch.object(target=sdk_container, attribute="set_pics"), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.test_suite" ".prompt_for_commissioning_mode", ) as mock_prompt_for_commissioning_mode, mock.patch( diff --git a/test_collections/sdk_tests/support/tests/python_tests/test_utils.py b/test_collections/sdk_tests/support/tests/python_tests/test_utils.py index f6663032..713f0188 100644 --- a/test_collections/sdk_tests/support/tests/python_tests/test_utils.py +++ b/test_collections/sdk_tests/support/tests/python_tests/test_utils.py @@ -20,16 +20,15 @@ from app.default_environment_config import default_environment_config from app.schemas.test_environment_config import DutConfig, DutPairingModeEnum from app.test_engine.logger import test_engine_logger -from test_collections.sdk_tests.support.chip.chip_tool import ChipTool -from test_collections.sdk_tests.support.chip.exec_run_in_container import ( - ExecResultExtended, -) +from test_collections.sdk_tests.support.exec_run_in_container import ExecResultExtended from test_collections.sdk_tests.support.python_testing.models.utils import ( EXECUTABLE, RUNNER_CLASS_PATH, + DUTCommissioningError, commission_device, generate_command_arguments, ) +from test_collections.sdk_tests.support.sdk_container import SDKContainer @pytest.mark.asyncio @@ -204,7 +203,36 @@ async def test_generate_command_arguments_omit_comissioning_method() -> None: @pytest.mark.asyncio async def test_commission_device() -> None: - chip_tool: ChipTool = ChipTool() + sdk_container: SDKContainer = SDKContainer() + + command_args = ["arg1", "arg2", "arg3"] + expected_command = [f"{RUNNER_CLASS_PATH} commission"] + expected_command.extend(command_args) + mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) + + with mock.patch.object( + target=sdk_container, attribute="send_command", return_value=mock_result + ) as mock_send_command, mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.utils" + ".generate_command_arguments", + return_value=command_args, + ), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.utils" + ".handle_logs" + ) as mock_handle_logs, mock.patch.object( + target=sdk_container, attribute="exec_exit_code", return_value=0 + ): + commission_device(default_environment_config, test_engine_logger) + + mock_send_command.assert_called_once_with( + expected_command, prefix=EXECUTABLE, is_stream=True, is_socket=False + ) + mock_handle_logs.assert_called_once() + + +@pytest.mark.asyncio +async def test_commission_device_failure() -> None: + sdk_container: SDKContainer = SDKContainer() command_args = ["arg1", "arg2", "arg3"] expected_command = [f"{RUNNER_CLASS_PATH} commission"] @@ -212,7 +240,7 @@ async def test_commission_device() -> None: mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) with mock.patch.object( - target=chip_tool, attribute="send_command", return_value=mock_result + target=sdk_container, attribute="send_command", return_value=mock_result ) as mock_send_command, mock.patch( target="test_collections.sdk_tests.support.python_testing.models.utils" ".generate_command_arguments", @@ -220,7 +248,11 @@ async def test_commission_device() -> None: ), mock.patch( target="test_collections.sdk_tests.support.python_testing.models.utils" ".handle_logs" - ) as mock_handle_logs: + ) as mock_handle_logs, mock.patch.object( + target=sdk_container, attribute="exec_exit_code", return_value=1 + ), pytest.raises( + DUTCommissioningError + ): commission_device(default_environment_config, test_engine_logger) mock_send_command.assert_called_once_with( diff --git a/test_collections/sdk_tests/support/tests/chip/test_docker_extension.py b/test_collections/sdk_tests/support/tests/test_exec_run_in_container.py similarity index 96% rename from test_collections/sdk_tests/support/tests/chip/test_docker_extension.py rename to test_collections/sdk_tests/support/tests/test_exec_run_in_container.py index 632263b2..c32997de 100644 --- a/test_collections/sdk_tests/support/tests/chip/test_docker_extension.py +++ b/test_collections/sdk_tests/support/tests/test_exec_run_in_container.py @@ -16,7 +16,7 @@ from typing import Generator from app.tests.utils.docker import make_fake_container -from test_collections.sdk_tests.support.chip.exec_run_in_container import ( +from test_collections.sdk_tests.support.exec_run_in_container import ( exec_run_in_container, ) diff --git a/test_collections/sdk_tests/support/tests/test_sdk_container.py b/test_collections/sdk_tests/support/tests/test_sdk_container.py new file mode 100644 index 00000000..69b83205 --- /dev/null +++ b/test_collections/sdk_tests/support/tests/test_sdk_container.py @@ -0,0 +1,214 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# type: ignore +# Ignore mypy type check for this file + +from unittest import mock + +import pytest + +from app.container_manager import container_manager +from app.core.config import settings +from app.tests.utils.docker import make_fake_container +from test_collections.sdk_tests.support.exec_run_in_container import ExecResultExtended +from test_collections.sdk_tests.support.sdk_container import ( + SDKContainer, + SDKContainerNotRunning, +) + + +@pytest.mark.asyncio +async def test_start() -> None: + sdk_container: SDKContainer = SDKContainer() + + # Values to verify + docker_image = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}" + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=container_manager, attribute="get_container", return_value=None + ), mock.patch.object( + target=container_manager, attribute="create_container" + ) as mock_create_container: + await sdk_container.start() + + mock_create_container.assert_called_once_with( + docker_image, SDKContainer.run_parameters + ) + assert sdk_container._SDKContainer__container is not None + + # clean up: + sdk_container._SDKContainer__container = None + + +@pytest.mark.asyncio +async def test_not_start_when_running() -> None: + sdk_container: SDKContainer = SDKContainer() + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=True + ), mock.patch.object( + target=container_manager, attribute="create_container" + ) as mock_create_container: + await sdk_container.start() + + mock_create_container.assert_not_called() + assert sdk_container._SDKContainer__container is None + + +@pytest.mark.asyncio +async def test_destroy_container_running() -> None: + sdk_container: SDKContainer = SDKContainer() + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=container_manager, attribute="get_container", return_value=None + ), mock.patch.object( + target=container_manager, attribute="destroy" + ) as mock_destroy, mock.patch.object( + target=container_manager, attribute="create_container" + ): + await sdk_container.start() + + assert sdk_container._SDKContainer__container is not None + + sdk_container.destroy() + + mock_destroy.assert_called() + assert sdk_container._SDKContainer__container is None + + +@pytest.mark.asyncio +async def test_destroy_container_not_running() -> None: + sdk_container: SDKContainer = SDKContainer() + + with mock.patch.object( + target=container_manager, attribute="destroy" + ) as mock_destroy: + sdk_container.destroy() + + mock_destroy.assert_not_called() + assert sdk_container._SDKContainer__container is None + + +@pytest.mark.asyncio +async def test_destroy_container_once() -> None: + sdk_container: SDKContainer = SDKContainer() + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=container_manager, attribute="get_container", return_value=None + ), mock.patch.object( + target=container_manager, attribute="destroy" + ) as mock_destroy, mock.patch.object( + target=container_manager, + attribute="create_container", + return_value=make_fake_container(), + ): + await sdk_container.start() + + sdk_container.destroy() + sdk_container.destroy() + + mock_destroy.assert_called_once() + assert sdk_container._SDKContainer__container is None + + +def test_send_command_without_starting() -> None: + sdk_container: SDKContainer = SDKContainer() + + with pytest.raises(SDKContainerNotRunning): + sdk_container.send_command("--help", prefix="cmd-prefix") + + +@pytest.mark.asyncio +async def test_send_command_default_prefix() -> None: + sdk_container: SDKContainer = SDKContainer() + fake_container = make_fake_container() + cmd = "--help" + cmd_prefix = "cmd-prefix" + mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=container_manager, attribute="get_container", return_value=None + ), mock.patch.object( + target=container_manager, + attribute="create_container", + return_value=fake_container, + ), mock.patch( + target="test_collections.sdk_tests.support.sdk_container.exec_run_in_container", + return_value=mock_result, + ) as mock_exec_run: + await sdk_container.start() + + result = sdk_container.send_command(cmd, prefix=cmd_prefix) + + mock_exec_run.assert_called_once_with( + fake_container, + f"{cmd_prefix} {cmd}", + socket=False, + stream=False, + stdin=True, + ) + assert result == mock_result + + # clean up: + sdk_container._SDKContainer__last_exec_id = None + sdk_container._SDKContainer__container = None + + +@pytest.mark.asyncio +async def test_send_command_custom_prefix() -> None: + sdk_container: SDKContainer = SDKContainer() + fake_container = make_fake_container() + cmd = "--help" + cmd_prefix = "cat" + mock_result = ExecResultExtended(0, "log output".encode(), "ID", mock.MagicMock()) + + with mock.patch.object( + target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=container_manager, attribute="get_container", return_value=None + ), mock.patch.object( + target=container_manager, + attribute="create_container", + return_value=fake_container, + ), mock.patch( + target="test_collections.sdk_tests.support.sdk_container.exec_run_in_container", + return_value=mock_result, + ) as mock_exec_run: + await sdk_container.start() + + result = sdk_container.send_command(cmd, prefix=cmd_prefix) + + mock_exec_run.assert_called_once_with( + fake_container, + f"{cmd_prefix} {cmd}", + socket=False, + stream=False, + stdin=True, + ) + assert result == mock_result + + # clean up: + sdk_container._SDKContainer__last_exec_id = None + sdk_container._SDKContainer__container = None diff --git a/test_collections/sdk_tests/support/tests/chip/test_test_suite.py b/test_collections/sdk_tests/support/tests/yaml_tests/test_chip_suite.py similarity index 97% rename from test_collections/sdk_tests/support/tests/chip/test_test_suite.py rename to test_collections/sdk_tests/support/tests/yaml_tests/test_chip_suite.py index cfe1251e..5538d825 100644 --- a/test_collections/sdk_tests/support/tests/chip/test_test_suite.py +++ b/test_collections/sdk_tests/support/tests/yaml_tests/test_chip_suite.py @@ -23,12 +23,12 @@ from app.models.test_suite_execution import TestSuiteExecution from app.user_prompt_support.constants import UserResponseStatusEnum from app.user_prompt_support.prompt_response import PromptResponse -from test_collections.sdk_tests.support.chip.test_case import PromptOption -from test_collections.sdk_tests.support.chip.test_suite import ( +from test_collections.sdk_tests.support.yaml_tests.models.chip_suite import ( ChipSuite, DUTCommissioningError, SuiteSetupError, ) +from test_collections.sdk_tests.support.yaml_tests.models.chip_test import PromptOption RETRY_PROMPT_RESPONSE = PromptResponse( response=PromptOption.RETRY, status_code=UserResponseStatusEnum.OKAY diff --git a/test_collections/sdk_tests/support/tests/yaml_tests/test_test_case.py b/test_collections/sdk_tests/support/tests/yaml_tests/test_test_case.py index ac05f74a..3b282fc9 100644 --- a/test_collections/sdk_tests/support/tests/yaml_tests/test_test_case.py +++ b/test_collections/sdk_tests/support/tests/yaml_tests/test_test_case.py @@ -23,12 +23,12 @@ from app.test_engine.logger import test_engine_logger from app.test_engine.models.manual_test_case import ManualVerificationTestStep from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType -from test_collections.sdk_tests.support.chip.test_case import TestError from test_collections.sdk_tests.support.models.matter_test_models import ( MatterTestStep, MatterTestType, ) from test_collections.sdk_tests.support.yaml_tests.models import YamlTestCase +from test_collections.sdk_tests.support.yaml_tests.models.chip_test import TestError from test_collections.sdk_tests.support.yaml_tests.models.test_case import ( YamlChipTestCase, YamlManualTestCase, diff --git a/test_collections/sdk_tests/support/tests/yaml_tests/test_test_suite.py b/test_collections/sdk_tests/support/tests/yaml_tests/test_test_suite.py index 4a13f815..92d7ca08 100644 --- a/test_collections/sdk_tests/support/tests/yaml_tests/test_test_suite.py +++ b/test_collections/sdk_tests/support/tests/yaml_tests/test_test_suite.py @@ -107,7 +107,8 @@ async def test_suite_setup_log_yaml_version() -> None: with mock.patch.object( target=test_engine_logger, attribute="info" ) as logger_info, mock.patch( - "test_collections.sdk_tests.support.chip.test_suite.ChipSuite.setup" + "test_collections.sdk_tests.support.yaml_tests.models.chip_suite.ChipSuite" + ".setup" ) as _: await suite_instance.setup() logger_info.assert_called() @@ -159,7 +160,8 @@ async def test_chip_suite_setup() -> None: "test_collections.sdk_tests.support.yaml_tests.models." "test_suite.YamlTestSuite.setup" ) as yaml_suite_setup, mock.patch( - "test_collections.sdk_tests.support.chip.test_suite.ChipSuite.setup" + "test_collections.sdk_tests.support.yaml_tests.models.chip_suite.ChipSuite" + ".setup" ) as chip_suite_setup: await suite_instance.setup() yaml_suite_setup.assert_called_once() diff --git a/test_collections/sdk_tests/support/chip/test_suite.py b/test_collections/sdk_tests/support/yaml_tests/models/chip_suite.py similarity index 94% rename from test_collections/sdk_tests/support/chip/test_suite.py rename to test_collections/sdk_tests/support/yaml_tests/models/chip_suite.py index a97b8212..85d778b8 100644 --- a/test_collections/sdk_tests/support/chip/test_suite.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/chip_suite.py @@ -28,7 +28,8 @@ from app.user_prompt_support.user_prompt_support import UserPromptSupport from test_collections.sdk_tests.support.chip import ChipTool from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType -from test_collections.sdk_tests.support.chip.test_case import PromptOption +from test_collections.sdk_tests.support.sdk_container import SDKContainer +from test_collections.sdk_tests.support.yaml_tests.models.chip_test import PromptOption CHIP_APP_PAIRING_CODE = "CHIP:SVR: Manual pairing code:" @@ -42,7 +43,8 @@ class DUTCommissioningError(Exception): class ChipSuite(TestSuite, UserPromptSupport): - chip_tool = ChipTool() + chip_tool = ChipTool(logger) + sdk_container: SDKContainer = SDKContainer(logger) border_router: Optional[ThreadBorderRouter] = None test_type: ChipTestType = ChipTestType.CHIP_TOOL __dut_commissioned_successfully: bool = False @@ -51,16 +53,17 @@ def __init__(self, test_suite_execution: TestSuiteExecution): super().__init__(test_suite_execution) async def setup(self) -> None: - logger.info("Setting up container") - # Use test engine logger to log all events to test run. - self.chip_tool.logger = logger + logger.info("Setting up SDK container") + await self.sdk_container.start() + + logger.info("Starting chip server") await self.chip_tool.start_server( self.test_type, self.config.dut_config.chip_use_paa_certs ) if len(self.pics.clusters) > 0: logger.info("Create PICS file for DUT") - self.chip_tool.set_pics(pics=self.pics, in_container=False) + self.chip_tool.set_pics(pics=self.pics) else: # Disable sending "-PICS" option when running chip server self.chip_tool.reset_pics_state() @@ -158,16 +161,13 @@ async def cleanup(self) -> None: if self.test_type == ChipTestType.CHIP_TOOL: logger.info("Unpairing chip_tool from device") await self.chip_tool.unpair() - # Need a better way to trigger unpair for chip-app. - # Currently the runner is being stopped automatically once - # the test completes. So we need to start it again to - # perform decommissioning elif self.test_type == ChipTestType.CHIP_APP: logger.info("Prompt user to perform decommissioning") await self.__prompt_user_to_perform_decommission() logger.info("Stopping SDK container") - await self.chip_tool.destroy_device() + self.sdk_container.destroy() + if self.border_router is not None: logger.info("Stopping border router container") self.border_router.destroy_device() diff --git a/test_collections/sdk_tests/support/chip/test_case.py b/test_collections/sdk_tests/support/yaml_tests/models/chip_test.py similarity index 98% rename from test_collections/sdk_tests/support/chip/test_case.py rename to test_collections/sdk_tests/support/yaml_tests/models/chip_test.py index dae5510e..1999d937 100644 --- a/test_collections/sdk_tests/support/chip/test_case.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/chip_test.py @@ -36,6 +36,7 @@ from app.user_prompt_support.user_prompt_support import UserPromptSupport from test_collections.sdk_tests.support.chip import ChipTool from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType +from test_collections.sdk_tests.support.sdk_container import SDKContainer CHIP_TOOL_DEFAULT_PROMPT_TIMEOUT_S = 60 # seconds OUTCOME_TIMEOUT_S = 60 * 10 # Seconds @@ -60,7 +61,7 @@ class TestError(Exception): class ChipTest(TestCase, UserPromptSupport, TestRunnerHooks, TestParserHooks): - chip_tool: ChipTool + chip_tool: ChipTool = ChipTool(test_engine_logger) chip_test_identifier: str test_type: ChipTestType @@ -221,11 +222,7 @@ async def setup(self) -> None: f"'{self.chip_test_identifier}'. Expected non-empty string." ) - self.chip_tool = ChipTool() - - # Use test engine logger to log all events to test run. - self.chip_tool.logger = test_engine_logger - if not self.chip_tool.is_running(): + if not SDKContainer().is_running(): raise TestError("Unable to execute test as SDK container is not available") async def execute(self) -> None: diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py index 9f6094c0..eadfee43 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py @@ -25,14 +25,14 @@ ) from app.test_engine.models.test_case import CUSTOM_TEST_IDENTIFIER from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType -from test_collections.sdk_tests.support.chip.test_case import ( - ChipManualPromptTest, - ChipTest, -) from test_collections.sdk_tests.support.models.matter_test_models import ( MatterTestStep, MatterTestType, ) +from test_collections.sdk_tests.support.yaml_tests.models.chip_test import ( + ChipManualPromptTest, + ChipTest, +) from .yaml_test_models import YamlTest diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_suite.py b/test_collections/sdk_tests/support/yaml_tests/models/test_suite.py index 9d41da5b..c815a69e 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_suite.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_suite.py @@ -19,7 +19,7 @@ from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestSuite from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType -from test_collections.sdk_tests.support.chip.test_suite import ChipSuite +from test_collections.sdk_tests.support.yaml_tests.models.chip_suite import ChipSuite class YamlTestSuiteFactoryError(Exception): diff --git a/test_collections/tool_unit_tests/test_suite_chip/tctr_chip_log_parsing/tctr_chip_log_parsing.py b/test_collections/tool_unit_tests/test_suite_chip/tctr_chip_log_parsing/tctr_chip_log_parsing.py index aa520ca3..626b11ae 100644 --- a/test_collections/tool_unit_tests/test_suite_chip/tctr_chip_log_parsing/tctr_chip_log_parsing.py +++ b/test_collections/tool_unit_tests/test_suite_chip/tctr_chip_log_parsing/tctr_chip_log_parsing.py @@ -18,7 +18,7 @@ from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestStep from test_collections.sdk_tests.support.chip.chip_tool import ChipTestType -from test_collections.sdk_tests.support.chip.test_case import ChipTest +from test_collections.sdk_tests.support.yaml_tests.models.chip_test import ChipTest class TCTRChipLogParsing(ChipTest):