From c50f2d8dc053eadf9d34b9eacc70484d23532450 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <116586593+rquidute@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:48:12 -0300 Subject: [PATCH] Add python tests model classes (#19) * Created model classes for SDK python tests * Added parser implementation to python test * Changed init from sdk_tests * Minor changes * Disable support to list python test cases * Reverted minor changes * Added Unit tests * Minor change * Fix merge conflict * Fix CI issues * Fix mypy and flake violations * Fix mypy and flake violations * Fix mypy and flake violations * Fixed unit tests * Add iniial code for Python Test runner * Fix code violation and add code to chip_tool for python runner * Fix code violation * Changes after code review * Removed unsued code * Few tweeks based on observation running end to end. * Created top level classes for test models * Fix code violation * Code changes after review * Code changes after code review * Putting back .version_information * Putting back .version_information * Fixed mypy violations * Removed unused imports * adding rpc client code and mounting new volumes on sdk container * Fixing sdk rpc file path * Code changed after code review * Update app/tests/python_tests/test_python_test_suite.py Co-authored-by: Carolina Lopes <116589288+ccruzagralopes@users.noreply.github.com> * Code changes after review * Remove whitespace from a blank line * Add support for PICS in python tests * Update some unit tests * Get python test parameters from project config * Fix Unit tests * Updated unit tests * Removed unused test files * Code updated due to code review * Make PICS optional for python tests * Added SDKPythonTestResult class in order to handle the python test responses * Create SDKPythonTestResultBase and make subclass for each result * Make default attribute type set in SDKPythonTestResultBase subclasses --------- Co-authored-by: Gladyston Franca Co-authored-by: Carolina Lopes <116589288+ccruzagralopes@users.noreply.github.com> Co-authored-by: Carolina Lopes --- app/chip_tool/chip_tool.py | 79 +++++-- app/chip_tool/test_suite.py | 4 +- app/tests/chip_tool/test_chip_tool.py | 41 ++-- app/tests/python_tests/test_python_folder.py | 58 +++++ app/tests/python_tests/test_python_parser.py | 92 ++++++++ .../test_python_script/TC_Sample.py | 55 +++++ .../python_tests/test_python_test_case.py | 205 ++++++++++++++++++ .../test_python_test_declarations.py | 61 ++++++ .../python_tests/test_python_test_suite.py | 167 ++++++++++++++ .../test_sdk_python_collection.py | 68 ++++++ .../yaml_tests/test_sdk_yaml_collection.py | 53 +++-- app/tests/yaml_tests/test_test_case.py | 72 +++--- app/tests/yaml_tests/test_yaml_folder.py | 20 +- app/tests/yaml_tests/test_yaml_parser.py | 34 +-- app/tests/yaml_tests/test_yamls/.version | 1 - cspell.json | 4 +- test_collections/sdk_tests/__init__.py | 1 + .../support/models/matter_test_models.py | 49 +++++ .../sdk_test_folder.py} | 17 +- .../support/python_testing/__init__.py | 22 ++ .../support/python_testing/models/__init__.py | 17 ++ .../models/python_test_models.py | 30 +++ .../models/python_test_parser.py | 130 +++++++++++ .../models/python_testing_hooks_proxy.py | 181 ++++++++++++++++ .../models/rpc_client/test_harness_client.py | 54 +++++ .../python_testing/models/test_case.py | 200 +++++++++++++++++ .../models/test_declarations.py | 65 ++++++ .../python_testing/models/test_suite.py | 81 +++++++ .../python_testing/sdk_python_tests.py | 110 ++++++++++ .../support/yaml_tests/models/test_case.py | 16 +- .../yaml_tests/models/test_declarations.py | 9 +- .../yaml_tests/models/yaml_test_models.py | 33 +-- .../yaml_tests/models/yaml_test_parser.py | 14 +- .../support/yaml_tests/sdk_yaml_tests.py | 20 +- 34 files changed, 1862 insertions(+), 201 deletions(-) create mode 100644 app/tests/python_tests/test_python_folder.py create mode 100644 app/tests/python_tests/test_python_parser.py create mode 100644 app/tests/python_tests/test_python_script/TC_Sample.py create mode 100644 app/tests/python_tests/test_python_test_case.py create mode 100644 app/tests/python_tests/test_python_test_declarations.py create mode 100644 app/tests/python_tests/test_python_test_suite.py create mode 100644 app/tests/python_tests/test_sdk_python_collection.py delete mode 100644 app/tests/yaml_tests/test_yamls/.version create mode 100644 test_collections/sdk_tests/support/models/matter_test_models.py rename test_collections/sdk_tests/support/{yaml_tests/models/yaml_test_folder.py => models/sdk_test_folder.py} (73%) create mode 100644 test_collections/sdk_tests/support/python_testing/__init__.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/__init__.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_models.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_parser.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_case.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_declarations.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_suite.py create mode 100644 test_collections/sdk_tests/support/python_testing/sdk_python_tests.py diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 48ff32fb..2e3970bd 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -107,6 +107,19 @@ XML_SPEC_DEFINITION_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path( "sdk_runner/specifications/chip/" ) +# Python Testing Folder +LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections" +LOCAL_PYTHON_TESTING_PATH = Path( + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing/scripts/sdk" +) +DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" + +# RPC Client Running on SDK Container +LOCAL_RPC_PYTHON_TESTING_PATH = Path( + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/support/python_testing/models/rpc_client/" + "test_harness_client.py" +) +DOCKER_RPC_PYTHON_TESTING_PATH = "/root/python_testing/test_harness_client.py" # Docker Network @@ -168,6 +181,14 @@ class ChipTool(metaclass=Singleton): "bind": DOCKER_CREDENTIALS_DEVELOPMENT_PATH, "mode": "ro", }, + LOCAL_PYTHON_TESTING_PATH: { + "bind": DOCKER_PYTHON_TESTING_PATH, + "mode": "rw", + }, + LOCAL_RPC_PYTHON_TESTING_PATH: { + "bind": DOCKER_RPC_PYTHON_TESTING_PATH, + "mode": "rw", + }, }, } @@ -203,6 +224,10 @@ def __init__( specifications_paths, self.pseudo_clusters ) + @property + def pics_file_created(self) -> bool: + return self.__pics_file_created + @property def node_id(self) -> int: """Node id is used to reference DUT during testing. @@ -347,14 +372,11 @@ def __get_gateway_ip(self) -> str: .get(DOCKER_GATEWAY_KEY, "") ) - async def start_container( - self, test_type: ChipToolTestType, use_paa_certs: bool = False - ) -> None: - """Creates the chip-tool container. - - Returns only when the container is created and all chip-tool services start. + async def start_container(self) -> None: + """ + Creates the chip-tool container without any server running + (ChipTool or ChipApp). """ - if self.is_running(): self.logger.info( "chip-tool container already running, no need to start a new container" @@ -363,30 +385,34 @@ async def start_container( # Ensure there's no existing container running using the same name. self.__destroy_existing_container() - # Async return when the container is running self.__chip_tool_container = await container_manager.create_container( self.image_tag, self.run_parameters ) - # Reset any previous states self.__last_exec_id = None self.__pics_file_created = False - # Generate new random node id for the DUT self.__reset_node_id() self.logger.info(f"New Node Id generated: {hex(self.node_id)}") - self.logger.info( f""" chip-tool started: {self.container_name} with configuration: {self.run_parameters} """ ) - # Server started is false after spinning up a new container. self.__server_started = False + async def start_server( + self, test_type: ChipToolTestType, use_paa_certs: bool = False + ) -> None: + """Creates the chip-tool container. + + Returns only when the container is created and all chip-tool services start. + """ + await self.start_container() + web_socket_config = WebSocketRunnerConfig() web_socket_config.server_address = self.__get_gateway_ip() self.__test_harness_runner = WebSocketRunner(config=web_socket_config) @@ -661,32 +687,37 @@ def __trace_file_params(self, topic: str) -> str: path = Path(DOCKER_LOGS_PATH) / filename return f'--trace_file "{path}" --trace_decode 1' - def set_pics(self, pics: PICS) -> None: - """Sends command to chip tool to create pics file inside the container. + def set_pics(self, pics: PICS, in_container: bool) -> None: + """Sends command to create pics file. Args: pics (PICS): PICS that contains all the pics codes + in_container (bool): Whether the file should be created in the SDK container + or not. YAML tests run directly in the backend and + python tests run in the SDK container. Raises: - ChipToolNotRunning: Raises exception if chip tool is not running. PICSError: If creating PICS file inside the container fails. - """ # List of default PICS which needs to set specifically in TH are added here. # These PICS are applicable for CI / Chip tool testing purposes only. # These PICS are unknown / not visible to external users. pics_codes = self.__pics_file_content(pics) + "\n".join(DEFAULT_PICS) - cmd = f"{SHELL_PATH} {SHELL_OPTION} " - cmd = cmd + f"\"{ECHO_COMMAND} '{pics_codes}\n' > {PICS_FILE_PATH}\"" - self.logger.info(f"Sending command: {cmd}") - result = subprocess.run(cmd, shell=True) - # When streaming logs, the exit code is not directly available. - # By storing the execution id, the exit code can be fetched from docker later. - self.__last_exec_id = str(result.returncode) + prefix = f"{SHELL_PATH} {SHELL_OPTION}" + cmd = f"\"{ECHO_COMMAND} '{pics_codes}' > {PICS_FILE_PATH}\"" + + if in_container: + exec_result = self.send_command(cmd, prefix=prefix) + success = exec_result.exit_code == 0 + else: + full_cmd = f"{prefix} {cmd}" + self.logger.info(f"Sending command: {full_cmd}") + result = subprocess.run(full_cmd, shell=True) + success = result.returncode == 0 - if result.returncode != 0: + if not success: raise PICSError("Creating PICS file failed") self.__pics_file_created = True diff --git a/app/chip_tool/test_suite.py b/app/chip_tool/test_suite.py index 9bc1c62f..57f0c1e5 100644 --- a/app/chip_tool/test_suite.py +++ b/app/chip_tool/test_suite.py @@ -54,13 +54,13 @@ async def setup(self) -> None: logger.info("Setting up chip_tool") # Use test engine logger to log all events to test run. self.chip_tool.logger = logger - await self.chip_tool.start_container( + await self.chip_tool.start_server( self.test_type, self.config.dut_config.chip_tool_use_paa_certs ) if len(self.pics.clusters) > 0: logger.info("Create PICS file for DUT") - self.chip_tool.set_pics(pics=self.pics) + self.chip_tool.set_pics(pics=self.pics, in_container=False) else: # Disable sending "-PICS" option when running chip-tool self.chip_tool.reset_pics_state() diff --git a/app/tests/chip_tool/test_chip_tool.py b/app/tests/chip_tool/test_chip_tool.py index aa45635d..bb65e96c 100644 --- a/app/tests/chip_tool/test_chip_tool.py +++ b/app/tests/chip_tool/test_chip_tool.py @@ -69,7 +69,7 @@ async def test_start_container() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) mock_start_chip_server.assert_awaited_once_with(test_type, False) @@ -98,7 +98,7 @@ async def test_start_container_using_paa_certs() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type, use_paa_certs=True) + await chip_tool.start_server(test_type, use_paa_certs=True) mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) mock_start_chip_server.assert_awaited_once_with(test_type, True) @@ -111,7 +111,6 @@ async def test_start_container_using_paa_certs() -> None: @pytest.mark.asyncio async def test_not_start_container_when_running() -> None: chip_tool = ChipTool() - test_type = ChipToolTestType.CHIP_TOOL with mock.patch.object( target=chip_tool, attribute="is_running", return_value=True @@ -120,7 +119,7 @@ async def test_not_start_container_when_running() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type) + await chip_tool.start_container() mock_create_container.assert_not_called() mock_start_chip_server.assert_not_called() @@ -359,8 +358,8 @@ async def test_destroy_container_running() -> None: ), mock.patch.object( target=chip_tool, attribute="start_chip_server" ): - await chip_tool.start_container(test_type) - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) + await chip_tool.start_server(test_type) assert chip_tool._ChipTool__chip_tool_container is not None @@ -403,7 +402,7 @@ async def test_destroy_container_once() -> None: ), mock.patch.object( target=chip_tool, attribute="start_chip_server" ): - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.destroy_device() await chip_tool.destroy_device() @@ -432,7 +431,7 @@ async def test_set_pics() -> None: "PICS_USER_PROMPT=1" ) expected_command = ( - f"{SHELL_PATH} {SHELL_OPTION} \"echo '{expected_pics_data}\n' " + f"{SHELL_PATH} {SHELL_OPTION} \"echo '{expected_pics_data}' " f'> {PICS_FILE_PATH}"' ) @@ -452,9 +451,9 @@ async def test_set_pics() -> None: target="app.chip_tool.chip_tool.subprocess.run", return_value=CompletedProcess(expected_command, 0), ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) - chip_tool.set_pics(pics) + chip_tool.set_pics(pics, in_container=False) mock_run.assert_called_once_with(expected_command, shell=True) assert chip_tool._ChipTool__pics_file_created is True @@ -473,7 +472,7 @@ def test_set_pics_with_error() -> None: target="app.chip_tool.chip_tool.subprocess.run", return_value=CompletedProcess("", 1), ), pytest.raises(PICSError): - chip_tool.set_pics(pics) + chip_tool.set_pics(pics, in_container=False) assert chip_tool._ChipTool__pics_file_created is False # clean up: @@ -505,7 +504,7 @@ async def test_send_command_default_prefix() -> None: target="app.chip_tool.chip_tool.exec_run_in_container", return_value=mock_result, ) as mock_exec_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) @@ -548,7 +547,7 @@ async def test_send_command_custom_prefix() -> None: target="app.chip_tool.chip_tool.exec_run_in_container", return_value=mock_result, ) as mock_exec_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) @@ -592,7 +591,7 @@ async def test_run_test_default_config() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -649,7 +648,7 @@ async def test_run_test_custom_timeout() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -700,7 +699,7 @@ async def test_run_test_with_custom_parameter() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -752,7 +751,7 @@ async def test_run_test_with_endpoint_parameter() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -803,7 +802,7 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -862,7 +861,7 @@ async def test_pairing_on_network_command_params() -> None: attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send on-network pairing command result = await chip_tool.pairing_on_network( @@ -913,7 +912,7 @@ async def test_pairing_ble_wifi_command_params() -> None: attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send BLE-WIFI pairing command result = await chip_tool.pairing_ble_wifi( @@ -968,7 +967,7 @@ async def test_pairing_ble_thread_command_params() -> None: return_value='{"results": []}', # '{ "results": [{ "error": "FAILURE" }] ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send BLE-THREAD pairing command result = await chip_tool.pairing_ble_thread( diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py new file mode 100644 index 00000000..3636f8bf --- /dev/null +++ b/app/tests/python_tests/test_python_folder.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder + +test_python_path = Path("/test/python") + + +def test_python_folder_version() -> None: + version_file_content = "python_test_version" + + # We mock open to read version_file_content and Path exists to ignore that we're + # testing with a fake path + with mock.patch( + "test_collections.sdk_tests.support.models.sdk_test_folder.open", + new=mock.mock_open(read_data=version_file_content), + ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: + python_test_folder = SDKTestFolder(test_python_path) + + assert python_test_folder.version == version_file_content + + +def test_python_folder_version_missing() -> None: + expected_version = "Unknown" + with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: + python_folder = SDKTestFolder(test_python_path) + assert python_folder.version == expected_version + + +def test_python_folder_filename_pattern() -> None: + """Test SDKTestFolder will search for files with filename pattern.""" + with mock.patch.object(target=Path, attribute="glob") as path_glob: + # Default file_name_pattern: * + python_folder = SDKTestFolder(test_python_path) + _ = python_folder.file_paths(extension=".py") + path_glob.assert_called_once_with("*.py") + + path_glob.reset_mock() + pattern = "TC_*" + python_test_folder = SDKTestFolder(test_python_path, filename_pattern=pattern) + _ = python_test_folder.file_paths(extension=".py") + path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py new file mode 100644 index 00000000..5f128891 --- /dev/null +++ b/app/tests/python_tests/test_python_parser.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +# Ignore flake8 check for this file +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_parser import ( + PythonParserException, + parse_python_test, +) + +sample_invalid_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + def test_steps_TC_Sample(self): + print("Test execution") + +""" + +sample_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + def test_TC_Sample(self): + print("Test execution") + + def pics_TC_Sample(self): + pics = ["MCORE.ROLE.COMMISSIONEE"] + +""" + + +def test_python_file_parser_throws_pythonparserexception() -> None: + file_path = Path("/test/file.py") + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser." + "open", + new=mock.mock_open(read_data=sample_invalid_python_file_content), + ): + try: + parse_python_test(file_path) + except PythonParserException as e: + assert "/test/file.py must have a class named file" == str(e) + + +def test_python_file_parser() -> None: + file_path = Path("/test/TC_Sample.py") + + # We mock builtin `open` method to read sample python file content, + # to avoid having to load a real file. + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser." + "open", + new=mock.mock_open(read_data=sample_python_file_content), + ) as file_open: + test = parse_python_test(file_path) + + file_open.assert_called_once_with(file_path, "r") + assert test.path == file_path diff --git a/app/tests/python_tests/test_python_script/TC_Sample.py b/app/tests/python_tests/test_python_script/TC_Sample.py new file mode 100644 index 00000000..4c29514b --- /dev/null +++ b/app/tests/python_tests/test_python_script/TC_Sample.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# type: ignore +# Ignore mypy type check for this file +# flake8: noqa +# Ignore flake8 check for this file +""" +This is just a sample test case that should come from SDK. +It should not compile or run. +""" + + +class TC_Sample(MatterBaseTest): + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + def test_TC_Sample(self): + print("Test execution") + + def pics_TC_Sample(self): + pics = ["PICS"] diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py new file mode 100644 index 00000000..78a303d8 --- /dev/null +++ b/app/tests/python_tests/test_python_test_case.py @@ -0,0 +1,205 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +# Ignore flake8 check for this file +from pathlib import Path +from typing import Any, Optional, Type +from unittest import mock + +import pytest + +from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.test_case import TestError +from app.models.test_case_execution import TestCaseExecution +from app.test_engine.logger import test_engine_logger +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) +from test_collections.sdk_tests.support.python_testing.models import PythonTestCase +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, +) + + +def python_test_instance( + name: str = "TC-Test-Python", + description: str = "Test Python Description", + PICS: set[str] = {"PICS.A", "PICS.B"}, + config: dict[str, Any] = { + "param1": "value1", + "param2": {"type": "config_type", "defaultValue": "value2"}, + }, + steps: list[MatterTestStep] = [], + type: MatterTestType = MatterTestType.AUTOMATED, + path: Optional[Path] = None, +) -> PythonTest: + return PythonTest( + name=name, + PICS=PICS, + config=config, + steps=steps, + type=type, + path=path, + description=description, + ) + + +def test_python_test_name() -> None: + """Test that test name is set as description in metadata.""" + name = "Another Test Name" + description = "Another Test Name Description" + test = python_test_instance(name=name, description=description) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["title"] == name + assert case_class.metadata["description"] == description + + +def test_python_test_python_version() -> None: + """Test that test case python version is set correctly in class factory.""" + test = python_test_instance() + python_test_version = "best_version" + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=python_test_version + ) + assert case_class.python_test_version == python_test_version + + +def test_python_test_python() -> None: + """Test that test python_test property is as expected in subclass of PythonTestCase.""" + test = python_test_instance() + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.python_test is test + + +def test_python_test_case_class_pics() -> None: + """Test that the PICS of the python test is available in the class method PICS on + TestCase.""" + test_PICS = set(["PICS.D", "PICS.C"]) + test = python_test_instance(PICS=test_PICS) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.pics() == test_PICS + + +def test_class_factory_test_public_id() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1' in python test name. + And set it as public_id in metadata""" + test_data = [ + {"name": "TC-AB-1.2", "public_id": "TC-AB-1.2"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["public_id"] == data["public_id"] + + +def test_class_factory_test_class_name() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1', convert it to + a safe class name, eg TC_XX_1_1""" + test_data = [ + {"name": "TC-AB-1.2", "class_name": "TC_AB_1_2"}, + {"name": "TC-CD-3.4", "class_name": "TC_CD_3_4"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.__name__ == data["class_name"] + + +@pytest.mark.asyncio +async def test_python_version_logging() -> None: + """Test that all Python tests will log Python test version to test_engine_logger. + + Note that since `chip-tool` is not setup, we except the TestError raised. + """ + for type in list(MatterTestType): + test = python_test_instance(type=type) + test_python_version = "PythonVersionTest" + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=test_python_version + ) + instance = case_class(TestCaseExecution()) + + with mock.patch.object( + target=test_engine_logger, attribute="info" + ) as logger_info: + try: + await instance.setup() + except TestError: + pass + logger_info.assert_called() + logger_info.assert_any_call("Test Setup") + + +def test_normal_steps_for_python_tests() -> None: + """Test that python tests include enabled steps.""" + for type in list(MatterTestType): + test_step = MatterTestStep(label="Step1") + test = python_test_instance(type=type, steps=[test_step]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + # Assert normal step is present + assert len(instance.test_steps) >= 1 + assert any(s.name == test_step.label for s in instance.test_steps) + + +def test_multiple_steps_for_python_tests() -> None: + """Test that python tests multiple enabled steps are all included.""" + for type in list(MatterTestType): + test_step = MatterTestStep(label="StepN") + no_steps = 5 + test = python_test_instance(type=type, steps=([test_step] * no_steps)) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + + # Assert all steps from python test are added + assert len(instance.test_steps) >= no_steps + steps_from_python = [ + s for s in instance.test_steps if s.name == test_step.label + ] + assert len(steps_from_python) == no_steps + + +@pytest.mark.asyncio +async def test_setup_super_error_handling() -> None: + # ignore requirement to create_tests on init + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ) as _: + test = PythonTestCase(TestCaseExecution()) + test.python_test_version = "some version" + # Assert this doesn't raise an exception + await test.setup() diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py new file mode 100644 index 00000000..57d660c4 --- /dev/null +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +# Ignore flake8 check for this file +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, +) +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonSuiteDeclaration, +) +from test_collections.sdk_tests.support.python_testing.models.test_suite import ( + SuiteType, +) + + +def test_python_suite_declaration() -> None: + name = "TestName" + type = SuiteType.AUTOMATED + version = "SomeVersionStr" + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestSuiteDeclaration.__init__" + ) as declaration_init: + PythonSuiteDeclaration(name=name, suite_type=type, version=version) + class_factory.assert_called_once_with( + name=name, suite_type=type, python_test_version=version + ) + declaration_init.assert_called_once() + + +def test_python_case_declaration() -> None: + test = PythonTest( + name="TestTest", description="TestTest description", config={}, steps=[] + ) + version = "SomeVersionStr" + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestCaseDeclaration.__init__" + ) as declaration_init: + PythonCaseDeclaration(test=test, python_test_version=version) + class_factory.assert_called_once_with(test=test, python_test_version=version) + declaration_init.assert_called_once() diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py new file mode 100644 index 00000000..199775ef --- /dev/null +++ b/app/tests/python_tests/test_python_test_suite.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +# Ignore flake8 check for this file +from typing import Type +from unittest import mock + +import pytest + +from app.chip_tool.chip_tool import ChipTool +from app.models.test_suite_execution import TestSuiteExecution +from app.schemas import PICS +from app.test_engine.logger import test_engine_logger +from app.tests.utils.test_pics_data import create_random_pics +from test_collections.sdk_tests.support.python_testing.models.test_suite import ( + PythonTestSuite, + SuiteType, +) + + +def test_python_suite_class_factory_name() -> None: + """Test that test suite name is set.""" + name = "AnotherTestSuite" + + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, name=name, python_test_version="version" + ) + + assert suite_class.__name__ == name + assert suite_class.public_id() == name + assert suite_class.metadata["title"] == name + assert suite_class.metadata["description"] == name + + +def test_python_test_suite_python_version() -> None: + """Test that test suite python version is set correctly in class factory.""" + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, + name="SomeSuite", + python_test_version=python_test_version, + ) + + assert suite_class.python_test_version == python_test_version + + +@pytest.mark.asyncio +async def test_suite_setup_log_python_version() -> None: + """Test that test suite python version is logged to test engine logger in setup.""" + chip_tool: ChipTool = ChipTool() + + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch.object( + target=test_engine_logger, attribute="info" + ) as logger_info, mock.patch.object( + target=chip_tool, attribute="start_container" + ), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=PICS, + ): + await suite_instance.setup() + logger_info.assert_called() + logger_info.assert_any_call(f"Python Test Version: {python_test_version}") + + +@pytest.mark.asyncio +async def test_suite_setup_without_pics() -> None: + chip_tool: ChipTool = ChipTool() + + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=PICS, + ), mock.patch.object( + target=chip_tool, attribute="set_pics" + ) as mock_set_pics, mock.patch.object( + target=chip_tool, attribute="reset_pics_state" + ) as mock_reset_pics_state: + await suite_instance.setup() + + mock_set_pics.assert_not_called() + mock_reset_pics_state.assert_called_once() + + +@pytest.mark.asyncio +async def test_suite_setup_with_pics() -> None: + chip_tool: ChipTool = ChipTool() + + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=create_random_pics, + ), mock.patch.object( + target=chip_tool, attribute="set_pics" + ) as mock_set_pics, mock.patch.object( + target=chip_tool, attribute="reset_pics_state" + ) as mock_reset_pics_state: + await suite_instance.setup() + + mock_set_pics.assert_called_once() + mock_reset_pics_state.assert_not_called() + + +@pytest.mark.asyncio +async def test_chip_tool_suite_setup() -> None: + """Test that PythonTestSuite.setup is called when PythonChipToolsSuite.setup is called. + We do this as PythonChipToolsSuite inherits from PythonTestSuite.""" + + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, + name="SomeSuite", + python_test_version="Some version", + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.setup" + ) as python_suite_setup: + await suite_instance.setup() + python_suite_setup.assert_called_once() diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py new file mode 100644 index 00000000..7e157cae --- /dev/null +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +# Ignore flake8 check for this file +from pathlib import Path +from unittest import mock + +import pytest + +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + MatterTestType, +) +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonCollectionDeclaration, +) +from test_collections.sdk_tests.support.python_testing.sdk_python_tests import ( + sdk_python_test_collection, +) + + +@pytest.fixture +def python_test_collection() -> PythonCollectionDeclaration: + test_sdk_python_path = Path(__file__).parent / "test_python_script" + with mock.patch.object(Path, "exists", return_value=True), mock.patch( + "test_collections.sdk_tests.support.models.sdk_test_folder.open", + new=mock.mock_open(read_data="unit-test-python-version"), + ): + folder = SDKTestFolder(path=test_sdk_python_path, filename_pattern="TC_*") + return sdk_python_test_collection(folder) + + +def test_sdk_python_test_collection( + python_test_collection: PythonCollectionDeclaration, +) -> None: + assert python_test_collection.name == "SDK Python Tests" + assert len(python_test_collection.test_suites.keys()) == 1 + assert python_test_collection.python_test_version == "unit-test-python-version" + + +def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: + expected_automated_test_cases = 1 + + # Assert automated tests cases + assert "Python Testing Suite" in python_test_collection.test_suites.keys() + automated_suite = python_test_collection.test_suites["Python Testing Suite"] + assert len(automated_suite.test_cases) == expected_automated_test_cases + + type_count = dict.fromkeys(MatterTestType, 0) + for test_case in automated_suite.test_cases.values(): + assert isinstance(test_case, PythonCaseDeclaration) + type_count[test_case.test_type] += 1 + + assert type_count[MatterTestType.AUTOMATED] == expected_automated_test_cases diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index 0519cefb..3f67d063 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -14,43 +14,40 @@ # limitations under the License. # from pathlib import Path +from unittest import mock import pytest -from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.yaml_tests.models.test_declarations import ( YamlCaseDeclaration, YamlCollectionDeclaration, ) -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder import ( - YamlTestFolder, -) -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( - YamlTestType, -) from test_collections.sdk_tests.support.yaml_tests.sdk_yaml_tests import ( sdk_yaml_test_collection, ) -VERSION_FILE_FILENAME = ".version" -VERSION_FILE_PATH = SDK_CHECKOUT_PATH - @pytest.fixture def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" - folder = YamlTestFolder(path=test_sdk_yaml_path, filename_pattern="UnitTest_TC_*") - return sdk_yaml_test_collection(folder) - - -def test_sdk_yaml_collection(yaml_collection: YamlCollectionDeclaration) -> None: + with mock.patch.object(Path, "exists", return_value=True), mock.patch( + "test_collections.sdk_tests.support.models.sdk_test_folder.open", + new=mock.mock_open(read_data="unit-test-yaml-version"), + ): + folder = SDKTestFolder( + path=test_sdk_yaml_path, filename_pattern="UnitTest_TC_*" + ) + return sdk_yaml_test_collection(folder) + + +def test_sdk_yaml_collection( + yaml_collection: YamlCollectionDeclaration, +) -> None: assert yaml_collection.name == "SDK YAML Tests" assert len(yaml_collection.test_suites.keys()) == 3 - - # test version number - test_sdk_yaml_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME - with open(test_sdk_yaml_version_path, "r") as version_file: - assert yaml_collection.yaml_version == version_file.read().rstrip() + assert yaml_collection.yaml_version == "unit-test-yaml-version" def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -61,7 +58,7 @@ def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(manual_suite.test_cases) == expected_manual_test_cases for test_case in manual_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == YamlTestType.MANUAL + assert test_case.test_type == MatterTestType.MANUAL def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -78,15 +75,17 @@ def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: == expected_automated_test_cases + expected_semi_automated_test_cases ) - type_count = dict.fromkeys(YamlTestType, 0) + type_count = dict.fromkeys(MatterTestType, 0) for test_case in automated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) type_count[test_case.test_type] += 1 - assert type_count[YamlTestType.AUTOMATED] == expected_automated_test_cases - assert type_count[YamlTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases - assert type_count[YamlTestType.SIMULATED] == expected_simulated_test_cases - assert type_count[YamlTestType.MANUAL] == expected_manual_test_cases + assert type_count[MatterTestType.AUTOMATED] == expected_automated_test_cases + assert ( + type_count[MatterTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases + ) + assert type_count[MatterTestType.SIMULATED] == expected_simulated_test_cases + assert type_count[MatterTestType.MANUAL] == expected_manual_test_cases def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -97,4 +96,4 @@ def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(simulated_suite.test_cases) == expected_simulated_test_cases for test_case in simulated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == YamlTestType.SIMULATED + assert test_case.test_type == MatterTestType.SIMULATED diff --git a/app/tests/yaml_tests/test_test_case.py b/app/tests/yaml_tests/test_test_case.py index a4b03998..9b77ec22 100644 --- a/app/tests/yaml_tests/test_test_case.py +++ b/app/tests/yaml_tests/test_test_case.py @@ -24,6 +24,10 @@ from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger from app.test_engine.models.manual_test_case import ManualVerificationTestStep +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) from test_collections.sdk_tests.support.yaml_tests.models import YamlTestCase from test_collections.sdk_tests.support.yaml_tests.models.test_case import ( YamlChipToolTestCase, @@ -33,8 +37,6 @@ ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( YamlTest, - YamlTestStep, - YamlTestType, ) @@ -45,8 +47,8 @@ def yaml_test_instance( "param1": "value1", "param2": {"type": "config_type", "defaultValue": "value2"}, }, - tests: list[YamlTestStep] = [], - type: YamlTestType = YamlTestType.AUTOMATED, + tests: list[MatterTestStep] = [], + type: MatterTestType = MatterTestType.AUTOMATED, path: Optional[Path] = None, ) -> YamlTest: return YamlTest( @@ -128,7 +130,7 @@ def test_yaml_test_case_class_default_test_parameters() -> None: def test_manual_test_case_class_factory_subclass_mapping() -> None: """Test Manual tests are created as a subclass of YamlManualTestCase.""" - test = yaml_test_instance(type=YamlTestType.MANUAL) + test = yaml_test_instance(type=MatterTestType.MANUAL) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -138,7 +140,7 @@ def test_manual_test_case_class_factory_subclass_mapping() -> None: def test_automated_test_case_class_factory_subclass_mapping() -> None: """Test Automated tests are created as a subclass of YamlChipToolTestCase.""" - test = yaml_test_instance(type=YamlTestType.AUTOMATED) + test = yaml_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -148,7 +150,7 @@ def test_automated_test_case_class_factory_subclass_mapping() -> None: def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSemiAutomatedChipToolTestCase.""" - test = yaml_test_instance(type=YamlTestType.SEMI_AUTOMATED) + test = yaml_test_instance(type=MatterTestType.SEMI_AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -158,7 +160,7 @@ def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: def test_simulated_test_case_class_factory_subclass_mapping() -> None: """Test Simulated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -168,7 +170,7 @@ def test_simulated_test_case_class_factory_subclass_mapping() -> None: def test_incomplete_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -219,13 +221,13 @@ def test_class_factory_test_title_semi_automated() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and append (Semi-automated) when appropriate.""" - for type in list(YamlTestType): + for type in list(MatterTestType): test = yaml_test_instance(type=type, name="TC-AB-1.2") case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) - if type == YamlTestType.SEMI_AUTOMATED: + if type == MatterTestType.SEMI_AUTOMATED: assert test.name in case_class.metadata["title"] assert "(Semi-automated)" in case_class.metadata["title"] else: @@ -235,12 +237,12 @@ def test_class_factory_test_title_semi_automated() -> None: def test_class_factory_test_title_steps_disabled() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and (Steps Disabled) when some but not all steps are disabled.""" - disabled_step = YamlTestStep(label="Step1", disabled=True) - enabled_step = YamlTestStep(label="Step2", disabled=False) + disabled_step = MatterTestStep(label="Step1", disabled=True) + enabled_step = MatterTestStep(label="Step2", disabled=False) for type in [ - YamlTestType.AUTOMATED, - YamlTestType.SEMI_AUTOMATED, - YamlTestType.SIMULATED, + MatterTestType.AUTOMATED, + MatterTestType.SEMI_AUTOMATED, + MatterTestType.SIMULATED, ]: test = yaml_test_instance( type=type, name="TC-AB-1.2", tests=[disabled_step, enabled_step] @@ -261,10 +263,10 @@ def test_steps_in_manual_yaml_test_case() -> None: - verification details are optional but passed when present in yaml """ steps = [ - YamlTestStep(label="Step1"), - YamlTestStep(label="Step2Verification", verification="Verification String"), + MatterTestStep(label="Step1"), + MatterTestStep(label="Step2Verification", verification="Verification String"), ] - test = yaml_test_instance(type=YamlTestType.MANUAL, tests=steps) + test = yaml_test_instance(type=MatterTestType.MANUAL, tests=steps) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -281,7 +283,7 @@ def test_steps_in_manual_yaml_test_case() -> None: def test_test_type_for_automated_tests() -> None: """Test that automated tests are set to use chip-tool""" - test = yaml_test_instance(type=YamlTestType.AUTOMATED) + test = yaml_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -292,7 +294,7 @@ def test_test_type_for_automated_tests() -> None: def test_test_type_for_simulated_tests() -> None: """Test that simulated tests are set to use chip-app""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -307,7 +309,7 @@ async def test_yaml_version_logging() -> None: Note that since `chip-tool` is not setup, we except the TestError raised. """ - for type in list(YamlTestType): + for type in list(MatterTestType): test = yaml_test_instance(type=type) test_yaml_version = "YamlVersionTest" case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -327,7 +329,7 @@ async def test_yaml_version_logging() -> None: def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: - test = yaml_test_instance(type=YamlTestType.AUTOMATED, tests=[]) + test = yaml_test_instance(type=MatterTestType.AUTOMATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -337,7 +339,7 @@ def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: - test = yaml_test_instance(type=YamlTestType.SIMULATED, tests=[]) + test = yaml_test_instance(type=MatterTestType.SIMULATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -348,10 +350,10 @@ def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: def test_disabled_steps_for_non_manual_test() -> None: """Test that non-manual tests skip disabled steps.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = YamlTestStep(label="Disabled Step", disabled=True) + test_step = MatterTestStep(label="Disabled Step", disabled=True) test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -363,10 +365,10 @@ def test_disabled_steps_for_non_manual_test() -> None: def test_normal_steps_for_non_manual_tests() -> None: """Test that non-manual tests include enabled steps.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = YamlTestStep(label="Step1") + test_step = MatterTestStep(label="Step1") test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -379,10 +381,10 @@ def test_normal_steps_for_non_manual_tests() -> None: def test_multiple_steps_for_non_manual() -> None: """Test that non-manual tests multiple enabled steps are all included.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = YamlTestStep(label="StepN") + test_step = MatterTestStep(label="StepN") no_steps = 5 test = yaml_test_instance(type=type, tests=([test_step] * no_steps)) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -397,13 +399,13 @@ def test_multiple_steps_for_non_manual() -> None: def test_prompt_steps_for_yaml_chip_tool_test_case() -> None: - test_step = YamlTestStep( + test_step = MatterTestStep( label="Step1", command="UserPrompt", verification="Verify that This happened", ) - test = yaml_test_instance(type=YamlTestType.AUTOMATED, tests=[test_step]) + test = yaml_test_instance(type=MatterTestType.AUTOMATED, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) diff --git a/app/tests/yaml_tests/test_yaml_folder.py b/app/tests/yaml_tests/test_yaml_folder.py index 65a5012e..160c537e 100644 --- a/app/tests/yaml_tests/test_yaml_folder.py +++ b/app/tests/yaml_tests/test_yaml_folder.py @@ -16,9 +16,7 @@ from pathlib import Path from unittest import mock -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder import ( - YamlTestFolder, -) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder test_yaml_path = Path("/test/yaml") @@ -29,10 +27,10 @@ def test_yaml_folder_version() -> None: # We mock open to read version_file_content and Path exists to ignore that we're # testing with a fake path with mock.patch( - "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", + "test_collections.sdk_tests.support.models.sdk_test_folder.open", new=mock.mock_open(read_data=version_file_content), ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: - yaml_folder = YamlTestFolder(test_yaml_path) + yaml_folder = SDKTestFolder(test_yaml_path) assert yaml_folder.version == version_file_content @@ -40,20 +38,20 @@ def test_yaml_folder_version() -> None: def test_yaml_folder_version_missing() -> None: expected_version = "Unknown" with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: - yaml_folder = YamlTestFolder(test_yaml_path) + yaml_folder = SDKTestFolder(test_yaml_path) assert yaml_folder.version == expected_version def test_yaml_folder_filename_pattern() -> None: - """Test YamlTestFolder will search for files with filename pattern.""" + """Test SDKTestFolder will search for files with filename pattern.""" with mock.patch.object(target=Path, attribute="glob") as path_glob: # Default file_name_patter: * - yaml_folder = YamlTestFolder(test_yaml_path) - _ = yaml_folder.yaml_file_paths() + yaml_folder = SDKTestFolder(test_yaml_path) + _ = yaml_folder.file_paths(extension=".y*ml") path_glob.assert_called_once_with("*.y*ml") path_glob.reset_mock() pattern = "TC_*" - yaml_folder = YamlTestFolder(test_yaml_path, filename_pattern=pattern) - _ = yaml_folder.yaml_file_paths() + yaml_folder = SDKTestFolder(test_yaml_path, filename_pattern=pattern) + _ = yaml_folder.file_paths(extension=".y*ml") path_glob.assert_called_once_with(f"{pattern}.y*ml") diff --git a/app/tests/yaml_tests/test_yaml_parser.py b/app/tests/yaml_tests/test_yaml_parser.py index 69be5216..dcf040a9 100644 --- a/app/tests/yaml_tests/test_yaml_parser.py +++ b/app/tests/yaml_tests/test_yaml_parser.py @@ -20,9 +20,9 @@ from pydantic.error_wrappers import ValidationError from app.tests.yaml_tests.test_test_case import yaml_test_instance -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( - YamlTestStep, - YamlTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_parser import ( YamlParserException, @@ -97,54 +97,54 @@ def test_yaml_file_parser() -> None: def test_test_type_all_disabled_steps() -> None: - disabled_step = YamlTestStep(label="Disabled Test Step", disabled=True) + disabled_step = MatterTestStep(label="Disabled Test Step", disabled=True) five_disabled_steps_test = yaml_test_instance(tests=[disabled_step] * 5) type = _test_type(five_disabled_steps_test) - assert type == YamlTestType.MANUAL + assert type == MatterTestType.MANUAL # simulated in path overrides test type to simulated five_disabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_disabled_steps_test) - assert type == YamlTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_some_disabled_steps() -> None: - disabled_step = YamlTestStep(label="Disabled Test Step", disabled=True) - enabled_step = YamlTestStep(label="Enabled Test Step", disabled=False) + disabled_step = MatterTestStep(label="Disabled Test Step", disabled=True) + enabled_step = MatterTestStep(label="Enabled Test Step", disabled=False) test = yaml_test_instance(tests=[disabled_step, enabled_step]) type = _test_type(test) - assert type == YamlTestType.AUTOMATED + assert type == MatterTestType.AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == YamlTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_all_enabled_steps_no_prompts() -> None: - enabled_step = YamlTestStep(label="Enabled Test Step") + enabled_step = MatterTestStep(label="Enabled Test Step") five_enabled_steps_test = yaml_test_instance(tests=[enabled_step] * 5) type = _test_type(five_enabled_steps_test) - assert type == YamlTestType.AUTOMATED + assert type == MatterTestType.AUTOMATED # simulated in path overrides test type to simulated five_enabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_enabled_steps_test) - assert type == YamlTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_all_enabled_steps_some_prompts() -> None: - enabled_step = YamlTestStep(label="Enable Test Step") - prompt_step = YamlTestStep(label="Prompt Test Step", command="UserPrompt") + enabled_step = MatterTestStep(label="Enable Test Step") + prompt_step = MatterTestStep(label="Prompt Test Step", command="UserPrompt") test = yaml_test_instance(tests=[enabled_step, prompt_step]) type = _test_type(test) - assert type == YamlTestType.SEMI_AUTOMATED + assert type == MatterTestType.SEMI_AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == YamlTestType.SIMULATED + assert type == MatterTestType.SIMULATED diff --git a/app/tests/yaml_tests/test_yamls/.version b/app/tests/yaml_tests/test_yamls/.version deleted file mode 100644 index 0fc82cc3..00000000 --- a/app/tests/yaml_tests/test_yamls/.version +++ /dev/null @@ -1 +0,0 @@ -unit-test-yaml-version \ No newline at end of file diff --git a/cspell.json b/cspell.json index 28ac7526..5963d04b 100644 --- a/cspell.json +++ b/cspell.json @@ -89,7 +89,9 @@ "isnot", "DBAPI", "repr", - "zinfo" + "zinfo", + "MCORE", + "elts" ], "allowCompoundWords": true, "ignorePaths": [ diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 54e59ed3..16c1b31e 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from .support.python_testing import sdk_python_collection from .support.yaml_tests import custom_collection, sdk_collection diff --git a/test_collections/sdk_tests/support/models/matter_test_models.py b/test_collections/sdk_tests/support/models/matter_test_models.py new file mode 100644 index 00000000..b173e484 --- /dev/null +++ b/test_collections/sdk_tests/support/models/matter_test_models.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from pathlib import Path +from typing import Any, Optional + +from pydantic import BaseModel + +### +# This file declares YAML models that are used to parse the YAML Test Cases. +### + + +class MatterTestType(Enum): + AUTOMATED = 0 + SEMI_AUTOMATED = 1 + MANUAL = 2 + SIMULATED = 3 + + +class MatterTestStep(BaseModel): + label: str + PICS: Optional[str] = None + verification: Optional[str] = None + command: Optional[str] + disabled: bool = False + arguments: Optional[dict[str, Any]] + + +class MatterTest(BaseModel): + name: str + PICS: set[str] = set() + config: dict[str, Any] + steps: list[MatterTestStep] + type: MatterTestType = MatterTestType.MANUAL + path: Optional[Path] diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py b/test_collections/sdk_tests/support/models/sdk_test_folder.py similarity index 73% rename from test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py rename to test_collections/sdk_tests/support/models/sdk_test_folder.py index a9aaa313..93b8e6a6 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py +++ b/test_collections/sdk_tests/support/models/sdk_test_folder.py @@ -21,10 +21,10 @@ VERSION_FILE_FILENAME = ".version" -class YamlTestFolder: - """Representing a folder with Test YAML files. +class SDKTestFolder: + """Representing a folder with SDK Test files. - Note: YAML version is read from .version file in folder on init. + Note: SDK version is read from .version file in folder on init. """ def __init__(self, path: Path, filename_pattern: str = "*") -> None: @@ -43,16 +43,15 @@ def __version(self) -> str: with open(version_file_path, "r") as file: return file.read().rstrip() - def yaml_file_paths(self) -> list[Path]: - """Get list of paths to yaml files in folder. + def file_paths(self, extension: str = "*.*") -> list[Path]: + """Get list of paths in folder. Filename filter can be applied if only some files should be selected. - Note: filter is without extension. Will search for .yml and .yaml files Args: - filename_pattern (str, optional): custom file filter. Defaults to "*". + extension (str): custom file extension filter. Defaults to "*.*". Returns: - list[Path]: list of paths to YAML test files. + list[Path]: list of paths to test files. """ - return list(self.path.glob(self.filename_pattern + ".y*ml")) + return list(self.path.glob(self.filename_pattern + extension)) diff --git a/test_collections/sdk_tests/support/python_testing/__init__.py b/test_collections/sdk_tests/support/python_testing/__init__.py new file mode 100644 index 00000000..138004de --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from app.test_engine.models.test_declarations import TestCollectionDeclaration + +from .sdk_python_tests import sdk_python_test_collection + +# Test engine will auto load TestCollectionDeclarations declared inside the package +# initializer +sdk_python_collection: TestCollectionDeclaration = sdk_python_test_collection() diff --git a/test_collections/sdk_tests/support/python_testing/models/__init__.py b/test_collections/sdk_tests/support/python_testing/models/__init__.py new file mode 100644 index 00000000..d15bf56b --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .test_case import PythonTestCase +from .test_suite import PythonTestSuite, SuiteType diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py new file mode 100644 index 00000000..26c52cf8 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any + +from ...models.matter_test_models import MatterTest, MatterTestType + +### +# This file declares Python test models that are used to parse the Python Test Cases. +### + + +class PythonTest(MatterTest): + description: str + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.type = MatterTestType.AUTOMATED diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py new file mode 100644 index 00000000..da2f2c15 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import ast +from pathlib import Path +from typing import List + +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) + +from .python_test_models import PythonTest + +ARG_STEP_DESCRIPTION_INDEX = 1 +KEYWORD_IS_COMISSIONING_INDEX = 0 +BODY_INDEX = 0 + + +class PythonParserException(Exception): + """Raised when an error occurs during the parser of python file.""" + + +def parse_python_test(path: Path) -> PythonTest: + """Parse a single Python test file into PythonTest model. + + This will also annotate parsed python test with it's path and test type. + + It's expected that the python script files has a class with the same file name, + and also the following methods: + * desc_[class_name] - This method should return a str with the test description + * pics_[class_name] - This method should return a list with the PICS required + for the test case + * steps_[class_name] - This method should return a list with the step description + * test_[class_name] - This method contains the test logic + + Example: file TC_ACE_1_3.py must have the methods desc_TC_ACE_1_3, pics_TC_ACE_1_3, + steps_TC_ACE_1_3 and test_TC_ACE_1_3. + """ + with open(path, "r") as python_file: + parsed_python_file = ast.parse(python_file.read()) + classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] + + tc_name = path.name.split(".")[0] + try: + class_ = next(c for c in classes if tc_name in c.name) + except StopIteration as si: # Raised when `next` doesn't find a matching method + raise PythonParserException(f"{path} must have a class named {tc_name}") from si + + return __parse_test_case_from_class(class_=class_, path=path, tc_name=tc_name) + + +def __parse_test_case_from_class( + class_: ast.ClassDef, path: Path, tc_name: str +) -> PythonTest: + # Currently config is not configured in Python Testing + tc_config: dict = {} + + desc_method_name = "desc_" + tc_name + steps_method_name = "steps_" + tc_name + pics_method_name = "pics_" + tc_name + + methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] + try: + desc_method = next(m for m in methods if desc_method_name in m.name) + tc_desc = desc_method.body[BODY_INDEX].value.value # type: ignore + + steps_method = next(m for m in methods if steps_method_name in m.name) + tc_steps = __retrieve_steps(steps_method) + except StopIteration as si: # Raised when `next` doesn't find a matching method + raise PythonParserException( + f"{path} did not contain valid definition for {tc_name}" + ) from si + + # PICS method is optional + try: + pics_method = next(m for m in methods if pics_method_name in m.name) + tc_pics = __retrieve_pics(pics_method) + except StopIteration: # Raised when `next` doesn't find a matching method + tc_pics = [] + + return PythonTest( + name=tc_name, + description=tc_desc, + steps=tc_steps, + config=tc_config, + PICS=tc_pics, + path=path, + type=MatterTestType.AUTOMATED, + ) + + +def __retrieve_steps(method: ast.FunctionDef) -> List[MatterTestStep]: + python_steps: List[MatterTestStep] = [] + for step in method.body[BODY_INDEX].value.elts: # type: ignore + step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value + arg_is_commissioning = False + if ( + step.keywords + and "is_commissioning" in step.keywords[KEYWORD_IS_COMISSIONING_INDEX].arg + ): + arg_is_commissioning = step.keywords[ + KEYWORD_IS_COMISSIONING_INDEX + ].value.value + + python_steps.append( + MatterTestStep(label=step_name, is_commissioning=arg_is_commissioning) + ) + + return python_steps + + +def __retrieve_pics(method: ast.FunctionDef) -> list: + python_steps: list = [] + for step in method.body[BODY_INDEX].value.elts: # type: ignore + python_steps.append(step.value) + + return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py new file mode 100644 index 00000000..78bd878e --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -0,0 +1,181 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from queue import Empty, Queue +from typing import Any, Optional, Union + +from matter_yamltests.hooks import TestRunnerHooks +from pydantic import BaseModel + + +class SDKPythonTestResultEnum(str, Enum): + START = "start" + STOP = "stop" + TEST_START = "test_start" + TEST_STOP = "test_stop" + STEP_SKIPPED = "step_skipped" + STEP_START = "step_start" + STEP_SUCCESS = "step_success" + STEP_FAILURE = "step_failure" + STEP_UNKNOWN = "step_unknown" + STEP_MANUAL = "step_manual" + + +class SDKPythonTestResultBase(BaseModel): + type: SDKPythonTestResultEnum + + def params_dict(self) -> dict: + return self.dict(exclude={"type"}) + + +class SDKPythonTestResultStart(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.START + count: int + + +class SDKPythonTestResultStop(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STOP + duration: int + + +class SDKPythonTestResultTestStart(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.TEST_START + filename: Optional[str] + name: Optional[str] + count: Optional[int] + + +class SDKPythonTestResultTestStop(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.TEST_STOP + duration: Optional[str] + exception: Any + + +class SDKPythonTestResultStepSkipped(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_SKIPPED + name: Optional[str] + expression: Optional[str] + + +class SDKPythonTestResultStepStart(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_START + name: Optional[str] + + +class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_SUCCESS + logger: Any + logs: Any + duration: int + request: Any + + +class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_FAILURE + logger: Any + logs: Any + duration: int + request: Any + received: Any + + +class SDKPythonTestResultStepUnknown(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_UNKNOWN + + +class SDKPythonTestResultStepManual(SDKPythonTestResultBase): + type = SDKPythonTestResultEnum.STEP_MANUAL + + +class SDKPythonTestRunnerHooks(TestRunnerHooks): + finished = False + results: Queue + + def __init__(self) -> None: + SDKPythonTestRunnerHooks.finished = False + SDKPythonTestRunnerHooks.results = Queue() + + def update_test(self) -> Union[dict, None]: + try: + result = self.results.get(block=False) + return result + except Empty: + return None + + def is_finished(self) -> bool: + return SDKPythonTestRunnerHooks.finished + + def start(self, count: int) -> None: + self.results.put(SDKPythonTestResultStart(count=count)) + + def stop(self, duration: int) -> None: + self.results.put(SDKPythonTestResultStop(duration=duration)) + SDKPythonTestRunnerHooks.finished = True + + def test_start(self, filename: str, name: str, count: int) -> None: + self.results.put( + SDKPythonTestResultTestStart( + filename=filename, + name=name, + count=count, + ) + ) + + def test_stop(self, exception: Exception, duration: int) -> None: + self.results.put( + SDKPythonTestResultTestStop( + exception=exception, + duration=duration, + ) + ) + + def SDKPythonTestResultStepSkipped(self, name: str, expression: str) -> None: + self.results.put(SDKPythonTestResultStepSkipped(expression=expression)) + + def step_start(self, name: str) -> None: + self.results.put(SDKPythonTestResultStepStart(name=name)) + + def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: + self.results.put( + SDKPythonTestResultStepSuccess( + logger=logger, + logs=logs, + duration=duration, + request=request, + ) + ) + + def step_failure( + self, logger: Any, logs: Any, duration: int, request: Any, received: Any + ) -> None: + self.results.put( + SDKPythonTestResultStepFailure( + logger=logger, + logs=logs, + duration=duration, + request=request, + received=received, + ) + ) + + def step_unknown(self) -> None: + self.results.put(SDKPythonTestResultStepUnknown()) + + def step_manual(self) -> None: + self.results.put(SDKPythonTestResultStepManual()) + + def step_start_list(self) -> None: + pass diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py new file mode 100644 index 00000000..57eb0757 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# type: ignore +# flake8: noqa + +import importlib +import sys +from multiprocessing.managers import BaseManager + +import matter_testing_support + + +class TestRunnerHooks: + pass + + +def main() -> None: + # Load python_testing as a module. This folder is where all python script is located + sys.path.append("/root/python_testing") + test_name = sys.argv[1] + + config_options = sys.argv[2:] + config = matter_testing_support.parse_matter_test_args(config_options) + + if config is None: + raise ValueError(f"Not a valid test id: {test_name}") + + module = importlib.import_module(test_name) + TestClassReference = getattr(module, test_name) + + BaseManager.register(TestRunnerHooks.__name__) + manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") + manager.connect() + test_runner_hooks = manager.TestRunnerHooks() # shared object proxy # type: ignore + + matter_testing_support.run_tests(TestClassReference, config, test_runner_hooks) + + +if __name__ == "__main__": + main() diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py new file mode 100644 index 00000000..0dff2cd1 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -0,0 +1,200 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import re +from asyncio import sleep +from multiprocessing.managers import BaseManager +from typing import Any, Type, TypeVar + +from app.chip_tool.chip_tool import PICS_FILE_PATH, ChipTool +from app.models import TestCaseExecution +from app.test_engine.logger import test_engine_logger as logger +from app.test_engine.models import TestCase, TestStep + +from .python_test_models import PythonTest +from .python_testing_hooks_proxy import ( + SDKPythonTestResultBase, + SDKPythonTestRunnerHooks, +) + +# Custom type variable used to annotate the factory method in PythonTestCase. +T = TypeVar("T", bound="PythonTestCase") + +# Command line params +RUNNER_CLASS = "test_harness_client.py" +RUNNER_CLASS_PATH = "/root/python_testing/" +EXECUTABLE = "python3" + + +class PythonTestCase(TestCase): + """Base class for all Python Test based test cases. + + This class provides a class factory that will dynamically declare a new sub-class + based on the test-type the Python test is expressing. + + The PythonTest will be stored as a class property that will be used at run-time + in all instances of such subclass. + """ + + python_test: PythonTest + python_test_version: str + + def __init__(self, test_case_execution: TestCaseExecution) -> None: + super().__init__(test_case_execution=test_case_execution) + self.chip_tool: ChipTool = ChipTool(logger) + self.__runned = 0 + self.test_stop_called = False + + def start(self, count: int) -> None: + pass + + def stop(self, duration: int) -> None: + if not self.test_stop_called: + self.current_test_step.mark_as_completed() + + def test_start(self, filename: str, name: str, count: int) -> None: + self.next_step() + + def test_stop(self, exception: Exception, duration: int) -> None: + self.test_stop_called = True + self.current_test_step.mark_as_completed() + + def step_skipped(self, name: str, expression: str) -> None: + self.current_test_step.mark_as_not_applicable("Test step skipped") + self.next_step() + + def step_start(self, name: str) -> None: + pass + + def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: + # TODO Handle Logs properly + self.next_step() + + def step_failure( + self, logger: Any, logs: str, duration: int, request: Any, received: Any + ) -> None: + # TODO Handle Logs properly + self.mark_step_failure("Python test step failure") + self.next_step() + + def step_unknown(self) -> None: + self.__runned += 1 + + @classmethod + def pics(cls) -> set[str]: + """Test Case level PICS. Read directly from parsed Python Test.""" + return cls.python_test.PICS + + @classmethod + def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: + """class factory method for PythonTestCase.""" + title = cls.__title(test.name) + class_name = cls.__class_name(test.name) + + return type( + class_name, + (cls,), + { + "python_test": test, + "python_test_version": python_test_version, + "metadata": { + "public_id": test.name, + "version": "0.0.1", + "title": title, + "description": test.description, + }, + }, + ) + + @staticmethod + def __class_name(identifier: str) -> str: + """Replace all non-alphanumeric characters with _ to make valid class name.""" + return re.sub("[^0-9a-zA-Z]+", "_", identifier) + + @staticmethod + def __title(identifier: str) -> str: + """Retrieve the test title in format TC_ABC_1_2""" + title: str = "" + elements = identifier.split("_") + + if len(elements) > 2: + title = "-".join(elements[0:2]) + "-" + ".".join(elements[2:]) + else: + title = identifier.replace("_", "-") + + return title + + async def setup(self) -> None: + logger.info("Test Setup") + + async def cleanup(self) -> None: + logger.info("Test Cleanup") + + async def execute(self) -> None: + try: + logger.info("Running Python Test: " + self.python_test.name) + + BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) + manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") + manager.start() + test_runner_hooks = manager.TestRunnerHooks() # type: ignore + + runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS + command = ( + f"{runner_class} {self.python_test.name}" + " --commissioning-method on-network --discriminator 3840 --passcode" + " 20202021 --storage-path /root/admin_storage.json" + " --paa-trust-store-path /paa-root-certs" + ) + + if self.chip_tool.pics_file_created: + command += f" --PICS {PICS_FILE_PATH}" + + # TODO Ignoring stream from docker execution + self.chip_tool.send_command( + command, + prefix=EXECUTABLE, + is_stream=True, + is_socket=False, + ).output + + while ((update := test_runner_hooks.update_test()) is not None) or ( + not test_runner_hooks.is_finished() + ): + if not update: + await sleep(0.0001) + continue + + self.__handle_update(update) + + finally: + pass + + def __handle_update(self, update: SDKPythonTestResultBase) -> None: + self.__call_function_from_name(update.type.value, update.params_dict()) + + def __call_function_from_name(self, func_name: str, kwargs: Any) -> None: + func = getattr(self, func_name, None) + if not func: + raise AttributeError(f"{func_name} is not a method of {self}") + if not callable(func): + raise TypeError(f"{func_name} is not callable") + func(**kwargs) + + def create_test_steps(self) -> None: + self.test_steps = [TestStep("Start Python test")] + for step in self.python_test.steps: + python_test_step = TestStep(step.label) + self.test_steps.append(python_test_step) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py new file mode 100644 index 00000000..62e8560a --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Type + +from app.test_engine.models.test_declarations import ( + TestCaseDeclaration, + TestCollectionDeclaration, + TestSuiteDeclaration, +) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder + +from .python_test_models import MatterTestType, PythonTest +from .test_case import PythonTestCase +from .test_suite import PythonTestSuite, SuiteType + + +class PythonCollectionDeclaration(TestCollectionDeclaration): + def __init__(self, folder: SDKTestFolder, name: str) -> None: + super().__init__(path=str(folder.path), name=name) + self.python_test_version = folder.version + + +class PythonSuiteDeclaration(TestSuiteDeclaration): + """Direct initialization for Python Test Suite.""" + + class_ref: Type[PythonTestSuite] + + def __init__(self, name: str, suite_type: SuiteType, version: str) -> None: + super().__init__( + PythonTestSuite.class_factory( + name=name, + suite_type=suite_type, + python_test_version=version, + ) + ) + + +class PythonCaseDeclaration(TestCaseDeclaration): + """Direct initialization for Python Test Case.""" + + class_ref: Type[PythonTestCase] + + def __init__(self, test: PythonTest, python_test_version: str) -> None: + super().__init__( + PythonTestCase.class_factory( + test=test, python_test_version=python_test_version + ) + ) + + @property + def test_type(self) -> MatterTestType: + return self.class_ref.python_test.type diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py new file mode 100644 index 00000000..18e3e44c --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from typing import Type, TypeVar + +from app.chip_tool import ChipTool +from app.test_engine.logger import test_engine_logger as logger +from app.test_engine.models import TestSuite + + +class SuiteType(Enum): + AUTOMATED = 1 + + +# Custom Type variable used to annotate the factory methods of classmethod. +T = TypeVar("T", bound="PythonTestSuite") + + +class PythonTestSuite(TestSuite): + """Base class for all Python tests based test suites. + + This class provides a class factory that will dynamically declare a new sub-class + based on the suite-type. + """ + + python_test_version: str + suite_name: str + chip_tool: ChipTool = ChipTool(logger) + + @classmethod + def class_factory( + cls, suite_type: SuiteType, name: str, python_test_version: str + ) -> Type[T]: + """Dynamically declares a subclass based on the type of test suite.""" + return type( + name, + (cls,), + { + "name": name, + "python_test_version": python_test_version, + "metadata": { + "public_id": name, + "version": "0.0.1", + "title": name, + "description": name, + }, + }, + ) + + async def setup(self) -> None: + """Override Setup to log Python Test version and set PICS.""" + logger.info("Suite Setup") + logger.info(f"Python Test Version: {self.python_test_version}") + + logger.info("Starting SDK container") + await self.chip_tool.start_container() + + if len(self.pics.clusters) > 0: + logger.info("Create PICS file for DUT") + self.chip_tool.set_pics(pics=self.pics, in_container=True) + else: + self.chip_tool.reset_pics_state() + + async def cleanup(self) -> None: + logger.info("Suite Cleanup") + + logger.info("Stopping SDK container") + await self.chip_tool.destroy_device() diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_tests.py new file mode 100644 index 00000000..b31a04aa --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_tests.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +from loguru import logger + +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder +from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH + +from .models.python_test_parser import PythonParserException, parse_python_test +from .models.test_declarations import ( + PythonCaseDeclaration, + PythonCollectionDeclaration, + PythonSuiteDeclaration, +) +from .models.test_suite import SuiteType + +### +# This file hosts logic to load and parse Python test cases, located in +# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. +# The `sdk` sub-folder here is automatically maintained using the +# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. +# +# The Python Tests are organized into 1 Test Suite: +# - Automated +### + +SDK_PYTHON_TEST_PATH = SDK_CHECKOUT_PATH / Path("python_testing/scripts/sdk") +SDK_PYTHON_TEST_FOLDER = SDKTestFolder( + path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*" +) + + +def _init_test_suites( + python_test_version: str, +) -> dict[SuiteType, PythonSuiteDeclaration]: + return { + SuiteType.AUTOMATED: PythonSuiteDeclaration( + name="Python Testing Suite", + suite_type=SuiteType.AUTOMATED, + version=python_test_version, + ), + } + + +def _parse_python_test_to_test_case_declaration( + python_test_path: Path, python_test_version: str +) -> PythonCaseDeclaration: + python_test = parse_python_test(python_test_path) + return PythonCaseDeclaration( + test=python_test, python_test_version=python_test_version + ) + + +def _parse_all_sdk_python_tests( + python_test_files: list[Path], python_test_version: str +) -> list[PythonSuiteDeclaration]: + """Parse all python test files and add them into Automated Suite""" + suites = _init_test_suites(python_test_version) + + for python_test_file in python_test_files: + try: + test_case = _parse_python_test_to_test_case_declaration( + python_test_path=python_test_file, + python_test_version=python_test_version, + ) + + suites[SuiteType.AUTOMATED].add_test_case(test_case) + except PythonParserException as e: + # If an exception was raised during parse process, the python file will be + # ignored and the loop will continue with the next file + logger.error( + f"Error while parsing Python File: {python_test_file} \nError:{e}" + ) + + return list(suites.values()) + + +def sdk_python_test_collection( + python_test_folder: SDKTestFolder = SDK_PYTHON_TEST_FOLDER, +) -> PythonCollectionDeclaration: + """Declare a new collection of test suites.""" + collection = PythonCollectionDeclaration( + name="SDK Python Tests", folder=python_test_folder + ) + + files = python_test_folder.file_paths(extension=".py") + version = python_test_folder.version + suites = _parse_all_sdk_python_tests( + python_test_files=files, python_test_version=version + ) + + for suite in suites: + suite.sort_test_cases() + collection.add_test_suite(suite) + + return collection diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py index 03697248..b0ddf9a2 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py @@ -25,8 +25,12 @@ TestCase, TestStep, ) +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) -from .yaml_test_models import YamlTest, YamlTestStep, YamlTestType +from .yaml_test_models import YamlTest # Custom type variable used to annotate the factory method in YamlTestCase. T = TypeVar("T", bound="YamlTestCase") @@ -76,11 +80,11 @@ async def setup(self) -> None: def class_factory(cls, test: YamlTest, yaml_version: str) -> Type[T]: """Dynamically declares a subclass based on the type of YAML test.""" case_class: Type[YamlTestCase] - if test.type == YamlTestType.MANUAL: + if test.type == MatterTestType.MANUAL: case_class = YamlManualTestCase - elif test.type == YamlTestType.SEMI_AUTOMATED: + elif test.type == MatterTestType.SEMI_AUTOMATED: case_class = YamlSemiAutomatedChipToolTestCase - elif test.type == YamlTestType.SIMULATED: + elif test.type == MatterTestType.SIMULATED: case_class = YamlSimulatedTestCase else: # Automated case_class = YamlChipToolTestCase @@ -143,7 +147,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: """ title = identifier - if test_yaml.type == YamlTestType.SEMI_AUTOMATED: + if test_yaml.type == MatterTestType.SEMI_AUTOMATED: title += " (Semi-automated)" if cls.__has_steps_disabled(test_yaml): @@ -151,7 +155,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: return title - def _append_automated_test_step(self, yaml_step: YamlTestStep) -> None: + def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: """ Disabled steps are ignored. (Such tests will be marked as 'Steps Disabled' elsewhere) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py index e4ab474d..fbe36139 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py @@ -20,15 +20,16 @@ TestCollectionDeclaration, TestSuiteDeclaration, ) +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from .test_case import YamlTestCase from .test_suite import SuiteType, YamlTestSuite -from .yaml_test_folder import YamlTestFolder -from .yaml_test_models import YamlTest, YamlTestType +from .yaml_test_models import YamlTest class YamlCollectionDeclaration(TestCollectionDeclaration): - def __init__(self, folder: YamlTestFolder, name: str) -> None: + def __init__(self, folder: SDKTestFolder, name: str) -> None: super().__init__(path=str(folder.path), name=name) self.yaml_version = folder.version @@ -59,5 +60,5 @@ def __init__(self, test: YamlTest, yaml_version: str) -> None: ) @property - def test_type(self) -> YamlTestType: + def test_type(self) -> MatterTestType: return self.class_ref.yaml_test.type diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py index ff59cff9..65918eee 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py @@ -13,38 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from enum import Enum -from pathlib import Path -from typing import Any, Optional +from typing import Any -from pydantic import BaseModel, Field from pydantic_yaml import YamlModelMixin +from ...models.matter_test_models import MatterTest + ### # This file declares YAML models that are used to parse the YAML Test Cases. ### -class YamlTestType(Enum): - AUTOMATED = 0 - SEMI_AUTOMATED = 1 - MANUAL = 2 - SIMULATED = 3 - - -class YamlTestStep(YamlModelMixin, BaseModel): - label: str - PICS: Optional[str] = None - verification: Optional[str] = None - command: Optional[str] - disabled: bool = False - arguments: Optional[dict[str, Any]] - - -class YamlTest(YamlModelMixin, BaseModel): - name: str - PICS: set[str] = set() - config: dict[str, Any] - steps: list[YamlTestStep] = Field(alias="tests") - type: YamlTestType = YamlTestType.MANUAL - path: Optional[Path] +class YamlTest(YamlModelMixin, MatterTest): + def __init__(self, **kwargs: Any) -> None: + super().__init__(steps=kwargs["tests"], **kwargs) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py index 007148ef..f4275660 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py @@ -18,14 +18,16 @@ from loguru import logger from pydantic import ValidationError -from .yaml_test_models import YamlTest, YamlTestType +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType + +from .yaml_test_models import YamlTest class YamlParserException(Exception): """Raised when an error occurs during the parser of yaml file.""" -def _test_type(test: YamlTest) -> YamlTestType: +def _test_type(test: YamlTest) -> MatterTestType: """Determine the type of a test based on the parsed yaml. This is mainly determined by the number of disabled test steps. @@ -41,20 +43,20 @@ def _test_type(test: YamlTest) -> YamlTestType: - Simulated: Tests where file name have "Simulated" """ if test.path is not None and "Simulated" in str(test.path): - return YamlTestType.SIMULATED + return MatterTestType.SIMULATED steps = test.steps # If all disabled: if all(s.disabled is True for s in steps): - return YamlTestType.MANUAL + return MatterTestType.MANUAL # if any step has a UserPrompt, categorize as semi-automated if any(s.command == "UserPrompt" for s in steps): - return YamlTestType.SEMI_AUTOMATED + return MatterTestType.SEMI_AUTOMATED # Otherwise Automated - return YamlTestType.AUTOMATED + return MatterTestType.AUTOMATED def parse_yaml_test(path: Path) -> YamlTest: diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index b7f05dd7..ff751996 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -18,6 +18,8 @@ from loguru import logger +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH from .models.test_declarations import ( @@ -26,8 +28,6 @@ YamlSuiteDeclaration, ) from .models.test_suite import SuiteType -from .models.yaml_test_folder import YamlTestFolder -from .models.yaml_test_models import YamlTestType from .models.yaml_test_parser import YamlParserException, parse_yaml_test ### @@ -43,10 +43,10 @@ YAML_PATH = SDK_CHECKOUT_PATH / "yaml_tests/yaml" SDK_YAML_PATH = YAML_PATH / "sdk" -SDK_YAML_TEST_FOLDER = YamlTestFolder(path=SDK_YAML_PATH, filename_pattern="Test_TC*") +SDK_YAML_TEST_FOLDER = SDKTestFolder(path=SDK_YAML_PATH, filename_pattern="Test_TC*") CUSTOM_YAML_PATH = YAML_PATH / "custom" -CUSTOM_YAML_TEST_FOLDER = YamlTestFolder( +CUSTOM_YAML_TEST_FOLDER = SDKTestFolder( path=CUSTOM_YAML_PATH, filename_pattern="Test_TC*" ) @@ -94,9 +94,9 @@ def _parse_all_yaml( yaml_path=yaml_file, yaml_version=yaml_version ) - if test_case.test_type == YamlTestType.MANUAL: + if test_case.test_type == MatterTestType.MANUAL: suites[SuiteType.MANUAL].add_test_case(test_case) - elif test_case.test_type == YamlTestType.SIMULATED: + elif test_case.test_type == MatterTestType.SIMULATED: suites[SuiteType.SIMULATED].add_test_case(test_case) else: suites[SuiteType.AUTOMATED].add_test_case(test_case) @@ -109,14 +109,14 @@ def _parse_all_yaml( def sdk_yaml_test_collection( - yaml_test_folder: YamlTestFolder = SDK_YAML_TEST_FOLDER, + yaml_test_folder: SDKTestFolder = SDK_YAML_TEST_FOLDER, ) -> YamlCollectionDeclaration: """Declare a new collection of test suites with the 3 test suites.""" collection = YamlCollectionDeclaration( name="SDK YAML Tests", folder=yaml_test_folder ) - files = yaml_test_folder.yaml_file_paths() + files = yaml_test_folder.file_paths(extension=".y*ml") version = yaml_test_folder.version suites = _parse_all_yaml(yaml_files=files, yaml_version=version) @@ -128,14 +128,14 @@ def sdk_yaml_test_collection( def custom_yaml_test_collection( - yaml_test_folder: YamlTestFolder = CUSTOM_YAML_TEST_FOLDER, + yaml_test_folder: SDKTestFolder = CUSTOM_YAML_TEST_FOLDER, ) -> Optional[YamlCollectionDeclaration]: """Declare a new collection of test suites.""" collection = YamlCollectionDeclaration( name="Custom YAML Tests", folder=yaml_test_folder ) - files = yaml_test_folder.yaml_file_paths() + files = yaml_test_folder.file_paths(extension=".y*ml") suites = _parse_all_yaml(yaml_files=files, yaml_version="custom") for suite in suites: